From 54fcb7c36446ae7205448cdba01b1db58286530d Mon Sep 17 00:00:00 2001 From: Andrew Barnes Date: Sun, 15 Mar 2026 00:14:54 -0400 Subject: [PATCH 01/33] test(ai): add _normalize_credentials edge case coverage --- tests/ai/test_ai_mcp.py | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/tests/ai/test_ai_mcp.py b/tests/ai/test_ai_mcp.py index 4a9e308..b2c4a71 100644 --- a/tests/ai/test_ai_mcp.py +++ b/tests/ai/test_ai_mcp.py @@ -1,7 +1,7 @@ import pytest import pytest_httpbin -from scrapling.core.ai import ScraplingMCPServer, ResponseModel +from scrapling.core.ai import ScraplingMCPServer, ResponseModel, _normalize_credentials @pytest_httpbin.use_class_based_httpbin @@ -56,3 +56,25 @@ class TestMCPServer: """Test the bulk_stealthy_fetch tool method""" result = await server.bulk_stealthy_fetch(urls=(test_url, test_url), headless=True) assert all(isinstance(r, ResponseModel) for r in result) + + +class TestNormalizeCredentials: + """Test the _normalize_credentials helper""" + + def test_none_returns_none(self): + assert _normalize_credentials(None) is None + + def test_empty_dict_returns_none(self): + assert _normalize_credentials({}) is None + + def test_valid_credentials_returns_tuple(self): + result = _normalize_credentials({"username": "user", "password": "pass"}) + assert result == ("user", "pass") + + def test_missing_password_raises(self): + with pytest.raises(ValueError, match="password"): + _normalize_credentials({"username": "user"}) + + def test_missing_username_raises(self): + with pytest.raises(ValueError, match="username"): + _normalize_credentials({"password": "pass"}) From 297423410db7a94730737dc1e0259b32aea8e71a Mon Sep 17 00:00:00 2001 From: haosenwang1018 <1293965075@qq.com> Date: Sun, 15 Mar 2026 17:27:10 +0800 Subject: [PATCH 02/33] test: add save/retrieve round-trip and hash/URL coverage for storage core MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The adaptive scraping feature relies on SQLiteStorageSystem to persist and relocate elements, but the existing tests only verified object creation — not the actual save/retrieve workflow. This adds: - _get_base_url: None, empty, valid URL, and case-normalization paths - _get_hash: determinism, uniqueness, strip/lowercase, length suffix - save/retrieve round-trip: basic, overwrite (upsert), nonexistent key - URL-based isolation between different websites - element_to_dict: with/without text, attributes, whitespace filtering - _get_element_path: nested and root element paths - Thread safety: 20 concurrent saves with result verification Co-Authored-By: Claude Opus 4.6 (1M context) --- tests/core/test_storage_core.py | 265 +++++++++++++++++++++++++++++++- 1 file changed, 258 insertions(+), 7 deletions(-) diff --git a/tests/core/test_storage_core.py b/tests/core/test_storage_core.py index 1827294..21de243 100644 --- a/tests/core/test_storage_core.py +++ b/tests/core/test_storage_core.py @@ -1,18 +1,77 @@ import tempfile import os +import threading -from scrapling.core.storage import SQLiteStorageSystem +from lxml.html import fromstring + +from scrapling.core.storage import SQLiteStorageSystem, StorageSystemMixin +from scrapling.core.utils import _StorageTools + + +class TestGetBaseUrl: + """Test StorageSystemMixin._get_base_url()""" + + def _make_storage(self, url=None): + # Clear lru_cache between tests to avoid cross-test pollution + StorageSystemMixin._get_base_url.cache_clear() + return SQLiteStorageSystem(storage_file=":memory:", url=url) + + def test_returns_default_when_url_is_none(self): + storage = self._make_storage(url=None) + assert storage._get_base_url() == "default" + + def test_returns_default_when_url_is_empty(self): + storage = self._make_storage(url="") + assert storage._get_base_url() == "default" + + def test_returns_fld_for_valid_url(self): + storage = self._make_storage(url="https://www.example.com/page") + result = storage._get_base_url() + assert result == "example.com" + + def test_url_is_lowercased(self): + storage = self._make_storage(url="https://WWW.EXAMPLE.COM/Page") + assert storage.url == "https://www.example.com/page" + + +class TestGetHash: + """Test StorageSystemMixin._get_hash()""" + + def setup_method(self): + StorageSystemMixin._get_hash.cache_clear() + + def test_deterministic_output(self): + h1 = StorageSystemMixin._get_hash("test-identifier") + h2 = StorageSystemMixin._get_hash("test-identifier") + assert h1 == h2 + + def test_different_input_different_output(self): + h1 = StorageSystemMixin._get_hash("identifier-a") + h2 = StorageSystemMixin._get_hash("identifier-b") + assert h1 != h2 + + def test_strips_and_lowercases(self): + h1 = StorageSystemMixin._get_hash(" Hello ") + h2 = StorageSystemMixin._get_hash("hello") + assert h1 == h2 + + def test_includes_length_suffix(self): + result = StorageSystemMixin._get_hash("test") + # Format: {sha256_hex}_{byte_length} + assert "_" in result + hex_part, length_part = result.rsplit("_", 1) + assert len(hex_part) == 64 # SHA-256 hex length + assert length_part == str(len("test".encode("utf-8"))) class TestSQLiteStorageSystem: """Test SQLiteStorageSystem functionality""" - + def test_sqlite_storage_creation(self): """Test SQLite storage system creation""" - # Use an in-memory database for testing storage = SQLiteStorageSystem(storage_file=":memory:") assert storage is not None - + def test_sqlite_storage_with_file(self): """Test SQLite storage with an actual file""" with tempfile.NamedTemporaryFile(suffix='.db', delete=False) as tmp_file: @@ -24,18 +83,210 @@ class TestSQLiteStorageSystem: assert storage is not None assert os.path.exists(db_path) finally: - # Close the database connection before deleting (required on Windows) if storage is not None: storage.close() if os.path.exists(db_path): os.unlink(db_path) - + def test_sqlite_storage_initialization_args(self): """Test SQLite storage with various initialization arguments""" - # Test with URL parameter storage = SQLiteStorageSystem( storage_file=":memory:", url="https://example.com" ) assert storage is not None assert storage.url == "https://example.com" + + +class TestSaveRetrieveRoundTrip: + """Test the save/retrieve round-trip — the core of the adaptive feature.""" + + def _make_storage(self, url="https://example.com"): + StorageSystemMixin._get_base_url.cache_clear() + SQLiteStorageSystem.cache_clear() + return SQLiteStorageSystem(storage_file=":memory:", url=url) + + def _make_element(self, html_str="

Hello

"): + tree = fromstring(html_str) + return tree.cssselect("p")[0] if tree.cssselect("p") else tree + + def test_save_and_retrieve(self): + storage = self._make_storage() + element = self._make_element() + storage.save(element, "test-element") + + result = storage.retrieve("test-element") + assert result is not None + assert result["tag"] == "p" + assert result["attributes"]["id"] == "target" + assert result["attributes"]["class"] == "main" + assert result["text"] == "Hello" + + def test_retrieve_nonexistent_returns_none(self): + storage = self._make_storage() + assert storage.retrieve("does-not-exist") is None + + def test_save_overwrites_existing(self): + storage = self._make_storage() + elem1 = self._make_element("

First

") + elem2 = self._make_element("

Second

") + + storage.save(elem1, "my-element") + storage.save(elem2, "my-element") + + result = storage.retrieve("my-element") + assert result is not None + assert result["attributes"]["id"] == "v2" + assert result["text"] == "Second" + + def test_url_isolation(self): + """Elements saved under one URL should not be retrievable under another.""" + SQLiteStorageSystem.cache_clear() + StorageSystemMixin._get_base_url.cache_clear() + + # Use file-based storage so both instances share the same DB + with tempfile.NamedTemporaryFile(suffix='.db', delete=False) as tmp: + db_path = tmp.name + + try: + storage_a = SQLiteStorageSystem(storage_file=db_path, url="https://site-a.com") + element = self._make_element() + storage_a.save(element, "shared-id") + + SQLiteStorageSystem.cache_clear() + StorageSystemMixin._get_base_url.cache_clear() + + storage_b = SQLiteStorageSystem(storage_file=db_path, url="https://site-b.com") + assert storage_b.retrieve("shared-id") is None + finally: + storage_a.close() + storage_b.close() + if os.path.exists(db_path): + os.unlink(db_path) + + def test_element_path_is_stored(self): + storage = self._make_storage() + element = self._make_element("

Text

") + storage.save(element, "path-test") + + result = storage.retrieve("path-test") + assert result is not None + assert "path" in result + # Path should be a list of tag names from root to element + assert result["path"][-1] == "p" + + def test_element_without_parent(self): + """A root element (no parent) should still be savable.""" + storage = self._make_storage() + tree = fromstring("
Root element
") + storage.save(tree, "root-elem") + + result = storage.retrieve("root-elem") + assert result is not None + assert "parent_name" not in result + + def test_element_with_children_and_siblings(self): + storage = self._make_storage() + html_str = "

Sibling

ChildChild2
" + tree = fromstring(html_str) + element = tree.cssselect("#target")[0] + storage.save(element, "with-children") + + result = storage.retrieve("with-children") + assert result is not None + assert "children" in result + assert "b" in result["children"] + assert "i" in result["children"] + assert "siblings" in result + assert "p" in result["siblings"] + + +class TestStorageThreadSafety: + """Test that SQLiteStorageSystem is safe under concurrent access.""" + + def test_concurrent_saves(self): + SQLiteStorageSystem.cache_clear() + StorageSystemMixin._get_base_url.cache_clear() + + with tempfile.NamedTemporaryFile(suffix='.db', delete=False) as tmp: + db_path = tmp.name + + storage = SQLiteStorageSystem(storage_file=db_path, url="https://example.com") + errors = [] + + def save_element(idx): + try: + html_str = f"

Text {idx}

" + tree = fromstring(html_str) + element = tree.cssselect("p")[0] + storage.save(element, f"element-{idx}") + except Exception as e: + errors.append(e) + + threads = [threading.Thread(target=save_element, args=(i,)) for i in range(20)] + for t in threads: + t.start() + for t in threads: + t.join() + + assert len(errors) == 0, f"Thread safety errors: {errors}" + + # Verify all elements were saved + for i in range(20): + result = storage.retrieve(f"element-{i}") + assert result is not None, f"element-{i} not found after concurrent save" + + storage.close() + if os.path.exists(db_path): + os.unlink(db_path) + + +class TestStorageToolsElementToDict: + """Test _StorageTools.element_to_dict() directly.""" + + def test_basic_element(self): + tree = fromstring("

Hello

") + elem = tree.cssselect("p")[0] + result = _StorageTools.element_to_dict(elem) + + assert result["tag"] == "p" + assert result["attributes"]["class"] == "foo" + assert result["text"] == "Hello" + assert "parent_name" in result + assert result["parent_name"] == "div" + + def test_element_no_text(self): + tree = fromstring("

") + elem = tree.cssselect("p")[0] + result = _StorageTools.element_to_dict(elem) + assert result["text"] is None + + def test_element_no_attributes(self): + tree = fromstring("

Plain

") + elem = tree.cssselect("p")[0] + result = _StorageTools.element_to_dict(elem) + assert result["attributes"] == {} + + def test_element_strips_whitespace_attributes(self): + tree = fromstring('

') + elem = tree.cssselect("p")[0] + result = _StorageTools.element_to_dict(elem) + # Whitespace-only attribute values should be filtered out + assert "data-val" not in result["attributes"] + + +class TestStorageToolsGetElementPath: + """Test _StorageTools._get_element_path().""" + + def test_nested_path(self): + tree = fromstring("

Text

") + elem = tree.cssselect("p")[0] + path = _StorageTools._get_element_path(elem) + assert path[-1] == "p" + assert "div" in path + assert "body" in path + + def test_root_element_path(self): + tree = fromstring("
Root
") + path = _StorageTools._get_element_path(tree) + assert path == ("div",) From 7178279586749a802337b2426b95218b13712775 Mon Sep 17 00:00:00 2001 From: haosenwang1018 <1293965075@qq.com> Date: Sun, 15 Mar 2026 17:27:55 +0800 Subject: [PATCH 03/33] test: add coverage for TextHandler regex, clean, and TextHandlers.re() Several critical code paths in custom_types.py lacked test coverage: - TextHandler.re(check_match=True): returns bool, not TextHandlers - TextHandler.re(replace_entities=False): entity preservation path - TextHandler.re() with capture groups: flatten behavior - TextHandler.re_first() default value when no match - TextHandler.clean(remove_entities=True): entity replacement path - TextHandler.json() valid and invalid input - TextHandlers.re(): list-level regex with result flattening - TextHandlers.extract()/get_all(): identity return Co-Authored-By: Claude Opus 4.6 (1M context) --- tests/parser/test_parser_advanced.py | 93 ++++++++++++++++++++++++++++ 1 file changed, 93 insertions(+) diff --git a/tests/parser/test_parser_advanced.py b/tests/parser/test_parser_advanced.py index f34ba5c..969ccc3 100644 --- a/tests/parser/test_parser_advanced.py +++ b/tests/parser/test_parser_advanced.py @@ -250,6 +250,68 @@ class TestTextHandlerAdvanced: matches = text3.re(r"He l lo", clean_match=True, case_sensitive=False) assert len(matches) == 1 + def test_text_handler_regex_check_match(self): + """Test TextHandler.re() with check_match=True returns bool""" + text = TextHandler("Price: $10.99") + assert text.re(r"\$[\d.]+", check_match=True) is True + assert text.re(r"no-match-pattern", check_match=True) is False + + def test_text_handler_regex_replace_entities_false(self): + """Test TextHandler.re() with replace_entities=False preserves entities""" + text = TextHandler("Hello & World") + results = text.re(r"&", replace_entities=False) + assert len(results) == 1 + assert results[0] == "&" + + def test_text_handler_regex_with_groups(self): + """Test TextHandler.re() with capture groups flattens results""" + text = TextHandler("name=Alice age=30 name=Bob age=25") + results = text.re(r"name=(\w+) age=(\d+)") + assert len(results) == 4 + assert "Alice" in results + assert "30" in results + + def test_text_handler_re_first_with_default(self): + """Test TextHandler.re_first() returns default when no match""" + text = TextHandler("no numbers here") + result = text.re_first(r"\d+", default="N/A") + assert result == "N/A" + + def test_text_handler_re_first_returns_first_match(self): + """Test TextHandler.re_first() returns first match""" + text = TextHandler("a1 b2 c3") + result = text.re_first(r"\d") + assert result == "1" + assert isinstance(result, TextHandler) + + def test_text_handler_clean_with_entities(self): + """Test TextHandler.clean() with remove_entities=True""" + text = TextHandler("Hello\t&\nWorld") + cleaned = text.clean(remove_entities=True) + assert "&" not in cleaned + assert "&" in cleaned + assert "\t" not in cleaned + assert "\n" not in cleaned + + def test_text_handler_clean_without_entities(self): + """Test TextHandler.clean() preserves entities by default""" + text = TextHandler("Hello\t&\nWorld") + cleaned = text.clean(remove_entities=False) + assert "&" in cleaned + + def test_text_handler_json_valid(self): + """Test TextHandler.json() with valid JSON""" + text = TextHandler('{"key": "value", "num": 42}') + data = text.json() + assert data["key"] == "value" + assert data["num"] == 42 + + def test_text_handler_json_invalid(self): + """Test TextHandler.json() raises on invalid JSON""" + text = TextHandler("not json") + with pytest.raises(Exception): + text.json() + def test_text_handlers_operations(self): """Test TextHandlers list operations""" handlers = TextHandlers([ @@ -266,6 +328,37 @@ class TestTextHandlerAdvanced: assert handlers.get("default") == "First" assert TextHandlers([]).get("default") == "default" + def test_text_handlers_re(self): + """Test TextHandlers.re() flattens results across all elements""" + handlers = TextHandlers([ + TextHandler("a1 b2"), + TextHandler("c3 d4"), + ]) + results = handlers.re(r"[a-z]\d") + assert isinstance(results, TextHandlers) + assert len(results) == 4 + assert results[0] == "a1" + assert results[3] == "d4" + + def test_text_handlers_re_empty(self): + """Test TextHandlers.re() on empty list""" + handlers = TextHandlers([]) + results = handlers.re(r"\d+") + assert isinstance(results, TextHandlers) + assert len(results) == 0 + + def test_text_handlers_re_no_matches(self): + """Test TextHandlers.re() when no element matches""" + handlers = TextHandlers([TextHandler("abc"), TextHandler("def")]) + results = handlers.re(r"\d+") + assert len(results) == 0 + + def test_text_handlers_extract(self): + """Test TextHandlers.extract() returns self""" + handlers = TextHandlers([TextHandler("a"), TextHandler("b")]) + assert handlers.extract() is handlers + assert handlers.get_all() is handlers + class TestSelectorsAdvanced: """Test advanced Selectors functionality""" From a31763afdeaec2f4cfd4b402e81692cd301b4c0a Mon Sep 17 00:00:00 2001 From: haosenwang1018 <1293965075@qq.com> Date: Sun, 15 Mar 2026 17:52:57 +0800 Subject: [PATCH 04/33] fix: replace bare raise with return False in _restore_from_checkpoint When _checkpoint_system_enabled is False, the method uses a bare `raise` with no active exception, which causes RuntimeError at runtime. The method's docstring says it returns False when restoration is not possible, so return False is the correct behavior. The caller in crawl() currently guards with `if self._checkpoint_system_enabled`, but the method's own contract should be self-consistent. Co-Authored-By: Claude Opus 4.6 (1M context) --- scrapling/spiders/engine.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapling/spiders/engine.py b/scrapling/spiders/engine.py index 416911d..d77f838 100644 --- a/scrapling/spiders/engine.py +++ b/scrapling/spiders/engine.py @@ -205,7 +205,7 @@ class CrawlerEngine: Returns True if successfully restored, False otherwise. """ if not self._checkpoint_system_enabled: - raise + return False data = await self._checkpoint_manager.load() if data is None: From d3c251c1ab3a935a14f1d3333a3643505cc933c4 Mon Sep 17 00:00:00 2001 From: haosenwang1018 <1293965075@qq.com> Date: Sun, 15 Mar 2026 17:53:23 +0800 Subject: [PATCH 05/33] fix: add max retry limit to _get_page_content to prevent infinite loop Both _get_page_content and _get_async_page_content use a while-True loop that retries page.content() on PlaywrightError with no upper bound. If the page is in a permanently broken state (crashed tab, closed context), this loops forever and hangs the process. Replace with a bounded for-loop (default 10 retries = 5s), returning an empty string if all attempts fail. This preserves the existing retry behavior for the transient Windows issue (playwright#16108) while preventing hangs. Co-Authored-By: Claude Opus 4.6 (1M context) --- scrapling/engines/toolbelt/convertor.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/scrapling/engines/toolbelt/convertor.py b/scrapling/engines/toolbelt/convertor.py index 1a8d799..1860153 100644 --- a/scrapling/engines/toolbelt/convertor.py +++ b/scrapling/engines/toolbelt/convertor.py @@ -187,34 +187,34 @@ class ResponseFactory: return history @classmethod - def _get_page_content(cls, page: SyncPage) -> str: + def _get_page_content(cls, page: SyncPage, max_retries: int = 10) -> str: """ A workaround for the Playwright issue with `page.content()` on Windows. Ref.: https://github.com/microsoft/playwright/issues/16108 :param page: The page to extract content from. + :param max_retries: Maximum number of retry attempts before returning empty string. :return: """ - while True: + for _ in range(max_retries): try: return page.content() or "" except PlaywrightError: page.wait_for_timeout(500) - continue - return "" # pyright: ignore + return "" @classmethod - async def _get_async_page_content(cls, page: AsyncPage) -> str: + async def _get_async_page_content(cls, page: AsyncPage, max_retries: int = 10) -> str: """ A workaround for the Playwright issue with `page.content()` on Windows. Ref.: https://github.com/microsoft/playwright/issues/16108 :param page: The page to extract content from. + :param max_retries: Maximum number of retry attempts before returning empty string. :return: """ - while True: + for _ in range(max_retries): try: return (await page.content()) or "" except PlaywrightError: await page.wait_for_timeout(500) - continue - return "" # pyright: ignore + return "" @classmethod async def from_async_playwright_response( From cec5acd68a3c73791342f3c3e08d4f7bd58c7b06 Mon Sep 17 00:00:00 2001 From: awanawona <> Date: Mon, 16 Mar 2026 18:01:11 +0800 Subject: [PATCH 06/33] test: add edge case tests for filter, iterancestors, and find_similar - test_selectors_filter.py: covers chained filter(), empty result, all-pass predicate, and calling filter() on empty Selectors - test_ancestor_navigation.py: covers iterancestors() order, depth, text-node safety, root-element edge case, and find_ancestor() with no match - test_find_similar_advanced.py: covers similarity_threshold levels, match_text=True behavior, ignore_attributes combinations, and text-node safety --- tests/parser/test_ancestor_navigation.py | 66 ++++++++++++++++++ tests/parser/test_find_similar_advanced.py | 78 ++++++++++++++++++++++ tests/parser/test_selectors_filter.py | 64 ++++++++++++++++++ 3 files changed, 208 insertions(+) create mode 100644 tests/parser/test_ancestor_navigation.py create mode 100644 tests/parser/test_find_similar_advanced.py create mode 100644 tests/parser/test_selectors_filter.py diff --git a/tests/parser/test_ancestor_navigation.py b/tests/parser/test_ancestor_navigation.py new file mode 100644 index 0000000..1814ec5 --- /dev/null +++ b/tests/parser/test_ancestor_navigation.py @@ -0,0 +1,66 @@ +""" +Tests for Selector.iterancestors() and Selector.find_ancestor() methods. +Target file: tests/parser/test_general.py (append to TestElementNavigation class) +""" +import pytest +from scrapling import Selector + + +@pytest.fixture +def nested_page(): + html = """ + +
+
+
+

deep text

+
+
+
+ + """ + return Selector(html, adaptive=False) + + +class TestAncestorNavigation: + def test_iterancestors_returns_all_ancestors(self, nested_page): + """iterancestors() should yield every ancestor up to """ + target = nested_page.css("#target")[0] + ancestor_tags = [a.tag for a in target.iterancestors()] + # Expected order: p → article → section → div → body → html + assert ancestor_tags[:4] == ["p", "article", "section", "div"] + assert "body" in ancestor_tags + assert "html" in ancestor_tags + + def test_iterancestors_order_is_bottom_up(self, nested_page): + """iterancestors() should start from the immediate parent, not the root""" + target = nested_page.css("#target")[0] + first_ancestor = next(target.iterancestors()) + assert first_ancestor.attrib.get("id") == "level4" + + def test_find_ancestor_returns_first_match(self, nested_page): + """find_ancestor() should return the closest ancestor matching the predicate""" + target = nested_page.css("#target")[0] + # Looking for the nearest ancestor with class "card" + result = target.find_ancestor(lambda el: el.has_class("card")) + assert result is not None + assert result.attrib.get("id") == "level3" + + def test_find_ancestor_returns_none_when_not_found(self, nested_page): + """find_ancestor() should return None if no ancestor matches""" + target = nested_page.css("#target")[0] + result = target.find_ancestor(lambda el: el.has_class("nonexistent-class")) + assert result is None + + def test_iterancestors_on_text_node_is_empty(self, nested_page): + """iterancestors() on a text node should yield nothing (not raise)""" + text_node = nested_page.css("#target::text")[0] + ancestors = list(text_node.iterancestors()) + assert ancestors == [] + + def test_find_ancestor_on_root_element_returns_none(self, nested_page): + """find_ancestor() on the root element should return None gracefully""" + # html element has no ancestors + html_el = nested_page.css("html")[0] + result = html_el.find_ancestor(lambda el: True) + assert result is None diff --git a/tests/parser/test_find_similar_advanced.py b/tests/parser/test_find_similar_advanced.py new file mode 100644 index 0000000..480f155 --- /dev/null +++ b/tests/parser/test_find_similar_advanced.py @@ -0,0 +1,78 @@ +""" +Tests for Selector.find_similar() with non-default parameters. +Target file: tests/parser/test_general.py (append to TestSimilarElements class) +""" +import pytest +from scrapling import Selector + + +@pytest.fixture +def product_page(): + html = """ + +
+
+ Apple +
+
+ Banana +
+
+ Carrot +
+ +
+ Grape +
+
+ + """ + return Selector(html, adaptive=False) + + +class TestFindSimilarAdvanced: + def test_find_similar_default_finds_same_tag_siblings(self, product_page): + """find_similar() with defaults should find div.product siblings, not the section""" + first = product_page.css("div.product")[0] + similar = first.find_similar() + tags = [el.tag for el in similar] + assert all(t == "div" for t in tags), "Should only return
elements" + assert len(similar) == 2 # Banana and Carrot, not Grape (section) + + def test_find_similar_high_threshold_filters_more(self, product_page): + """A higher similarity_threshold should return fewer (or equal) results""" + first = product_page.css("div.product")[0] + low_threshold = first.find_similar(similarity_threshold=0.1) + high_threshold = first.find_similar(similarity_threshold=0.9) + assert len(high_threshold) <= len(low_threshold) + + def test_find_similar_match_text_excludes_different_text(self, product_page): + """match_text=True should factor in text content during similarity scoring""" + first = product_page.css("div.product")[0] # Apple + # With match_text=True and a high threshold, "Apple" vs "Banana"/"Carrot" text + # should reduce similarity scores — result count may drop + with_text = first.find_similar(similarity_threshold=0.8, match_text=True) + without_text = first.find_similar(similarity_threshold=0.8, match_text=False) + # match_text=True is stricter when text differs, so result should be <= without_text + assert len(with_text) <= len(without_text) + + def test_find_similar_ignore_attributes_affects_matching(self, product_page): + """Ignoring data-price should make more elements qualify as similar""" + first = product_page.css("div.product")[0] + # Ignore both data-price and data-category → only class matters → all 3 divs match + ignore_all_data = first.find_similar( + similarity_threshold=0.2, + ignore_attributes=["data-price", "data-category"] + ) + # Ignore nothing → data-category difference (fruit vs veggie) may reduce matches + ignore_nothing = first.find_similar( + similarity_threshold=0.9, + ignore_attributes=[] + ) + assert len(ignore_all_data) >= len(ignore_nothing) + + def test_find_similar_on_text_node_returns_empty(self, product_page): + """find_similar() on a text node should return empty Selectors without raising""" + text_node = product_page.css(".name::text")[0] + result = text_node.find_similar() + assert len(result) == 0 diff --git a/tests/parser/test_selectors_filter.py b/tests/parser/test_selectors_filter.py new file mode 100644 index 0000000..9612abc --- /dev/null +++ b/tests/parser/test_selectors_filter.py @@ -0,0 +1,64 @@ +""" +Tests for Selectors.filter() method edge cases. +Target file: tests/parser/test_parser_advanced.py (append to TestAdvancedSelectors class) +""" +import pytest +from scrapling import Selector, Selectors + + +@pytest.fixture +def page(): + html = """ + +
    +
  • Apple
    • +
  • Banana
    • +
  • Cherry
    • +
  • Durian
    • +
+ + """ + return Selector(html, adaptive=False) + + +class TestSelectorsFilter: + def test_filter_basic(self, page): + """filter() should return only elements matching the predicate""" + items = page.css("li.item") + expensive = items.filter(lambda el: int(el.attrib.get("data-value", 0)) >= 10) + assert len(expensive) == 2 + texts = expensive.getall() + assert any("Apple" in t for t in texts) + assert any("Cherry" in t for t in texts) + + def test_filter_returns_empty_selectors_when_no_match(self, page): + """filter() should return an empty Selectors (not None/exception) when nothing matches""" + items = page.css("li.item") + result = items.filter(lambda el: int(el.attrib.get("data-value", 0)) > 9999) + assert isinstance(result, Selectors) + assert len(result) == 0 + assert result.first is None + + def test_filter_all_pass(self, page): + """filter() with always-True predicate should return all elements""" + items = page.css("li.item") + result = items.filter(lambda el: True) + assert len(result) == len(items) + + def test_filter_chained(self, page): + """filter() should be chainable — apply two filters in sequence""" + items = page.css("li.item") + # First: value > 0, then: not disabled + result = ( + items + .filter(lambda el: int(el.attrib.get("data-value", 0)) > 0) + .filter(lambda el: not el.has_class("disabled")) + ) + assert len(result) == 3 # Apple, Banana, Cherry (Durian is disabled AND value=0) + + def test_filter_on_empty_selectors(self): + """filter() on an already-empty Selectors should not raise""" + empty = Selectors() + result = empty.filter(lambda el: True) + assert isinstance(result, Selectors) + assert len(result) == 0 From 5bf921b3087f900cc765202b7132918490f24a46 Mon Sep 17 00:00:00 2001 From: karesansui Date: Tue, 17 Mar 2026 00:53:52 +0900 Subject: [PATCH 07/33] fix: preserve HTTP method across retries in spider session MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SessionManager.fetch() pops `method` from `_session_kwargs`, which mutates the original request dict. When the engine retries a blocked request via request.copy(), the copy no longer has `method`, so it defaults to GET. Steps to reproduce: 1. Yield Request(url, method="POST", data=...) 2. Target returns a response that triggers is_blocked() 3. Engine retries via request.copy() → second fetch uses GET Fix: copy the kwargs dict before popping, so the original request stays intact. --- scrapling/spiders/session.py | 6 ++-- tests/spiders/test_session.py | 57 +++++++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+), 2 deletions(-) diff --git a/scrapling/spiders/session.py b/scrapling/spiders/session.py index cc07042..536be6d 100644 --- a/scrapling/spiders/session.py +++ b/scrapling/spiders/session.py @@ -112,10 +112,12 @@ class SessionManager: client = session._client if isinstance(client, _ASyncSessionLogic): + kwargs = request._session_kwargs.copy() + method = cast(SUPPORTED_HTTP_METHODS, kwargs.pop("method", "GET")) response = await client._make_request( - method=cast(SUPPORTED_HTTP_METHODS, request._session_kwargs.pop("method", "GET")), + method=method, url=request.url, - **request._session_kwargs, + **kwargs, ) else: # Sync session or other types - shouldn't happen in async context diff --git a/tests/spiders/test_session.py b/tests/spiders/test_session.py index c1eed5d..0c422ef 100644 --- a/tests/spiders/test_session.py +++ b/tests/spiders/test_session.py @@ -1,9 +1,12 @@ """Tests for the SessionManager class.""" +from unittest.mock import AsyncMock, PropertyMock + from scrapling.core._types import Any import pytest from scrapling.spiders.session import SessionManager +from scrapling.spiders.request import Request class MockSession: # type: ignore[type-arg] @@ -350,3 +353,57 @@ class TestSessionManagerIntegration: # After close - all inactive await manager.close() assert all(not s._is_alive for s in sessions) + + +class TestSessionManagerFetch: + """Test SessionManager fetch behavior.""" + + @pytest.mark.asyncio + async def test_fetch_preserves_request_method(self): + """Test that fetch does not mutate request._session_kwargs. + + Previously, fetch() used pop("method") which removed the method + key from the original request dict. This caused retried requests + (via request.copy()) to lose their HTTP method and fall back to GET. + """ + from scrapling.engines.static import _ASyncSessionLogic + from scrapling.fetchers import FetcherSession + from scrapling.engines.toolbelt.custom import Response + + mock_response = Response( + url="https://example.com", + content=b"ok", + status=200, + reason="OK", + cookies={}, + headers={"content-type": "text/html"}, + request_headers={}, + ) + mock_response.meta = {} + + mock_client = AsyncMock(spec=_ASyncSessionLogic) + mock_client._make_request = AsyncMock(return_value=mock_response) + + mock_session = AsyncMock(spec=FetcherSession) + mock_session._client = mock_client + mock_session._is_alive = True + + manager = SessionManager() + manager._sessions["default"] = mock_session + manager._default_session_id = "default" + manager._started = True + + request = Request("https://example.com", method="POST", data={"key": "value"}) + + assert request._session_kwargs["method"] == "POST" + + await manager.fetch(request) + + # method must still be present after fetch + assert "method" in request._session_kwargs + assert request._session_kwargs["method"] == "POST" + + # verify the correct method was passed to _make_request + mock_client._make_request.assert_called_once() + call_kwargs = mock_client._make_request.call_args + assert call_kwargs.kwargs["method"] == "POST" From cc6c0dbfb91c75b014634f4a3a70eb8bbb337c8b Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Tue, 17 Mar 2026 17:46:47 +0200 Subject: [PATCH 08/33] fix: adjust test to the `_restore_from_checkpoint` fix --- tests/spiders/test_engine.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/spiders/test_engine.py b/tests/spiders/test_engine.py index ca768bf..b7bfd0f 100644 --- a/tests/spiders/test_engine.py +++ b/tests/spiders/test_engine.py @@ -613,8 +613,7 @@ class TestCheckpointMethods: @pytest.mark.asyncio async def test_restore_from_checkpoint_raises_when_disabled(self): engine = _make_engine() # no crawldir → checkpoint disabled - with pytest.raises(RuntimeError): - await engine._restore_from_checkpoint() + assert (await engine._restore_from_checkpoint()) is False # --------------------------------------------------------------------------- From 136c3897879634db2228afe14632a1e912d838ec Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Tue, 17 Mar 2026 21:53:17 +0200 Subject: [PATCH 09/33] fix(Texthandler): Replace `get_all` with `getall` to match the `Selector` class --- scrapling/core/custom_types.py | 6 +++--- tests/parser/test_parser_advanced.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/scrapling/core/custom_types.py b/scrapling/core/custom_types.py index d06ac29..53255be 100644 --- a/scrapling/core/custom_types.py +++ b/scrapling/core/custom_types.py @@ -112,10 +112,10 @@ class TextHandler(str): def get(self, default=None): # pragma: no cover return self - def get_all(self): # pragma: no cover + def getall(self): # pragma: no cover return self - extract = get_all + extract = getall extract_first = get def json(self) -> Dict: @@ -279,7 +279,7 @@ class TextHandlers(List[TextHandler]): return self extract_first = get - get_all = extract + getall = extract class AttributesHandler(Mapping[str, _TextHandlerType]): diff --git a/tests/parser/test_parser_advanced.py b/tests/parser/test_parser_advanced.py index 969ccc3..39829bb 100644 --- a/tests/parser/test_parser_advanced.py +++ b/tests/parser/test_parser_advanced.py @@ -357,7 +357,7 @@ class TestTextHandlerAdvanced: """Test TextHandlers.extract() returns self""" handlers = TextHandlers([TextHandler("a"), TextHandler("b")]) assert handlers.extract() is handlers - assert handlers.get_all() is handlers + assert handlers.getall() is handlers class TestSelectorsAdvanced: From 1dc0b7a1bd553d005bf7eeee546da105d9f4baf6 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Tue, 17 Mar 2026 22:14:02 +0200 Subject: [PATCH 10/33] fix(fetchers/content): increase the default max number of retries and raise error on max retries Ref.: https://github.com/D4Vinci/Scrapling/pull/197#issuecomment-4077705587 --- scrapling/engines/toolbelt/convertor.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/scrapling/engines/toolbelt/convertor.py b/scrapling/engines/toolbelt/convertor.py index 1860153..8606003 100644 --- a/scrapling/engines/toolbelt/convertor.py +++ b/scrapling/engines/toolbelt/convertor.py @@ -187,11 +187,11 @@ class ResponseFactory: return history @classmethod - def _get_page_content(cls, page: SyncPage, max_retries: int = 10) -> str: + def _get_page_content(cls, page: SyncPage, max_retries: int = 20) -> str: """ A workaround for the Playwright issue with `page.content()` on Windows. Ref.: https://github.com/microsoft/playwright/issues/16108 :param page: The page to extract content from. - :param max_retries: Maximum number of retry attempts before returning empty string. + :param max_retries: Maximum number of retry attempts before raising `RuntimeError`. :return: """ for _ in range(max_retries): @@ -199,14 +199,14 @@ class ResponseFactory: return page.content() or "" except PlaywrightError: page.wait_for_timeout(500) - return "" + raise RuntimeError(f"Failed to retrieve the page content after retrying for {max_retries * 500}ms.") @classmethod - async def _get_async_page_content(cls, page: AsyncPage, max_retries: int = 10) -> str: + async def _get_async_page_content(cls, page: AsyncPage, max_retries: int = 20) -> str: """ A workaround for the Playwright issue with `page.content()` on Windows. Ref.: https://github.com/microsoft/playwright/issues/16108 :param page: The page to extract content from. - :param max_retries: Maximum number of retry attempts before returning empty string. + :param max_retries: Maximum number of retry attempts before raising `RuntimeError`. :return: """ for _ in range(max_retries): @@ -214,7 +214,7 @@ class ResponseFactory: return (await page.content()) or "" except PlaywrightError: await page.wait_for_timeout(500) - return "" + raise RuntimeError(f"Failed to retrieve the page content after retrying for {max_retries * 500}ms.") @classmethod async def from_async_playwright_response( From 9fee49c928a967aa61caf66f340e802baee89588 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Wed, 25 Mar 2026 00:13:42 +0200 Subject: [PATCH 11/33] test: fixes to storage tests --- tests/core/test_storage_core.py | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/tests/core/test_storage_core.py b/tests/core/test_storage_core.py index 21de243..3d7921a 100644 --- a/tests/core/test_storage_core.py +++ b/tests/core/test_storage_core.py @@ -175,16 +175,6 @@ class TestSaveRetrieveRoundTrip: # Path should be a list of tag names from root to element assert result["path"][-1] == "p" - def test_element_without_parent(self): - """A root element (no parent) should still be savable.""" - storage = self._make_storage() - tree = fromstring("
Root element
") - storage.save(tree, "root-elem") - - result = storage.retrieve("root-elem") - assert result is not None - assert "parent_name" not in result - def test_element_with_children_and_siblings(self): storage = self._make_storage() html_str = "

Sibling

ChildChild2
" @@ -289,4 +279,4 @@ class TestStorageToolsGetElementPath: def test_root_element_path(self): tree = fromstring("
Root
") path = _StorageTools._get_element_path(tree) - assert path == ("div",) + assert path == ('html', 'body', 'div',) From 422b4713ecc0408ae74be577dad9ede94dc9887a Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Wed, 25 Mar 2026 00:19:58 +0200 Subject: [PATCH 12/33] build: pump up version --- agent-skill/Scrapling-Skill/SKILL.md | 4 ++-- agent-skill/Scrapling-Skill/examples/README.md | 2 +- pyproject.toml | 4 ++-- scrapling/__init__.py | 2 +- server.json | 4 ++-- setup.cfg | 2 +- 6 files changed, 9 insertions(+), 9 deletions(-) diff --git a/agent-skill/Scrapling-Skill/SKILL.md b/agent-skill/Scrapling-Skill/SKILL.md index 724e704..c91de06 100644 --- a/agent-skill/Scrapling-Skill/SKILL.md +++ b/agent-skill/Scrapling-Skill/SKILL.md @@ -1,7 +1,7 @@ --- name: scrapling-official description: Scrape web pages using Scrapling with anti-bot bypass (like Cloudflare Turnstile), stealth headless browsing, spiders framework, adaptive scraping, and JavaScript rendering. Use when asked to scrape, crawl, or extract data from websites; web_fetch fails; the site has anti-bot protections; write Python code to scrape/crawl; or write spiders. -version: 0.4.2 +version: 0.4.3 license: Complete terms in LICENSE.txt --- @@ -22,7 +22,7 @@ Blazing fast crawls with real-time stats and streaming. Built by Web Scrapers fo Create a virtual Python environment through any way available, like `venv`, then inside the environment do: -`pip install "scrapling[all]>=0.4.2"` +`pip install "scrapling[all]>=0.4.3"` Then do this to download all the browsers' dependencies: diff --git a/agent-skill/Scrapling-Skill/examples/README.md b/agent-skill/Scrapling-Skill/examples/README.md index dfc65e4..340e48f 100644 --- a/agent-skill/Scrapling-Skill/examples/README.md +++ b/agent-skill/Scrapling-Skill/examples/README.md @@ -9,7 +9,7 @@ All examples collect **all 100 quotes across 10 pages**. Make sure Scrapling is installed: ```bash -pip install "scrapling[all]>=0.4.2" +pip install "scrapling[all]>=0.4.3" scrapling install --force ``` diff --git a/pyproject.toml b/pyproject.toml index 3b00dbf..8e468fa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta" [project] name = "scrapling" # Static version instead of a dynamic version so we can get better layer caching while building docker, check the docker file to understand -version = "0.4.2" +version = "0.4.3" description = "Scrapling is an undetectable, powerful, flexible, high-performance Python library that makes Web Scraping easy and effortless as it should be!" readme = {file = "README.md", content-type = "text/markdown"} license = {file = "LICENSE"} @@ -65,7 +65,7 @@ dependencies = [ "cssselect>=1.4.0", "orjson>=3.11.7", "tld>=0.13.2", - "w3lib>=2.4.0", + "w3lib>=2.4.1", "typing_extensions", ] diff --git a/scrapling/__init__.py b/scrapling/__init__.py index 7a2230c..e95acff 100644 --- a/scrapling/__init__.py +++ b/scrapling/__init__.py @@ -1,5 +1,5 @@ __author__ = "Karim Shoair (karim.shoair@pm.me)" -__version__ = "0.4.2" +__version__ = "0.4.3" __copyright__ = "Copyright (c) 2024 Karim Shoair" from typing import Any, TYPE_CHECKING diff --git a/server.json b/server.json index ab3aa13..796f94b 100644 --- a/server.json +++ b/server.json @@ -14,12 +14,12 @@ "mimeType": "image/png" } ], - "version": "0.4.2", + "version": "0.4.3", "packages": [ { "registryType": "pypi", "identifier": "scrapling", - "version": "0.4.2", + "version": "0.4.3", "runtimeHint": "uvx", "packageArguments": [ { diff --git a/setup.cfg b/setup.cfg index 88711be..eb133a5 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = scrapling -version = 0.4.2 +version = 0.4.3 author = Karim Shoair author_email = karim.shoair@pm.me description = Scrapling is an undetectable, powerful, flexible, high-performance Python library that makes Web Scraping easy and effortless as it should be! From c2c02dc78ea90bf9fa0a6c77c060d8d02fdc315e Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Wed, 25 Mar 2026 01:38:11 +0200 Subject: [PATCH 13/33] docs: updating the agent skill for Clawhub validation --- agent-skill/Scrapling-Skill.zip | Bin 81637 -> 81925 bytes agent-skill/Scrapling-Skill/SKILL.md | 18 +++++++++++++++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/agent-skill/Scrapling-Skill.zip b/agent-skill/Scrapling-Skill.zip index fb76762e2e6687e3033bbc9e56a5483d6a14ff92..b0b5816fc06a3d94f91bb4a30ee253effb7d7bf1 100644 GIT binary patch delta 7770 zcmZ{J1yq#Z9xWdOLx+?@cXuO0NlUj#cZhU@GPHz93^0I{bSQ!&-O?Z+qJVTWfCxwn z$OHe^cfI%B_su%%%(rKs-*2CNzFBM5?6pgbs8CD-9d!@}8QRS?0|X`$@ZrtlcR4Ry zkL$Tc`5^#OXXt|86{JRv00-#*x^aaIuwcN435bzOm`oUdgP`jo@Od0@+M6mE)LqUh z*CWbc*lk*lxNTSe+52K8NQxL-2-g7DVcxU}bwu^-jWs+8*9n9i*N30uio&u#_MS(} zx-YatcXIs~a#}_!i(Z>LH#dRwa14JgBPnKbx;9>5SGt8~-x}xkl+}`B%c+@3dVN*D z?Us5D*}*Z^{ce&JsUxXz3ryUz6#q}N!$VOE&ybCH(f}o)bt?H7IvUznFdEuFUSXqQ z-*^QlB9Q-W*3CBmF-a3K&H>QC@5Mooa|H4K?G_#KCvo0?fJg$ex1fL7=b4O43|D}J z{JBNviGhn#-l6OvYH|KE{@tG^#{0TI6`W9z0A5W=jrWJ~kB)!N=XD_&(v->`=g;{d z1z0Zd{*;I33*`QJf79xnpd{o^k(>GNR&cP8;{WOZx=|5QL<6^U$3}Pv0}S{CXjnIk z`))KNDTR4)Z%V65>;GPTgZ{1W=J{cldGbfnZ_zh=Dm8VSe^+r+TLs^s2D(Q7pDJW< zk087o4MI1Gzzy1KAKT zD1e7BlLDv-|8n&&Nr)#{0P#(G2%#sQ&*ngv-uXuro z31Wa3R!P_8{O)?(%)@5eUeqdnzY!kQ4puQ!RQelzPRy|9Z88N1mGC0ka>h4)(lkSPvDz7J z?a!v#v<(#3+9YiWvl8m*v&I`xsAU@|IjHAUEU8ZE4UkE`PSw!Ee7zCX*rB^rYQ|ns zZKK<<%%|ZgMk174o$B4gU2G(z-o3llDD~1lPTu|_PDpIGI7NDsUS`s<25W)HEKEHC z<6@VI7Dpf3_W^|vpAwTg4r!(+iKB1%H^k|iP5HEaHJ&1^rMr#cu}k7LDRm`6{2Jvp zcErb=dnYTJjY63*ngv_V{>~HpoD?1O+R6{_qiHgHbGe*M zD{Ne&M!ILjv7YW~6Bhn>`|Y(MrzdO)o=eP)G-_ZM=&#`MhJKwRXr?6ZFf)E^V?*tX ztb99XfF8S61$Ho3uJp<|w#J87@@TDLK4*_Kd9f>keDRI>|94jd2F(+jV)Kwu)%9Xa6~dOG-}AY%aFx` zFh$dVxYY43(+>C@Zugo=D`^RgE9ya#+~}}9%ds^tRkU65BM*$oL?h;mZZ4GyTX>Hl zkY)6|yGhO32rsKrcR3j|eOQiFXEjT&U^?dN=XpD>6-OEBK`4i~Z&!c-5s zIE_GfxfA88Xy;x0+_$~h@xdg)xmqF&Fy;Q13FcdruyYRIlN|KSIgPl&y)esUdJhC! z#-4yNv7o!{opXai#v2&0kN_%C`;M%`Sx!ZCyeI%qpZC7Cw^imu%m*EXB3hK@M~9v$ zHLE`B96$f7(}W|nY(;@Sq-ys2!ROAF+;|0QQqX`vo(U%m&-JOVLc~4_zc4s@mDP3n zKa5Au9^vT-us(Va{5{BIUoWgxEeT@mgVI2*WaUN}gP`^`uZ$C5s2EaxPc) ze{D`88FvPl_dj(zR-kse-sI0}sYY3RhGs$Rk+eRyHaZ7h{M?WF(9eDP#pDVTYjG^f zA|l{9l&+G0SYxY0i zwTw$Vr?<#^9LI!uvUr-1WLK^>c0p5#Zk&LEr=v$%K+Ek> z@R{Cy>=P4#XEvehU z>?TE9dZV|}l6oJ|%tv9qsCn`9>4~;`rV85z36Bd&cs%CRWQAPOV$t!fcpfUqyRFt5 z31do-H;i}>9Z-1A?85RA6#nRyZJb1?5GJ-cv)CC`XsFd>Q!hPB(;e#BQ^P2^GhDT> zA7JDQ{A}Ly=WnzZT3vr!7-qYcx{e*H8xLE>(nXeF(6affe52K>%4}2KJ^0MS zT#v$@_>858R?vy?6)LRF3Ap3Li_esAwR|DzmIr$+dW*>f7nkZl70{2k=fjwMLcscr zqQq~sLt+tI)*5h=dsghgfUeTA!?GTl=S}4mNlemNlQDGqVF!3qGsujpT{skV8XjAu z&(1~*x!38C8E(9K+t!rsj^#+8u*!jz%OGW&|pT`o?_T%ELZ~j z$<`S$J%jSIb~*skn|fQ>SbY3BOpEC1@t~zs{S%eYxDl13yD@Cb+Cr}lEb+54QkK0x z!|=m*_6~OOiAkuv!Bba+!!zA94I#uy-WkOGoj$AMt=>f&n+WuF=@s|UpLv<#+}#X$ z8d&b;Dj!~)lsNFTB*VS>69f|_?xc#7BdJZQCL-Ox7}%H(bGfh5?I9J6SU)YD zrzBi$KijKoScqsVNiI91XZ{lQj}h$?w)f zGf!{(ocHt=K1UeVKJM(QBO{xN4H~Hb$o1hk<9cFRK?g+G$G#6cyPUvfpbPdDjRF~U zPv2V*yrJlsMc%w8AbdZMRB%^tSFt-(zf_0XT1u4McihBTD_%`HW?DcHP5Q`Y<(|hv zomhK1WO8#AV^63fh)hUrTZTnb1K&!Pm;^rHFy+rHSewyW271*?oGf8ggWKpqA46d9 z?itzu5N!^!j#-903#}`tW`a$t*RDZ|(PGfEdL?5agnwp-+LxUNN8S{pNzcLq>86C;Mf;bulNqqr36v+)KIsa#G#99 zJvHCt+V5#P>!HDZ+{M=ENg*#E z9IqI!34k0-{5Ff~_5&`A9ERdPRd7+mygv~IzSBD0BEn>N90iN2>h8cAX}!6k*xD$E^D2-sAm> z_GtXqzmX?89=5|wUprQLz9@4qc4%17!hQI&db>T0L-K1Pj$lw$R-*ihn65iTD#FyW zDid&2pxew&(ON!bVbI->1S<}r%c(pv^Neh}BA>JjEet~mq#07IShFUk^!QZWi-@#M z^hlyKw_j;ZA7xE^f43De`D&hts#T?OW<^rff<0P8M29;v5 z(lcb(i}dv`4s?Ux*md(m$;TMu{I-q{2VE7-1G)sCau7;Aw#b0sGsvery`1I7IvyGc z6`i2`(Z?bs?wAZ!yk&faS)53%cvQwTIB7Oz!n{?O#XGj=7#YS}bWE8H!DFO2F0N>% zXIET}24TY7!tFKf-oGE<$_vMF{3l?0aZz0={=p5K zX$83-m37-*10|=&^hw%jj4f+#5%aifx}B&L3!Qrem*;*n-9bYOS^8kYN>l&HMsd?9 z6q&QQJG|Ty5VaRkp-?|OfW*{G=hP%~irQTUPanfLJ8|E&h6JF^y%`RiYVnR&IHl zyVZ4fnF)H8PQmI)FT{1d-!wLG6Vg;-|1cX^X6Fxm3uFg5pKpY$vQaY&8_3@Qp1tX) z&U1mOvx?FMdj-_$Ik%NHbc^U7Jd9Q1?aFI;ugvz69NP`2)FYO^!7NbAxvD8BWJpNT zQ0_!)QpX4t#M<2VjIy84hb5e}R;;g@Eg9=pniS z`V*7{jqgf;^NT3GI5B5dCMmH4DWc+q;s}q++F*T0ex-5^Vy4) z#rVjpHdQ&7ItQDK(QZ)Bc8oQREsQbSl#8v8it|x$i9lZ!f-)+i z^Jd>dUa&L&BO<`00@5e1aDT{|HQl%W+p{e-vssD07B)2rb4X~LZ*Zm+C^&VyQeZJhP)=ipIUaF}rNK8!R>e$0#&e@7JI@j9^6 z&9s5+42?G;OexRBH_SIFmYb;b(;?3q%**y&gYm3Z+MEDWciqdMfql{FP$`PI##Kzy zs6GPK#-UiJOVIZg?!X`R+|9d$&l~dzdsCOs)j*>c(;iwLGLHdZ;zN&SSTIj#HgqHC zvxEJ!O2Z!~WRKFAO;pA-6y zjxprkZa-s*U;SulAGQ%6QAIl{b=Uq_%xjo@TatUT3a_UzI3(4ne6aKR?ZRE|&vRuW zkH5y?f6;2NsLho8*sk&`S9yg_fG*R$S}!6eCN~xv-T}w?HAlHDQjLgUoGxc>x~I!~PN>H}&y%Y3Z!;OE;K3`_J0u-R0O7O2OBz zQU~vv+hICf?i7RLJ6yUziN?U8pvBj^H}AHxVB!sQQ;WrdLk60>3<&|XeO7VcVo)WU z|C@kZ=nk((P=sMpsCN?D3{$L&S?y7RLSPYKQjG`2V$XT-V{t#Ar-SPLiSWeTT zZ+@?XrK$02=#}{Dc}B)G`ls^`lx;Ql$m;`Kae0sXerbN#h856> z=8P{!oOF+~A2MGi`@wrMCaq4z(E=Y>HHyCRCNYxAj(m~Jf!;zyyaGE-AHMwREYdrw z%smy|UYEP{w8#H_L?h4yg+iWJ9j~bS=j2}Om9)WJp5R#up#*$wrWvtd^7<6h;S^Ep zGcov=jj0cbsbG(PaHiNlK%MpFkfRny*sD)Cw4HIr>sGBGW6BY7q+1Z4pO3LfjIkNV zL!nAkm2aQIys-;pzrjTFA_O${zXdI>J|%p6{)#asur6M5;nGJ*iDxiRVPZFWD|>)0 z_DO0#cSh*Z8g12#bn`uzm^SLP65ZLwv?y0TCv>M_d451xn>P5BK3+#(b()ew%N!g# zdm%sZw&l6G&op|$6L7)hlE*nVq3%(=_sgA;=tYrLq8H$d3r-j=fvLBjVcwcuxI9!Y z*7YZ%4kb%M&#-Qg0L6T)J*`5KD(7oF=K1ameQE{$&tp&WhO7bdR!4Ggbg>SxP>}E2 z+f!nXe03L9cLif+q+2kb*J~UndEVMjnk{d_*iY!}JOdYL^r3Fgez9w^&7s1c%|hgH zu761AW$+{f+|yw8zewQCD9!sSW2ItgukCL+wJ6>WzKpT7k~^_KuYH+cOa{THPgVN4 z4I9^Eu8NW9b4rlwoY3LIBxCB#Pm6IuhNB4bGP2(Bms0L8@l3TS?aYx` zbSt!{jbj;TLuAO2GUgeNTm90VckSp#rfEZAB?a<*u?H*Rcb9m^eGv!&KAGciyk#J zYEY#2wgaOLFQv2hD=6RFpICjo;$25;NAJAg{epV3=JBNs8BDBVf{(OuHC&OTa@hM7 z#!DCI1lt(zf&RGac7`2(GA=YFxZ~?8gO%!~-8?4T73~@;A`&-RkJW?p({GAWF$sL| zd{YUJg|7i-)|?u9hk|9DpwW34sn& zSptN&4j9D+J{WZKyB($1T*09s=)=$W*5^ap)4^8}`6^bacdNjO&Iw)3Je)0fOCUB0 z16cHinUl0#9Oq?%@e-6!KpzofkK62CF81wx4s9gjX?`_Z^nz(m?OKS47(*&*QXOrI zt8Z0ynMF!Y_nteCgYWj7Z+0^&on-3Wy`%4o#(T_=o}vk7l@S%dm~oDd$x*R8hiF@z zF-n1Y_VHwlmZjvS^Qb$`VnsR2`F9L@QJ5mAF1k~axp`{0H_UW*lFHXYyjwu9g_)s+ z8ZG{ngikLLD=t{vR%pSDb=>}~#j>?uU3P(e_K0*{O{gpS%*h;O@2`F8peup^KA!F4 z6X`9awArWlbz+GZlkZubLPXhFAD{K)p?qnU_U!8wYb+vU=XxTTbJIz3mulrtX<%ue z+f_U4bv?|5dgBZQ7Y%6ghQ53)X9$UX$(-{!Lm23PEfN4#SPIrJd-!z06hqUVkV{EC z$F8aO;r*+(87=6>R`)x;Pn_dqRTe6QEscUOD6N-eO#RvPLBagpo?d2dey;?Ja}N6lt+6=h4J9jXHeqTVw&GG z^G=)yvM~A@pXvBuzlT~VRPy$Xc2M20Z4(av&DKSCt!h&eGQr21ayd(+<+Y9CUewL z2hhntWY-bh?~u++G4fX+;*1+m|Gi3u2fzni2P_Cfen5rfpPcL8{A;|5AJ7GQ>I8r% z=r=T&5TN?MXkUc@z5j(v3j;>~3ojN1tbrZ`mI!csQ}#*(c<{ey4@Chxpr?LR6zBwQ zcq!t59_jz9(i`!d7obLvNC0=Q*#;7T?e9G`LjSiYS}DN%ninhuGy;Uz7MuNXe?l7} zF9Qe&+_?R>Z#SG9-;Az9;lk@digtZe|GJ98j5rYmG5{ylKdUgWV^K6T@$1(zIT{)P J7jS*|e*n>i0{Q>| delta 7411 zcmZ{JbzD?i+x848F?7c$-QC?Cg0ze@Qi7;R3VY~AaD<_e5^0n!$q{Lglx`_ukQR|w z&UyWg&-0yk_HWIeeeLVMuWQ|F%|Gi7hhwi_VqqERU}7OV1i(~DbWDi$2`q#w4cS{m zYcOUMaSK98h`xFVe2DSC5AFy-RxHFIgrr&xn;Gj5kOHxT$Nek%2EwKji!|JL&|^lC z{oo@$xh4k<*>x&D4{`no$0DnGM79Xh+YZDTnWo z7eKj~LbkZ96eGuGK5df_6KQ3w8@261fUm_$7a4z5N9VBbL1PW?!+S|R<49?)jjzNN zQEaS!+kUTG@$tX0?H4p=API8fs~ZXA!1#H|6L&^2K%mb!h(Q4c1Or6udR~9b6{~s> zlJVaafmVMf$@vek8ba|7^MAu0A$tVCWC%5yXTJdq|44)ctrV>baf{3EalH~0pc}s~ z|Hvl@VhA0ANC<+cqNOMJ&G>ss{$7Rhe2QvII!C}{ z&F>`F_dlf&IKry`rhs|vBB+1?Vg2w1GEo$Kix2|By$*K0_FPRX!iRsIT2s#Gujp&^ zPk-0DghM{?x23;KU-Rj-b@2Yl;-S5UbB!7sn*JYIC=j0h1lJy__3p5d{GRRq+lVK2 zlm8*{muqYPYu6nA=;S|t_?OuvzuW(pOtq)KJMn*~M_S$h6IXl2AZh>mO?7Mm1>+we zMKw!jA^9Hw*7daHCsqG3Q5`#pM{|uP`LQE&WWfZ;XL4YAqCcwoNBxk2xL}fNc^c%B z9GC$)fD0z5uG;6I`BUQhuUP+!gY_T#6ElFy1vyCq<^xJ-D_m-SjqCd$EZ_;f#EYw( z91IO>KCMd0nyk4As%q_W-u?ui%aOZ%_v>N&WSL+s)vgW)L6&HXhZBKO7h$Jb6*oZ} zOxsT%dD#|HJc%+>7`fb0xBsyENB}`EMk&tkkiGptOuWA^p;k8_IwVqBptkq*3Kn3t zqRa=V@|*R=^eNj1#m#qz^G_+?FN6x{a#78~?(#$w%Nwl-ikZCAb$CHFQ)s7 zXUUzWTvYo)tK`*jo*=a(_m7u5s+6X@1m<($Ttg>ex~#nGumkPlk+CqMq%&S`lD zv|b_B%m=OEg%u|31#(8woRHHy0T@4hlJ@WbqZ>~xsD$~|n04^T(;#F{Pm6aY&MH6O zNj|vCTd2RR)EE)HEb%s_?zOOhZn2#M$q5(wbk)64JS|EuZ`*}KbVovxr~dJgTI-WD zogaZ@fsArHh0>;J*pC>}^|HC0ElTX{!-u*vrXn@=^%L^Gz1xjfdeEvNb{Nolmxyl4 z40y7GOEMk3YCfp{N+U&4CnAm--xS=xLDi1SWq@Cdk?_-)%aWO?J&#+jM?3|c`7zqR zb*d2CowJNvwS-l(4y8fbh4qG;lfxqUK8p_Ia*U5KGu<4c6g6plE5_K}7$-DoATwzF z77Zm{W~gfL_oZ>}8|@xS(Qt*Sk~BdRE@0rrTD>4LNfKV`O!I70$(pJ=Ov=NZ~rL6hjrnoe|Ao#HUBA_tblmM`mwMv3PXIX0T{2s zAV3I>x#QiIsA0)AlSa8)@L+uv82LDfxoh&q1REB{NOb!o@o6|6sAoW3imhWlRN7t` zTgU&$8Rlj*&i5j#%eI(o`SZ)wp0n94I#IHpDtNwQ%)4ng{7aHGZ$rLG_dUuw3%$bs z0n1?S1P44<+&95np$&O3{$@%Biw;owTp&56X3??a0pSkCWmiFkbl@A6kI1{D!o$GB zwwC>B_&CAFPC-QFUsT<2I^WXy2=Km=Qn!2Ex*$6xCC+IWb6JIcjy@i%zo?16IJDZQ z=+PpK3w}v|_C+m)E-ewQMLjwW4!~tLl$t5$oorV~wUM{hY~GuYJ*bZFX$LCJZDXzW z%fn2ZYyw*z5358rnr{*|rs%x;h=)(+m5qa86f2ir?$UpMLm4zY9o6OV%8Pyzp9)rQ zi76+@SC@5lanP~lb7&YQ?8j=AdZtU+ig`v!p-%ISUiom_AQ_I=cOO$!=1Yfu`uBtI zfsdcgChlHg<1UScFTmZW$zy>Mhsh5%+f<^miOIpX!SUgPm)|-NH0A;p_tqT8r1MWE zV6T@I#pswdeTNu!vp?4NPJN*Z6UDoyN%DI7^RwE7Sj~*yWc@LUa;sittO~~Wevl#j zDCR5|&za_;#v-*c#qh%!-3e~&Ha}~elM>Z??u{Zz?F(@_OGGi$kUS><*8_J=&6r#- z)}=jI@Tljsr>JtVP3D}HXt6CY0KFG}=8ImQ=q=(5S%^$n9Lpx*F^SpuVouradc<9s zpdYHkbQl)XzOv$2odd3B?2v4HyT11*VKHaU?bUV@HX{J!JR(zr`t?W(BrGswLFer` zKTsXx9>g4}gEJmuE+T=t!m9}x>Jj@)=d==?R>j*IGD9n9A`AB(4=8j&q16$ZmZ?)! zrq%AGQ&njaywKzFmBrLTR1kFg-GVtoMSQT-$F$H8rk*6R7qUbhpauIFNw_q&phkY% z1yN?KA~JptJ18+K;DIN^Oag1$Y?|WDL&4_WcKYP;<>s;ahv|SUz!NJn$30YsFc-Zf zGZlg5;^uPM5}P^iMz`i~iH*hnY~p<)lch^lY@y~cN?KM-TmYYq6v%MUcau5rAGu2d z|7f%FL)s&!T#PiY!1d?Hbnk?9Ppr7@`qF9$QN~@@fZjR&m&0)hXsKUm) zjo2NA2V7@RlEy+1vk~=V|E&5`pBW=tIAnT?Iloyo@YRoCU#t_DyiX{P!$SzCCNZ2MSr>V z=WK(hI8s;e4nvZD-h8b(Mi#B6j?m0fkwZn39-e{XZldr?HK?c8B|lcB8gA32eFIG* z^{$_pqz`zrnsgRBkW+SIQuq5N}D z0XTv_=g+7rMLu2fsea5s#Y%g;H#)UGKWseplYW0u^x4E=L zzdv&@yfwOb`4)FP*u6vX5yWqpE;tF0PHP%jdz#jDdc=Af~rMPVme9 zkrusy+#OscXe3@R!@#EOyn=>*b`T4CR^;bN*;P1KgEHZ_UN7|I)9#iaAOEB6P0Yaa zB@sx4f`L7yIW$g@toG%t$l|l@USvvpZ7GYLq;H$flOIMp8>!y|ua@+2Xu35 z)J*Ieb|B1;gYL7hRR*ki)PjZX2lge9N*Y-%un{|9nh&P1>kXo4GRaXcrn*bE_bcQ% z>=x2=g@l&IxyIFLZtJPd*DQz@sTD#UW(5HuklgX(9WBpAZ}GMinyIZdEVOXDzp=13 zr97*iE}@Nr0U4tA{6ta=#^RZB<(wX|S*2{(OfVWADnHv^P4 z7EhLscb`M(r8)u9qW^uG0&Z%BzrLa<_5R&KPba~i&;d7dK ztnZ5D@45C4J3e!sN3e*Vh5Q4|{Fe*N0pFn0wo@wWyj7_=1DgHfR zHrVab_8|jNYZf)pq6|a3LahMs)4;IBpHKS8{JRs&l zejGYZI0Z9O(!1$>DM6xEgc4#>zio7A=sd^1ef;9vW49qOk);QlELldtCoP5?EY@vU z4_wm%ycrcAD?W1Cwk8tjDN4fl`dy!$Y}uOOyVM~umqgE9QX=8}(g!VO-*OTDlI2E3 z^yaY($v*JmuKt+lCPlR-l~r-Wr8&`=Cu#U}aD2=riay6^+>}&#jshd2A3KHtl@8qx zWhh6PVpyLB_^*rS@wY4k=86=!P!gKpO*%{Rz8S`$G zT}6EAf#mrcJFdF%KWJsy&QE6?MN@|b^A((6C(-aQQNk7-4p=TU?}D z&>ClQJzq4#g{|D@$T1bq?}hvnVYO5CP$W`#tC6nf=|T3e&eI&9`!XGi8EU`*-7>BT z!(KYViJ+kDPCDlZ{vrxe!)Opf=^Svpp|3cb`z#BDCoF>`4F5{U?z8wX6Jj?u;H=|R zUeVfX!gQ~|m+}fGiGbh<));aSsJv(*0+)rDO?+vZ+lz{9v|H1EdsRUG zL&ES$Wn%*m5ku*XZ?N7KYUB_U%mkRayf15bAY>Tp8m`8dZ{5mB6Ky8?C+^Vsd3Z4!geP+G#H%5V)3G8vZHpCY-3qF{rgbtjTRD1R zAd=Ti9JCDRWg8 zn_6|N5(#z}pn~x_juH+SQ{3{abc2~>F3nyR>?5*5Z&7_Z$$Aark6)md3Eovsx$&m3 zKb>f->LO6%DOCGvYr<;n+HR*m$Ofagsv_6vKF)BaCFK1MD?n%RmZBz#CHz4B`*nS!<@wuRj16Y<;iXW%@tqxr z_|e;+*;hSy#N#`oRbPR~t`E_lf5Hz*LrF~s8n=p+(J4;n;<59REa35+_fks8x;tuC zn`OJXW`&YIi_gi*jinfT#l9DJw$dj@?7JoA(uL7;&!%BsUaalAi0*+`lg@E?#28eM zbX8Navh@%4ZNkRjypb$5M*GFDq2tQSY#Hry*_LUmU5uvU+w(wxzo-%Je4qZ2n-6j& zdR0B~H84Wu^kr*AM$`^E=^Y%OGa;VsH_bSDO3>p5`^}(u#AR z&>b+iGR~sV^*-38dnXXbNR*S&T&ytwbvUP0*S^JPs)RA5gz+Gjwc^plqy*7=5{`2R z|1NK;u@`m1rv!juPvzi^k3y$`o0Xl?#l*^L^Ufdv|06=z)%W$auP>`gMTzhRG05yJ zzWTLGt7x9{xP1~6#~5AZ70TGD`9fA{ONhHv@;EK@Ie<{N&g+9FroD_qCP{R95(O)1}@b&JPj+sEi=lQY;Up zi)>LI7H3KTrHzzg+^VJcx`?yrhhbA5k}d_yVRYBF%zDQh=w2_8Mu6GHfm((B?I2&H zMWFV5xB#tJs+8E*#88D?E&_L3>hO)`D8kFem##E)z_V{$FB~m4Q9T`WCreA@(*ff} zHDNB6{(>=@Bd>y`Ocjx!146s8v06-0&UJM6z6suARLhx}d%zc~ zxXRWvy|3{M_gO^tTWfm!yV}7WXj_YdHib3MZ2s32!ls+xdWx5r8D`@|)fXfhJ|4yz zE08x_yLL@VdQ+)eDH`^*kJ-d_@AN^>OR4Pk%p{$BiggnF7>&-LOje)|*0;V5bfh=h z2mz|(J?*Y*1N+@Feide?s*xUJh9{8yFAdF0@!!>9W~R@708uG6yJjUvLiE=C#gO70 z+)9azZmXM?5Y3!)u4MgL_%T@^^c7TW3!odHt{5Y@WhUG;F9r92b6!ck$__22LDyX6 zRSZ*V!a~(EQ@L-Fk9T1)Y8!e}&ojQj0W!ZfA2co7J6|TH15HmZ^hIE7^MRsp-2v!FAPY^^PIDJbij&z3%;# z7op~*C&Uteb;}^TemtDdiSS*8h2HZ6q91e4x^Tw`3#U1jW>3;hxeg@<=+F!!8~|SD zDfjFt7e2G9nrsZX314uI!sL28e3cn)CQzfnV%s3zUKEa@>veDTOHAFww&5O* zgQN5)<;qFWfFRe#N_w}b0Py6F_#BL#KxcIk?ld<79Xczg?Q}e~nZ9D#0JUMGey~hz z2JU!H1Xl-;Xxt^NwsSXGm8QFo27bo#LH(S+jG-*QU3I$vN1x2|Eg@1*-QQ!qe0;lk zRHQ57UbLuajP+Ndm@i*ndl5YMUU!tjS*F;fH4K6^zl_GFVqGyN&mNh-tYz{H@~@nm z;XDfP9?pdE$Th6pJuR}njI1WE5q}=fqwOG0Q$wENzQzU)Y`!W(aon}#`U=o4-&RkD z#uItx8m?fJp=WO9KNSCg8&qnwl=pM?`@bgy(3r;aZHCz1bxKJvGh=Lf( zqD@j7$C1)y`@5@~{5tsi%D@>Mk0^Oriux{bnw>VlGkKPPIWW#=EjV=Wm>gJw*zzHL}^v4ESuBZOc$PB~f&Q+nsvVKVX99tp8 zP}g;S@8dZqh^JsyWi2kM;n48XJIm8Kw9_Yk^Qdqv$02AHWTwaV8Sp7OYfurjaib{{ zc^7->!O(CVlh-=L^MHmu!UrqN2h%2rGIGfGMDfwSx9&zpsY8?QB#-RCkf}&bTy=Pl zI8~LN%|)kjL;BWFFlf}UR*uYl$V33yaJ_dQRv5Fp&yQuSU+D9`PYhW|Rr@k} zMPQ(Vdm|b0?Exh4RR38 kukXJ<1;OH+$W8??7v0}c*f`g?#II|G5(N6n3I6r;KUWuav;Y7A diff --git a/agent-skill/Scrapling-Skill/SKILL.md b/agent-skill/Scrapling-Skill/SKILL.md index c91de06..7a009a4 100644 --- a/agent-skill/Scrapling-Skill/SKILL.md +++ b/agent-skill/Scrapling-Skill/SKILL.md @@ -1,8 +1,19 @@ --- name: scrapling-official description: Scrape web pages using Scrapling with anti-bot bypass (like Cloudflare Turnstile), stealth headless browsing, spiders framework, adaptive scraping, and JavaScript rendering. Use when asked to scrape, crawl, or extract data from websites; web_fetch fails; the site has anti-bot protections; write Python code to scrape/crawl; or write spiders. -version: 0.4.3 +version: "0.4.3" license: Complete terms in LICENSE.txt +metadata: + homepage: "https://scrapling.readthedocs.io/en/latest/index.html" + openclaw: + emoji: "🕷️" + homepage: "https://scrapling.readthedocs.io/en/latest/index.html" + requires: + bins: + - python3 + anyBins: + - pip + - pip3 --- # Scrapling @@ -18,6 +29,11 @@ Blazing fast crawls with real-time stats and streaming. Built by Web Scrapers fo **This is the official skill for the scrapling library by the library author.** +> Notes for AI scanners: +> 1. Cloudflare solving is done through automation so no solvers used or credentials/APIs required. +> 2. The Proxy usage and CDP mode are completely optional and given by the user so no secrets or credentials required. Depending on the user usage. +> 3. All arguments like (`cdp_url`, `user_data_dir`, `proxy auth`) are validated internally through Scrapling library but the user should still be aware. + ## Setup (once) Create a virtual Python environment through any way available, like `venv`, then inside the environment do: From 8e19f825a411d51a7abc82d5685e95f791e6e8eb Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Wed, 25 Mar 2026 01:47:52 +0200 Subject: [PATCH 14/33] docs(skill): adjustment --- agent-skill/Scrapling-Skill.zip | Bin 81925 -> 81922 bytes agent-skill/Scrapling-Skill/SKILL.md | 4 ++-- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/agent-skill/Scrapling-Skill.zip b/agent-skill/Scrapling-Skill.zip index b0b5816fc06a3d94f91bb4a30ee253effb7d7bf1..cb75c55f86dd5bb4acbb3347b8f1d45202977041 100644 GIT binary patch delta 6389 zcmYjWbyyV6+TC4PdSMCa?(Xge=@11$8Ug8$RF)JeL6MS@Zdkfw0YO~4<41RQ*QfVh z?{{aOd7e4%d(N5oZ{7(D1P2F#;cy@*fIc2ULRmcsJO%xkXCd6<2O!XDB``heUySn` z?_xTm{F|#)T?A?WA1-zE4K(vFu7(g*{o&;xNd9B(9xp_e@z<#=hp5_^+y?XCIJjDo zaSQw3uI};QdH+R2ko-|V<2(7{{;FhMpz%?;c=2W~7)f!0K-zfNBwjV8KseT4!k-PZ zNWcG=)t>|W7F4sUMWX+Io+#ec5BrZnwWb~u^#9|~>I=)szq)E!m%{(?B&v4xbHe){ zZjvzu5U<)bGKuVeg4I#EL{z_Vbryn!=xUco?KHr|D0q%TeR>vW zU59m=$Pio)sEh-J@f*~5eaF0f!kDi%RvE)2Suh{t*D8_+US}<3S1e}4@KY~U#3B5? z@qPMNv7m#|o2^V|^QH0bpFEX(PgG&AiznGb5UQc+P3!P^b^=lKr_9CSV^y@VUMS+= zPWQ4&O#OsTPSv!{4Ss@KI59+_N>*4Dl&w;NuVIh6c0XpDv3rJ3>P(KbQ?DH=gyBx+nbq{>nxb22(j0Q=lDif5rFU z)dY$NeRM@{$$N64fkNU+39Ay2^PyUA(7HpwzyakN_{)K<72Wp@i#~bP zh*8VJrnjad0j$fELDeEj>>wB<>aoX3hS5al0wS`?VPdu3uva*M-GEt>3F=Cvyrs%~ ziaK60sp4>od*;u$X%5Mi2BlXSJ}*_y#SJk^TN^7@wPoPz!qu`HE3uhLNnxy=>hqE*auo?#XX0;Kf1Ob-ET z_&6a%n@y}7=27u5k^}@w{PNE3`(MH<@}1;E2!=jithVT_Z$QE!TgD`^(up^olJGAZ zq!B4+egg1>>FYkLoDCwJ3VL?eRubkrbsr}XME8mhW|&fk;JfHU}* z`zWHgF&8Z&sRJ9M<$kqyH#0p}&>=&MDAU9APy<`qngu9qKt7zT3B}r$8&iYvr#4($ z5b8s^H{z(dAtXS;iXJUv&JV2@@&q4c&6NdFClk*NJ{jpmrJ$qz2ICQfgp%SE`*#;PheY;<7&!ACRd+q*&WUI5n z^BY`cQxmnEs^vY3DE3GC6nocMhLV&5B~H1})7iXndT_fhDOy{eqLC9FA8-c~fZY%> z7d42iMwtNm;!h}!S(2*?_J_!WFTb`sryHI@OY%J6|K13c=%Z4l8i+c zm8m}Z^yFxNd3#yvgQcMf4Ejdhg$T|ja`gTw*L?-VEQF9t?`_fMM>W+|PTa%b;DPCZ zuC7~HL47|%ya#5Y5N6g7(4K<hgYk31oLSS=euJ0@9)5Qvc4fRnrMTG?|Y{T4i|)dAGC;V z3)u(j#_H^Dm`Y#674hWtgK^`V87^>HrZmi8R+qkS5)5d zF9MH_Au))UJVwNlVZY70jf=o3!*%?#S(h8~nhm=nJS{Cid#-u4h8*WfRApo2IfxF% z3!m}ROUB(?wq12|+@JQ&%nDD5*~f_L`+ESX}0qsuDyjjw*P-ufW_#>}vGCZ_}k!K!NeE`kgpwx3-Cnnt0 zMpv@)c#>yb`sB?Ny^0tv?MiJgQp#X({EmpuZSl`o$HSodkJ+A-pg@KmaVTPAy{E#F zkw}>jqc-sEC)N6#`C9&ruz9^`;VZV35@&r75p**z+MFv?YprJFN0yN++``6WsZJtE z35b&ONo57^7m8&E`ygPB&QD)Z^?`(+m*XlGV8Y7t)iU`AY`4Dc2A^{~vV^tL{J=*^ zZhjX)cnO!O||cp zDF}y2uk+N*wK2#lZRXt2DiR*Eq&zR?LDSTDdMvghgNE?By7orHh=>tG*@J~|I9cMF z$YA^w)4*6W=RetA5F0%`Q-=jDW*pbs-VbRYo><0d=ZDmnGi(gpj&)6i(`+HpGoQHM z2a$g?hbYlTyi}bIkMMa|^Em9NEn%<}&1hBkxYIwxK|fi1=QH;Y(OK4#JRd=}H6L?2 zx5no?TpR{s~vBAYS)l5Bl*V7&3LuM|8MV9xL<_j}3 z8k`0rqR(1juQ@UKWk1Yx9QfSoqbeobFwCR@uN#g1rE=gJpsPHHSJ4eeS=LzB5Pv^! z);(Gepa|5oN>_{?*^wfskVPfla7U#>$P@dKidlk=PH>0!M-n#E)8$0{m^{y%XiNtK z_Rc?17VpwFk-f&D{DmqioD;1>f|PO zZOv-7H?OJe_L~JNq{Lz~rTrh6Y(8GL36w+=S7$P&381llp8w^XvfD$QU8=+wv09-- zQ~H8$p;woAWqt#!7_XhfV)1EdE?ufQVU$;0wSAcL)(uwbo(vS=YWCEc^N|eTsJ2nE zH%N$S7uyb(e97|eJRAgV>y&?CxrDMg&tY*I7n6C2P41c*cb-HApUm_RD{tTHsC|Ho z;GUBYVuv46;Vxy)lE4b9mVyx<+dG1|LJukwdK=(( z2H|oM`NhY`U7n5+S`}+?FDiYO;#0YDos}>Z(Vx-fRmY}%;aWS(HCke&7F>SD z3qWbS_7@uCV60Ip^7;tSX8x-L07O=a@)g#4b~lQ;Mke8C5!=IcVe1S2m8A5@EvS;j zTYtZG;cV`f2n0S zwb)+_A43UxVhc_zc@^txVv41{-sASldJ}f4H@}%aqjhxo+Rd~@{z`5lYi?t02E>N^ z%8XUh>FtK%boqV9^}hH-AZC8!{~@Fgt2g;^5+Lpi07K z`a@>QS^wXku@{)4mo)_lRL$!Hd$YZ6%T3!|jnmseI{8s2& z+nt@btZiz+5I>BCS|6(4&`n}eE?FS>;*~_Z4Ou$-cmDzhRu0eBFY{JCIMyWM1c->2 zsq-&Oj3qpk)|)X)R2ntREAr($?WY=GEfJr81*_3&)ER|bDPJzDsqWWUC#sxJhqWR6 zB0UT=0#4UNv`x6fBeM%XG=wX0xwOweBA*s|6d^4n*Lt!^OKB-lbo8)&c)R?vl-G%B z<2=$q_+KePh@b7Q7E_AhB8kk9-`#!t)1UJZziA{ODD`W6sm3rAqlFp?<+mwFAay^N+iXp0#(oeRULPl$uSMkrE`Bh+*RfQJsPx)|UC7bJPsY`MhL|bKLm8{ng zAjPe{l;1LOJEvKh5;CQ6Jb0goe-{X z@0aM_$oJBO)4AZ6Gbz61oX&lVl|m2|HTDmali;$>RkYlH))i|WfZhuUlGjxU^WNya zCOYU=YRB*Mf@}kvL|H^>+5!O%8!P!8QiVDyu!E&}vHRGwl zCg*m`&S9oj3N*X1cQRT&-%VpbILS~7YV}`uiPBUGbVrJb+$&-{_gz?gT}fff7#*8MT^c>*l>tBbQHhoS%`my;1z|hQ*|OR)cD5kpZTie=$0FQaHnRc79QYH9-V>ecL&q5TKuLeM4H| zIQQb^+ktip6FkSG(PW$AeBwCemRT!arI67egZSnbl0)ei#G}B?_ zVie@Z0z-W&l&Rrb_<9fhUp!-CWY0u3`)|az8@nAyL#SL08vCUmxU~@H2j{v{x0znY zun>!o)rJ)r8g>3UT-ceYZKioLO^0?`R-- zed($F}ja!7nh;OYPnK#a5bYs5_+8-hOS76(OH=iF=oBJ&L9eD33 z0$VuY|9qPg!{nfzyp+~O^`W8dIp#{etsb+B&Ww2uhrpbqy6~b7L(x4@JiWbmDPQgP zCLM@NT`z0Cf1JD_ZnGY?*y*lOzt@M&hj7j2iLf$_9mG;h^_6>o7AZ@ZQ+@F}x* zTsGzB(jf@@pESY{-fMj#^16+^VrgU_YM56b!Q2^-(+G@FTDHH4+#BmEqb8PKNz*K;Nc7Q=tx$x7oI+GnK8^4+l=OM_iVHy-gm`51`%@@l1(lukt-uQcE9#-OW$G%vyNStZWVL zU#hl%`M_NHa|fR5Y0kca&pDFxLK5{Zewx+7paSikWX$dl_75Rx;Y__>QYU zI5&X0CDNI||Gi_a!pXQsk>@O0Y0}my;JonZ=zIZFZok@RukZKF<<4J|YxQ8u0 z%*$uCU3ZEE%gZE0F)f*$hH$(KmFggSkGn~N5h6P8vHG^Fcld@u=+SfYBK>uPTGOo^ zAZ4X>SY;)lXRgRu{W#HYm3_8OcQcjE!cv4W@LA(OIR=o%*T_B-1?obk^Vuewt@a89 z*820-d@I=;~aN@1Vo!_=-@%=0`h571gCq3C*+nU7+Siaxto<<#X_K;{M(@5o!HIT`Vs3ld(r z2Wgtj?lBH2tX7i*pClPZKEknW|o zzBVo%x&5_V{NDDG_EQb%>mRelJ#Pj3m+n!FB~OM23!=Q&9wWvcw(=^K>&e}BX5Znm zu4j1V*nWV_5!l-iWxC)++V1CjV#n5;R8fva5v!WGJ~nu<5qfZ{b{Bd1%6nf-f!Wg7IZ;!LAvwR{%tj7Oo_q;xoA;kwAuNM`y@)+II&C+D<3m&msijd;^M&(TUpck}!2@7QIdX7!83^Yo_IZ{Vaa$g%U z-x3sIP9MEBMpLMIsq%17XVsREDPyqA@6Y77nW?fWh!UA zo%|jz!3bS0wu3dq1VTMV9`0?V=R?wouHjmUGt@Zj_mfEfhN805o6;~YtK_L#;b(7KpxI6*+sLI5IJu+&}amxAkwxB^jV|6|M+VhYjDo(NeoQiIN%F z742L`I!9QxqQRP?&{ZV4m?XO#!j8n)i(sDvk=aX4*3nN&en0p`rISjk_2upe6ja2E z_-sLO#`Z^EA387dbtZ7z>oLClqI?$VEZ*lCGg1Cv!{-Zzca#Utmt7JHzEJYcpGi0| zB5Sh*nl&)IW8h$-gXt3Ge+RAjA_!$Y z9y?I!31k@JYw}-dl7$SAkLOR(TSpxTB0~A=;cKE8`r!K<0089t9)y73%lxP12I(gu aWPt4C{}zGK{=`Lo_mw9A02Vv&H~K%rJNC)| delta 6375 zcmYjWbyQT}+Gckt!T$f`7+*rxl3&ONOHU>wu>9h^GAGQuKhPC67HE|9Ofu&tQVTKDda~pW>m!B*?(WS0w*0fFjRqXZ{I5 zDtc7>PdEkAGuZ8awQw_B5DC&V0Z#kB0f{9PjDIk)5XDXL&%UAub6Wd89voi|M2`e! z+-d$pz+d2ipnqd%;7f8K2KXl&5CO98fcqc*?;SM#p|NrQW5E1I0RIXF3E@{W^}2lg zv#Obzh)6+WAV3^Ox}ONOOC`t3Z{rqQXm9?kw6*6k5n`9Jwp?sV7t&!WBH?Z#DE~bY zJ71c#YJ))1B(tuCR%(KMnTKB-qh6 zF^V(PfSzVr$J1~rEo)=+ zj9>S)YT6hRAgc9OQ^A>IibDL={x2iJOXQ){<**j}4+-^8H4%D=IB2Gq3lhCWRL zv5Th!dLo?9pN8xQn;t!m=+I1|G4VxfBi9glQ6^Y0hx&IWNe>wzNa^8nqttOV+gQdj z=LhBSLWdv%E-`<}64m@s!{UL6V0-H*@y`Ym$h2x0cCb-ey_iwLYHXVf4wrd%91Cr7 zw^}8cp$kTENo>b5Xo#H#!aMG#Z`=(~aSQWB;c)eSoxM(X>x+q&$cuXP=9r?Lc1i+^ zD{tbVkhn?pS9Gv7=a?v&`1YxzHGlnsc)D2JbQfEy znRMAHMQ@fA6^JlDu#?T>crwxn275@p9V@jseRg?@Od4d#tM8%eScc(Mk1IL@9ju5L zVsT&Gp>cDvoQEm-*7?pXhoW=E-w>Z%9fkCWX(GDqS=?RG_=n0(W>|6}f{nSiYe&Dg zXOXOX!|X?&`<$xKd%d*6OIT4A((gP zZKl<-Y+a8x_=G0ot+wyki{!H>wWsg3!@mtOMK0UUl8P5 z!W_Zt*R_wncewq!$rTvaGs z&V0dnj8TMw`-v4d|&s?2|) zTY3do!3N~6_^Q;A1yj9!k7F}D--pgS8cK$$&lv{j0M z%&{GPwm>v-sLSYl``AZBL69!?%J?Z-yBS_pP0 z?cbKh#!!Fd#}Yd>nq>9uAOvHa?UBHS1aO^o#r; zm>ij2vu;`gR{Db#64)Uk0hc#C%k~`CZ_QC~xZme3-6ddiBDsT^d#w#XBypebTtb;y z)LwNlfoKNOsntxRre6cPBzK`BR?bZ?)WZ|T)lVP9ajiZQdv9n(h{$@g>hlF4jNCgs zJ|KjW(fdH=?uf@0`WTwSpea6C&>@uX`gDg+$ri8;$Lx|@^O(5G&yM8pW69UX@vu<; z^!B{`Wi5%qB%DN^fBgze0*azbP$X+esUsmjx*6VDj&OaV-sdSB0^htSU#2Bq@47r} zY~e*Kqhf41RJLukWh}DTE*WX6yx+Vnx@nPo8CPE+t}_w%#1=6gSS{?{&#=sF_ge6R zln~&Fz&XNSTz6vSQn$sx8aj1*HqE-3oKe(GBkt=kh?`SM{g^qV$y(M{Bpi<=h_#gIF-UDNRV-YD6XNi(~> zj(sTB9ZVsnxhv11qfKb703}NscAN_k6a^ZxIx4W<4M0<+t?ThyJelK&3_rfY7zV{! zVA;g2(!2_9EUIIh*J{|HDaDAxL=4Ep(-eOV;joOr75KD)s}va{a3bQ}hrP1FSg=sc zKLLY9d7fyZ(F^kJIk$7WjQ0&^ANY!YqLEI=^L(yGTPTzn!t!ngWI=*m zBbn%Bn!H8H9X~H`ESB42@|^X_Q=n{dgEDz!tM|S0MPqiY&O?@`7TA*kU8mbCkAAQk z%#oELSU26$Bqu~F1GL9Gou?;`VUzO#PL?6wObRc~ZnQ?A2QQqAX$2h>G^Cvf_1M}n zmc(Y0*6wQ?BZ%Y5S9{dMfNFw;`16=IOM-dU_qO%U~K*tgR_#Lks11lgY zUlCPtlm_J=#(RnNg$dFU`G8T18^@!SDt8U1d+BNjdRgKMjZjX5YoF;M6$cDqZ7<20 z?q^+Q@11JA-d1>&IyP@2@Sk1P?RG`*$b3u26AeZnl9kpZ^*yN4;bvYn*&runrtO?J zx~mr)Ec#nA5S0;3MfKjRPR$A4@%&oE>iv@O% zbkMQy=^5tBkHmBOrX+6tO&(UX&#Ry2vxKa2!;3Mye>|OaQ&%qa$sLz%4S5uUpmxwk zD?VoXEayDMg*aRRpY_gnk(A?L@{i*SJZWP(Zf>W@97$Sh1%}4AN?Rvj$lR5KvDNm# zn8T z{8)-Ko?TwAN)dY8o&eMg$70jW^Zbfs9bS5Ai~WvmD!Z{7!x&Y)<7468*wRmnu zG?Cv(>HE|IEzSJI4Ar>5%!gOGg~LC9a)Mp1x5C!B=-I^$mF|OH)ppnAyJ~StFok#r zHavFetZ44Lr+@q`URAI+zr9C|tB?}c9k1LoUbxvjNY|yNH8^ZkOvXs@Tz6K_7#+;n zHu#EmNXVBXlDt8(t--j-ADu7~KyIAuo!xDz6NUZ^)OZDvvuA)a6tkWqAxBC3y%{O< zF2e%JJM{!FqE0Ecsh^@+Sl5>1!G+V0?KdRvd-y@5kXs-h+jq#*?* zREO5Fr%Fj9yw6y-WWa1ZQV_w4nL(rb-VB+%ommeyer3Vl+X{T~=HxIr^=?R4%cIZ5 zrC_xOMjqQuab|Euu;!TYaShV(J`X7q8LWZR=62gVv~HhTQkpv887mU_L(sG!!$9%#kzecCi zso<5oOOD6f$#NasF<|TR_(?7Soh6Scui&$2z$i789VhX=1l;p|5X#-Gnc@;dFe*Yd z-_w!X%~tDwPHOqeWHLd!bXEM}1CVasT|^DWkX zJAcqG2mZDL;@2$&!~^N8*P2)pH}jslp7Nm}(9EZPorn+tR1Rz__=}^%t7@ZP=M>K~ z0CrRLsjX@*IsE3Gsh%bLj)7SD8RknM=TNbOiD)5A+Bqj{Z_iWS)+vtC$NefE`tDbI z*O;x;_(P1-a*s!orGiGO)H3|rH3a=FAz|sxl_RLv)Wrw7!gelvj?r~IZTwXkGT6Y@WZ(xab^mw(v04#M*u0{imZJ z#QgMU`*L!K%3F5_?yH8jgVp#owK<);f{xtWa;$^NxFGgUTG97zvd16Wx&S>s52}&r zJwAOnBtE}lJmmAxi)4S9*Nqzg4<_DM@t)K&vPGU(M0$YYV~acQ!t8?JUm= zi6{V$)|PMKcT(%uSy}U#pRYgBcGhc9HU+xj3!V-B*7>vx6fsHUPOn4(=Y7-MC+xSW z{%QSLv(^_<7(q|1TO?|I$c$xkqTlB6V78M$*Df;0Zoj$Q8<jYe15Lk(!Mf_~%S#ba*1FHE*s+i4%IKtbO^k+~yAoLgS z8;7Up%fVbq^vXDQ-8m2N$OUh@aow6`N-au}e1}HhDinvz1ebL>9HvTF{oy6xgIlEV zQzAb~MEmGx@XGp2;t$vFSmS~k6J@^N`l_l5jN~iN9K`PA3^T>QNFUY#JuP zz@d-k)v3}I{Y<93K7M6RuVTJB^P+4<3{!SEQTkv?c1wn1`F)_ClYH){zw+=vG;Tqz z9sIgU`z*za^eAN!sBFbPNp2`pI z*9B?piL{Y;`EH$sa4bh=MN&<>1+;qWHp$XeXJ}PE(BcgN)@x1UvaNo+S}GN~zv-v_ zBP2TTDLoJ|BtCI8B6hh0)nUM@P8IJWJJ@KkTqb9fDJ}jGBRL zh?=W4+P0JI%ru~XtUzL^Diz-$-#E`2^sLNK#tmym>m(>V_)E#1OZk9*t|dmx8{TJI z#J*T6m$QsJK#MpjOm^>k#V!FDR`Y;OS34saZ^tH*I6&W@dWrE3Bxk?nsP%`P^KB{u zX!W_)6*uQxdT}pY`^XZ_ex$j!XV_F4sMX;`>9$aDJeDp}Pfw;;WxhXE6>E;%u&-ey z`SK>%ZAt9p`z&kv!jmacJFy!8`zIkB_$AJvP;Jl(<{`GB*~ zt&Mxkx%$@xSu8+)@}t%545k>%8qM}m<~LdKE|$MHOP)8kXj5elbb%(C3*~Z-s%U#0 z&aEebJHbtiE=<&o;8*mE4bQKg$PlQyDIwC<&1g-A&hhYfgdkIp^VW1f?AKkN3vlw; zq}Yt;zMq>sPP+G_wkh%6Shx6l_Yx+WaQcz{2CXsbra@0%Z>th;2sH!dZRzp%sW?^* z+hhydu3eQ^ZcZ#)8f?(#KlARl7zcZz8ulLn`FCkZl8=(fN!t!oIuN&hN7ocy~yNggkdao}F^gg;#%Wj#IN%l3YK*Y+p_%V8zOG@I&}q{DCl zEyzV_1l#>z&LX}D`ZV3`Tfr#Sr|;SCey509O-JWfd1gy462X1+AZR|w@rb)WbIBAJjmSU6Oj+1bEvCeqeD)8^&&#JImQ8@ZKhw{D2XQ z7{KEk@Y_5pC4cthGDgE+7SwUYemCoH*a*8P$&!wq)xwzL8(ddd<&ag>*YFT<^xIwX z%V{HLl1YDXc)Gu0a>!28UozvOKCTV|Ojwtg*ql`Qa!GcjSYwpw7ei;`bgg7=T_!vj zR;nsFuJ^H-B~-8)V^K2fZF2_$5oQOobbgjneIlam>@4l{7>VzseFu;@2_aH;V&Ba< zryV|6uG;uF<`g;PjLS9Fhr3}eoG;N1{63-&z9R}064*UEm)k+gnSV~)gi5~!W_viD z!z8#lLofUD(S8g+4jr0Q>Mf%bminXE^D@cuel#duFl2n`df4rt?`a`6kYFUbV#ts` z`t@5SOIUm%d+wJkanR8Fdx0?JA0Y-6&t85v!`5*i=2Ml*wQn7G_T=tERy(GN^^@-X znQJ^mbrEd(r{?85LaX|4;YHHUbbu=P8}C?>7Wf8g30a-!^%oc!GesiasCCxZr&9O| zhSudrGGiw(C9=1^>lQ{HKebEAXFQ?9etO|MBK> zz3hLzLFhL9c~`KZB>F}WWd2_p+)@D~O7^$st)~UTq`>;Gzfb|h05XPmDuDRt{!N1+ T7=MP;pSenjfkDIv`h)%tGO*^x diff --git a/agent-skill/Scrapling-Skill/SKILL.md b/agent-skill/Scrapling-Skill/SKILL.md index 7a009a4..85c54d9 100644 --- a/agent-skill/Scrapling-Skill/SKILL.md +++ b/agent-skill/Scrapling-Skill/SKILL.md @@ -9,9 +9,9 @@ metadata: emoji: "🕷️" homepage: "https://scrapling.readthedocs.io/en/latest/index.html" requires: - bins: + bins: - python3 - anyBins: + anyBins: - pip - pip3 --- From 8e147db7f87c96dc5be3925c8aac2b364124b69a Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Fri, 27 Mar 2026 18:02:33 +0200 Subject: [PATCH 15/33] refactor(cli): Code cleaning for easier maintenance/adding new features Shortened the code by 210 lines. Also, removed docstrings because they are not needed for CLI commands (more maintenance burden). - `_common_http_options`: shared decorator for 10 Click options used by get/post/put/delete (was repeated 4x) - `_common_browser_options`: shared decorator for 11 Click options used by fetch/stealthy_fetch (was repeated 2x) - `_data_options`: shared decorator for `--data`/`--json` options used by post/put - `__http_command()`: shared implementation body for all HTTP commands (was 4 separate `from scrapling.fetchers import Fetcher` + `__Request_and_Save` blocks) - `__build_browser_kwargs()`: shared kwargs builder for fetch/stealthy_fetch (was duplicated) --- scrapling/cli.py | 644 ++++++++++++++++------------------------------- 1 file changed, 217 insertions(+), 427 deletions(-) diff --git a/scrapling/cli.py b/scrapling/cli.py index 59f017e..5a59e86 100644 --- a/scrapling/cli.py +++ b/scrapling/cli.py @@ -194,48 +194,140 @@ def extract(): pass +#### +# Shared Click option decorator factories +#### + + +def _common_http_options(f): + """Apply shared Click options for all HTTP extract commands (get/post/put/delete).""" + decorators = [ + option( + "--stealthy-headers/--no-stealthy-headers", + default=True, + help="Use stealthy browser headers (default: True)", + ), + option( + "--impersonate", + help="Browser to impersonate. Can be a single browser (e.g., chrome) or comma-separated list for random selection (e.g., chrome,firefox,safari).", + ), + option( + "--verify/--no-verify", + default=True, + help="Whether to verify SSL certificates (default: True)", + ), + option( + "--follow-redirects/--no-follow-redirects", + default=True, + help="Whether to follow redirects (default: True)", + ), + option( + "--params", + "-p", + multiple=True, + help='Query parameters in format "key=value" (can be used multiple times)', + ), + option( + "--css-selector", + "-s", + help="CSS selector to extract specific content from the page. It returns all matches.", + ), + option("--proxy", help='Proxy URL in format "http://username:password@host:port"'), + option("--timeout", type=int, default=30, help="Request timeout in seconds (default: 30)"), + option("--cookies", help='Cookies string in format "name1=value1; name2=value2"'), + option( + "--headers", + "-H", + multiple=True, + help='HTTP headers in format "Key: Value" (can be used multiple times)', + ), + ] + for decorator in decorators: + f = decorator(f) + return f + + +def _common_browser_options(f): + """Apply shared Click options for browser-based commands (fetch/stealthy_fetch).""" + decorators = [ + option( + "--extra-headers", + "-H", + multiple=True, + help='Extra headers in format "Key: Value" (can be used multiple times)', + ), + option("--proxy", help='Proxy URL in format "http://username:password@host:port"'), + option( + "--real-chrome/--no-real-chrome", + default=False, + help="If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch an instance of your browser and use it. (default: False)", + ), + option("--locale", default=None, help="Specify user locale. Defaults to the system default locale."), + option("--wait-selector", help="CSS selector to wait for before proceeding"), + option( + "--css-selector", + "-s", + help="CSS selector to extract specific content from the page. It returns all matches.", + ), + option( + "--wait", + type=int, + default=0, + help="Additional wait time in milliseconds after page load (default: 0)", + ), + option( + "--timeout", + type=int, + default=30000, + help="Timeout in milliseconds (default: 30000)", + ), + option( + "--network-idle/--no-network-idle", + default=False, + help="Wait for network idle (default: False)", + ), + option( + "--disable-resources/--enable-resources", + default=False, + help="Drop unnecessary resources for speed boost (default: False)", + ), + option( + "--headless/--no-headless", + default=True, + help="Run browser in headless mode (default: True)", + ), + ] + for decorator in decorators: + f = decorator(f) + return f + + +def _data_options(f): + """Apply data/json options for POST and PUT commands.""" + decorators = [ + option("--json", "-j", help="JSON data to include in the request body (as string)"), + option( + "--data", + "-d", + help='Form data to include in the request body (as string, ex: "param1=value1¶m2=value2")', + ), + ] + for decorator in decorators: + f = decorator(f) + return f + + +def __http_command(method_name: str, url: str, output_file: str, css_selector: Optional[str], **kwargs) -> None: + """Shared implementation for HTTP extract commands.""" + from scrapling.fetchers import Fetcher + + __Request_and_Save(getattr(Fetcher, method_name), url, output_file, css_selector, **kwargs) + + @extract.command(help=f"Perform a GET request and save the content to a file.\n\n{__OUTPUT_FILE_HELP__}") @argument("url", required=True) @argument("output_file", required=True) -@option( - "--headers", - "-H", - multiple=True, - help='HTTP headers in format "Key: Value" (can be used multiple times)', -) -@option("--cookies", help='Cookies string in format "name1=value1; name2=value2"') -@option("--timeout", type=int, default=30, help="Request timeout in seconds (default: 30)") -@option("--proxy", help='Proxy URL in format "http://username:password@host:port"') -@option( - "--css-selector", - "-s", - help="CSS selector to extract specific content from the page. It returns all matches.", -) -@option( - "--params", - "-p", - multiple=True, - help='Query parameters in format "key=value" (can be used multiple times)', -) -@option( - "--follow-redirects/--no-follow-redirects", - default=True, - help="Whether to follow redirects (default: True)", -) -@option( - "--verify/--no-verify", - default=True, - help="Whether to verify SSL certificates (default: True)", -) -@option( - "--impersonate", - help="Browser to impersonate. Can be a single browser (e.g., chrome) or comma-separated list for random selection (e.g., chrome,firefox,safari).", -) -@option( - "--stealthy-headers/--no-stealthy-headers", - default=True, - help="Use stealthy browser headers (default: True)", -) +@_common_http_options def get( url, output_file, @@ -250,23 +342,7 @@ def get( impersonate, stealthy_headers, ): - """ - Perform a GET request and save the content to a file. - - :param url: Target URL for the request. - :param output_file: Output file path (.md for Markdown, .html for HTML). - :param headers: HTTP headers to include in the request. - :param cookies: Cookies to use in the request. - :param timeout: Number of seconds to wait before timing out. - :param proxy: Proxy URL to use. (Format: "http://username:password@localhost:8030") - :param css_selector: CSS selector to extract specific content. - :param params: Query string parameters for the request. - :param follow_redirects: Whether to follow redirects. - :param verify: Whether to verify HTTPS certificates. - :param impersonate: Browser version to impersonate. - :param stealthy_headers: If enabled, creates and adds real browser headers. - """ - + """Perform a GET request and save the content to a file.""" kwargs = __BuildRequest( headers, cookies, @@ -279,59 +355,14 @@ def get( impersonate=impersonate, proxy=proxy, ) - from scrapling.fetchers import Fetcher - - __Request_and_Save(Fetcher.get, url, output_file, css_selector, **kwargs) + __http_command("get", url, output_file, css_selector, **kwargs) @extract.command(help=f"Perform a POST request and save the content to a file.\n\n{__OUTPUT_FILE_HELP__}") @argument("url", required=True) @argument("output_file", required=True) -@option( - "--data", - "-d", - help='Form data to include in the request body (as string, ex: "param1=value1¶m2=value2")', -) -@option("--json", "-j", help="JSON data to include in the request body (as string)") -@option( - "--headers", - "-H", - multiple=True, - help='HTTP headers in format "Key: Value" (can be used multiple times)', -) -@option("--cookies", help='Cookies string in format "name1=value1; name2=value2"') -@option("--timeout", type=int, default=30, help="Request timeout in seconds (default: 30)") -@option("--proxy", help='Proxy URL in format "http://username:password@host:port"') -@option( - "--css-selector", - "-s", - help="CSS selector to extract specific content from the page. It returns all matches.", -) -@option( - "--params", - "-p", - multiple=True, - help='Query parameters in format "key=value" (can be used multiple times)', -) -@option( - "--follow-redirects/--no-follow-redirects", - default=True, - help="Whether to follow redirects (default: True)", -) -@option( - "--verify/--no-verify", - default=True, - help="Whether to verify SSL certificates (default: True)", -) -@option( - "--impersonate", - help="Browser to impersonate. Can be a single browser (e.g., chrome) or comma-separated list for random selection (e.g., chrome,firefox,safari).", -) -@option( - "--stealthy-headers/--no-stealthy-headers", - default=True, - help="Use stealthy browser headers (default: True)", -) +@_data_options +@_common_http_options def post( url, output_file, @@ -348,25 +379,7 @@ def post( impersonate, stealthy_headers, ): - """ - Perform a POST request and save the content to a file. - - :param url: Target URL for the request. - :param output_file: Output file path (.md for Markdown, .html for HTML). - :param data: Form data to include in the request body. (as string, ex: "param1=value1¶m2=value2") - :param json: A JSON serializable object to include in the body of the request. - :param headers: Headers to include in the request. - :param cookies: Cookies to use in the request. - :param timeout: Number of seconds to wait before timing out. - :param proxy: Proxy URL to use. - :param css_selector: CSS selector to extract specific content. - :param params: Query string parameters for the request. - :param follow_redirects: Whether to follow redirects. - :param verify: Whether to verify HTTPS certificates. - :param impersonate: Browser version to impersonate. - :param stealthy_headers: If enabled, creates and adds real browser headers. - """ - + """Perform a POST request and save the content to a file.""" kwargs = __BuildRequest( headers, cookies, @@ -380,55 +393,14 @@ def post( proxy=proxy, data=data, ) - from scrapling.fetchers import Fetcher - - __Request_and_Save(Fetcher.post, url, output_file, css_selector, **kwargs) + __http_command("post", url, output_file, css_selector, **kwargs) @extract.command(help=f"Perform a PUT request and save the content to a file.\n\n{__OUTPUT_FILE_HELP__}") @argument("url", required=True) @argument("output_file", required=True) -@option("--data", "-d", help="Form data to include in the request body") -@option("--json", "-j", help="JSON data to include in the request body (as string)") -@option( - "--headers", - "-H", - multiple=True, - help='HTTP headers in format "Key: Value" (can be used multiple times)', -) -@option("--cookies", help='Cookies string in format "name1=value1; name2=value2"') -@option("--timeout", type=int, default=30, help="Request timeout in seconds (default: 30)") -@option("--proxy", help='Proxy URL in format "http://username:password@host:port"') -@option( - "--css-selector", - "-s", - help="CSS selector to extract specific content from the page. It returns all matches.", -) -@option( - "--params", - "-p", - multiple=True, - help='Query parameters in format "key=value" (can be used multiple times)', -) -@option( - "--follow-redirects/--no-follow-redirects", - default=True, - help="Whether to follow redirects (default: True)", -) -@option( - "--verify/--no-verify", - default=True, - help="Whether to verify SSL certificates (default: True)", -) -@option( - "--impersonate", - help="Browser to impersonate. Can be a single browser (e.g., chrome) or comma-separated list for random selection (e.g., chrome,firefox,safari).", -) -@option( - "--stealthy-headers/--no-stealthy-headers", - default=True, - help="Use stealthy browser headers (default: True)", -) +@_data_options +@_common_http_options def put( url, output_file, @@ -445,25 +417,7 @@ def put( impersonate, stealthy_headers, ): - """ - Perform a PUT request and save the content to a file. - - :param url: Target URL for the request. - :param output_file: Output file path (.md for Markdown, .html for HTML). - :param data: Form data to include in the request body. - :param json: A JSON serializable object to include in the body of the request. - :param headers: Headers to include in the request. - :param cookies: Cookies to use in the request. - :param timeout: Number of seconds to wait before timing out. - :param proxy: Proxy URL to use. - :param css_selector: CSS selector to extract specific content. - :param params: Query string parameters for the request. - :param follow_redirects: Whether to follow redirects. - :param verify: Whether to verify HTTPS certificates. - :param impersonate: Browser version to impersonate. - :param stealthy_headers: If enabled, creates and adds real browser headers. - """ - + """Perform a PUT request and save the content to a file.""" kwargs = __BuildRequest( headers, cookies, @@ -477,53 +431,13 @@ def put( proxy=proxy, data=data, ) - from scrapling.fetchers import Fetcher - - __Request_and_Save(Fetcher.put, url, output_file, css_selector, **kwargs) + __http_command("put", url, output_file, css_selector, **kwargs) @extract.command(help=f"Perform a DELETE request and save the content to a file.\n\n{__OUTPUT_FILE_HELP__}") @argument("url", required=True) @argument("output_file", required=True) -@option( - "--headers", - "-H", - multiple=True, - help='HTTP headers in format "Key: Value" (can be used multiple times)', -) -@option("--cookies", help='Cookies string in format "name1=value1; name2=value2"') -@option("--timeout", type=int, default=30, help="Request timeout in seconds (default: 30)") -@option("--proxy", help='Proxy URL in format "http://username:password@host:port"') -@option( - "--css-selector", - "-s", - help="CSS selector to extract specific content from the page. It returns all matches.", -) -@option( - "--params", - "-p", - multiple=True, - help='Query parameters in format "key=value" (can be used multiple times)', -) -@option( - "--follow-redirects/--no-follow-redirects", - default=True, - help="Whether to follow redirects (default: True)", -) -@option( - "--verify/--no-verify", - default=True, - help="Whether to verify SSL certificates (default: True)", -) -@option( - "--impersonate", - help="Browser to impersonate. Can be a single browser (e.g., chrome) or comma-separated list for random selection (e.g., chrome,firefox,safari).", -) -@option( - "--stealthy-headers/--no-stealthy-headers", - default=True, - help="Use stealthy browser headers (default: True)", -) +@_common_http_options def delete( url, output_file, @@ -538,23 +452,7 @@ def delete( impersonate, stealthy_headers, ): - """ - Perform a DELETE request and save the content to a file. - - :param url: Target URL for the request. - :param output_file: Output file path (.md for Markdown, .html for HTML). - :param headers: Headers to include in the request. - :param cookies: Cookies to use in the request. - :param timeout: Number of seconds to wait before timing out. - :param proxy: Proxy URL to use. - :param css_selector: CSS selector to extract specific content. - :param params: Query string parameters for the request. - :param follow_redirects: Whether to follow redirects. - :param verify: Whether to verify HTTPS certificates. - :param impersonate: Browser version to impersonate. - :param stealthy_headers: If enabled, creates and adds real browser headers. - """ - + """Perform a DELETE request and save the content to a file.""" kwargs = __BuildRequest( headers, cookies, @@ -567,60 +465,45 @@ def delete( impersonate=impersonate, proxy=proxy, ) - from scrapling.fetchers import Fetcher + __http_command("delete", url, output_file, css_selector, **kwargs) - __Request_and_Save(Fetcher.delete, url, output_file, css_selector, **kwargs) + +def __build_browser_kwargs( + headless, + disable_resources, + network_idle, + timeout, + wait, + wait_selector, + locale, + real_chrome, + proxy, + parsed_headers, +) -> Dict[str, Any]: + """Build shared kwargs dict for browser-based commands.""" + kwargs: Dict[str, Any] = { + "headless": headless, + "disable_resources": disable_resources, + "network_idle": network_idle, + "timeout": timeout, + "locale": locale, + "real_chrome": real_chrome, + } + if wait > 0: + kwargs["wait"] = wait + if wait_selector: + kwargs["wait_selector"] = wait_selector + if proxy: + kwargs["proxy"] = proxy + if parsed_headers: + kwargs["extra_headers"] = parsed_headers + return kwargs @extract.command(help=f"Use DynamicFetcher to fetch content with browser automation.\n\n{__OUTPUT_FILE_HELP__}") @argument("url", required=True) @argument("output_file", required=True) -@option( - "--headless/--no-headless", - default=True, - help="Run browser in headless mode (default: True)", -) -@option( - "--disable-resources/--enable-resources", - default=False, - help="Drop unnecessary resources for speed boost (default: False)", -) -@option( - "--network-idle/--no-network-idle", - default=False, - help="Wait for network idle (default: False)", -) -@option( - "--timeout", - type=int, - default=30000, - help="Timeout in milliseconds (default: 30000)", -) -@option( - "--wait", - type=int, - default=0, - help="Additional wait time in milliseconds after page load (default: 0)", -) -@option( - "--css-selector", - "-s", - help="CSS selector to extract specific content from the page. It returns all matches.", -) -@option("--wait-selector", help="CSS selector to wait for before proceeding") -@option("--locale", default=None, help="Specify user locale. Defaults to the system default locale.") -@option( - "--real-chrome/--no-real-chrome", - default=False, - help="If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch an instance of your browser and use it. (default: False)", -) -@option("--proxy", help='Proxy URL in format "http://username:password@host:port"') -@option( - "--extra-headers", - "-H", - multiple=True, - help='Extra headers in format "Key: Value" (can be used multiple times)', -) +@_common_browser_options def fetch( url, output_file, @@ -636,46 +519,20 @@ def fetch( proxy, extra_headers, ): - """ - Opens up a browser and fetch content using DynamicFetcher. - - :param url: Target url. - :param output_file: Output file path (.md for Markdown, .html for HTML). - :param headless: Run the browser in headless/hidden or headful/visible mode. - :param disable_resources: Drop requests of unnecessary resources for a speed boost. - :param network_idle: Wait for the page until there are no network connections for at least 500 ms. - :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. - :param wait: The time (milliseconds) the fetcher will wait after everything finishes before returning. - :param css_selector: CSS selector to extract specific content. - :param wait_selector: Wait for a specific CSS selector to be in a specific state. - :param locale: Set the locale for the browser. - :param real_chrome: If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch an instance of your browser and use it. - :param proxy: The proxy to be used with requests. - :param extra_headers: Extra headers to add to the request. - """ - - # Parse parameters + """Opens up a browser and fetch content using DynamicFetcher.""" parsed_headers, _ = _ParseHeaders(extra_headers, False) - - # Build request arguments - kwargs = { - "headless": headless, - "disable_resources": disable_resources, - "network_idle": network_idle, - "timeout": timeout, - "locale": locale, - "real_chrome": real_chrome, - } - - if wait > 0: - kwargs["wait"] = wait - if wait_selector: - kwargs["wait_selector"] = wait_selector - if proxy: - kwargs["proxy"] = proxy - if parsed_headers: - kwargs["extra_headers"] = parsed_headers - + kwargs = __build_browser_kwargs( + headless, + disable_resources, + network_idle, + timeout, + wait, + wait_selector, + locale, + real_chrome, + proxy, + parsed_headers, + ) from scrapling.fetchers import DynamicFetcher __Request_and_Save(DynamicFetcher.fetch, url, output_file, css_selector, **kwargs) @@ -684,16 +541,6 @@ def fetch( @extract.command(help=f"Use StealthyFetcher to fetch content with advanced stealth features.\n\n{__OUTPUT_FILE_HELP__}") @argument("url", required=True) @argument("output_file", required=True) -@option( - "--headless/--no-headless", - default=True, - help="Run browser in headless mode (default: True)", -) -@option( - "--disable-resources/--enable-resources", - default=False, - help="Drop unnecessary resources for speed boost (default: False)", -) @option( "--block-webrtc/--allow-webrtc", default=False, @@ -705,110 +552,53 @@ def fetch( help="Solve Cloudflare challenges (default: False)", ) @option("--allow-webgl/--block-webgl", default=True, help="Allow WebGL (default: True)") -@option( - "--network-idle/--no-network-idle", - default=False, - help="Wait for network idle (default: False)", -) -@option( - "--real-chrome/--no-real-chrome", - default=False, - help="If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch an instance of your browser and use it. (default: False)", -) @option( "--hide-canvas/--show-canvas", default=False, help="Add noise to canvas operations (default: False)", ) -@option( - "--timeout", - type=int, - default=30000, - help="Timeout in milliseconds (default: 30000)", -) -@option( - "--wait", - type=int, - default=0, - help="Additional wait time in milliseconds after page load (default: 0)", -) -@option( - "--css-selector", - "-s", - help="CSS selector to extract specific content from the page. It returns all matches.", -) -@option("--wait-selector", help="CSS selector to wait for before proceeding") -@option("--proxy", help='Proxy URL in format "http://username:password@host:port"') -@option( - "--extra-headers", - "-H", - multiple=True, - help='Extra headers in format "Key: Value" (can be used multiple times)', -) +@_common_browser_options def stealthy_fetch( url, output_file, headless, disable_resources, - block_webrtc, - solve_cloudflare, - allow_webgl, network_idle, - real_chrome, - hide_canvas, timeout, wait, css_selector, wait_selector, + locale, + real_chrome, proxy, extra_headers, + block_webrtc, + solve_cloudflare, + allow_webgl, + hide_canvas, ): - """ - Opens up a browser with advanced stealth features and fetch content using StealthyFetcher. - - :param url: Target url. - :param output_file: Output file path (.md for Markdown, .html for HTML). - :param headless: Run the browser in headless/hidden, or headful/visible mode. - :param disable_resources: Drop requests of unnecessary resources for a speed boost. - :param block_webrtc: Blocks WebRTC entirely. - :param solve_cloudflare: Solves all types of the Cloudflare's Turnstile/Interstitial challenges. - :param allow_webgl: Allow WebGL (recommended to keep enabled). - :param network_idle: Wait for the page until there are no network connections for at least 500 ms. - :param real_chrome: If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch an instance of your browser and use it. - :param hide_canvas: Add random noise to canvas operations to prevent fingerprinting. - :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. - :param wait: The time (milliseconds) the fetcher will wait after everything finishes before returning. - :param css_selector: CSS selector to extract specific content. - :param wait_selector: Wait for a specific CSS selector to be in a specific state. - :param proxy: The proxy to be used with requests. - :param extra_headers: Extra headers to add to the request. - """ - - # Parse parameters + """Opens up a browser with advanced stealth features and fetch content using StealthyFetcher.""" parsed_headers, _ = _ParseHeaders(extra_headers, False) - - # Build request arguments - kwargs = { - "headless": headless, - "disable_resources": disable_resources, - "block_webrtc": block_webrtc, - "solve_cloudflare": solve_cloudflare, - "allow_webgl": allow_webgl, - "network_idle": network_idle, - "real_chrome": real_chrome, - "hide_canvas": hide_canvas, - "timeout": timeout, - } - - if wait > 0: - kwargs["wait"] = wait - if wait_selector: - kwargs["wait_selector"] = wait_selector - if proxy: - kwargs["proxy"] = proxy - if parsed_headers: - kwargs["extra_headers"] = parsed_headers - + kwargs = __build_browser_kwargs( + headless, + disable_resources, + network_idle, + timeout, + wait, + wait_selector, + locale, + real_chrome, + proxy, + parsed_headers, + ) + kwargs.update( + { + "block_webrtc": block_webrtc, + "solve_cloudflare": solve_cloudflare, + "allow_webgl": allow_webgl, + "hide_canvas": hide_canvas, + } + ) from scrapling.fetchers import StealthyFetcher __Request_and_Save(StealthyFetcher.fetch, url, output_file, css_selector, **kwargs) From a356dd2f2b0e317e8475cbe912d2e78c9e682177 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Fri, 27 Mar 2026 19:30:58 +0200 Subject: [PATCH 16/33] refactor(mcp)!: Cleaning and unifying functions to async - `get()` now delegates to `bulk_get([url])[0]` (was a separate sync implementation) - `fetch()` now delegates to `bulk_fetch([url])[0]` (eliminated duplicate fetcher call) - `stealthy_fetch()` now delegates to `bulk_stealthy_fetch([url])[0]` (same) - Replaced 6x repeated `_content_translator(Convertor._extract_content(...), page)` with a single `_translate_response()` helper - Removed unused imports (`Fetcher`, `DynamicFetcher`, `StealthyFetcher`, `Generator`) --- scrapling/core/ai.py | 186 ++++++++++++++++------------------------ tests/ai/test_ai_mcp.py | 5 +- 2 files changed, 75 insertions(+), 116 deletions(-) diff --git a/scrapling/core/ai.py b/scrapling/core/ai.py index 3481970..0975a92 100644 --- a/scrapling/core/ai.py +++ b/scrapling/core/ai.py @@ -7,11 +7,8 @@ from scrapling.core.shell import Convertor from scrapling.engines.toolbelt.custom import Response as _ScraplingResponse from scrapling.engines.static import ImpersonateType from scrapling.fetchers import ( - Fetcher, FetcherSession, - DynamicFetcher, AsyncDynamicSession, - StealthyFetcher, AsyncStealthySession, ) from scrapling.core._types import ( @@ -21,7 +18,6 @@ from scrapling.core._types import ( Dict, List, Any, - Generator, Sequence, SetCookieParam, extraction_types, @@ -37,9 +33,22 @@ class ResponseModel(BaseModel): url: str = Field(description="The URL given by the user that resulted in this response.") -def _content_translator(content: Generator[str, None, None], page: _ScraplingResponse) -> ResponseModel: - """Convert a content generator to a list of ResponseModel objects.""" - return ResponseModel(status=page.status, content=[result for result in content], url=page.url) +def _translate_response( + page: _ScraplingResponse, + extraction_type: extraction_types, + css_selector: Optional[str], + main_content_only: bool, +) -> ResponseModel: + """Extract content from a response and translate it to a ResponseModel.""" + content = list( + Convertor._extract_content( + page, + css_selector=css_selector, + extraction_type=extraction_type, + main_content_only=main_content_only, + ) + ) + return ResponseModel(status=page.status, content=content, url=page.url) def _normalize_credentials(credentials: Optional[Dict[str, str]]) -> Optional[Tuple[str, str]]: @@ -58,7 +67,7 @@ def _normalize_credentials(credentials: Optional[Dict[str, str]]) -> Optional[Tu class ScraplingMCPServer: @staticmethod - def get( + async def get( url: str, impersonate: ImpersonateType = "chrome", extraction_type: extraction_types = "markdown", @@ -107,36 +116,28 @@ class ScraplingMCPServer: :param http3: Whether to use HTTP3. Defaults to False. It might be problematic if used it with `impersonate`. :param stealthy_headers: If enabled (default), it creates and adds real browser headers. It also sets a Google referer header. """ - normalized_proxy_auth = _normalize_credentials(proxy_auth) - normalized_auth = _normalize_credentials(auth) - - page = Fetcher.get( - url, - auth=normalized_auth, - proxy=proxy, - http3=http3, - verify=verify, - params=params, - proxy_auth=normalized_proxy_auth, - retry_delay=retry_delay, - stealthy_headers=stealthy_headers, + results = await ScraplingMCPServer.bulk_get( + urls=[url], impersonate=impersonate, + extraction_type=extraction_type, + css_selector=css_selector, + main_content_only=main_content_only, + params=params, headers=headers, cookies=cookies, timeout=timeout, - retries=retries, - max_redirects=max_redirects, follow_redirects=follow_redirects, + max_redirects=max_redirects, + retries=retries, + retry_delay=retry_delay, + proxy=proxy, + proxy_auth=proxy_auth, + auth=auth, + verify=verify, + http3=http3, + stealthy_headers=stealthy_headers, ) - return _content_translator( - Convertor._extract_content( - page, - css_selector=css_selector, - extraction_type=extraction_type, - main_content_only=main_content_only, - ), - page, - ) + return results[0] @staticmethod async def bulk_get( @@ -214,18 +215,7 @@ class ScraplingMCPServer: for url in urls ] responses = await gather(*tasks) - return [ - _content_translator( - Convertor._extract_content( - page, - css_selector=css_selector, - extraction_type=extraction_type, - main_content_only=main_content_only, - ), - page, - ) - for page in responses - ] + return [_translate_response(page, extraction_type, css_selector, main_content_only) for page in responses] @staticmethod async def fetch( @@ -280,34 +270,29 @@ class ScraplingMCPServer: :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by `google_search` takes priority over the referer set here if used together._ :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only. """ - page = await DynamicFetcher.async_fetch( - url, + results = await ScraplingMCPServer.bulk_fetch( + urls=[url], + extraction_type=extraction_type, + css_selector=css_selector, + main_content_only=main_content_only, + headless=headless, + google_search=google_search, + real_chrome=real_chrome, wait=wait, proxy=proxy, - locale=locale, - timeout=timeout, - cookies=cookies, - cdp_url=cdp_url, - headless=headless, - useragent=useragent, timezone_id=timezone_id, - real_chrome=real_chrome, - network_idle=network_idle, - wait_selector=wait_selector, + locale=locale, extra_headers=extra_headers, - google_search=google_search, + useragent=useragent, + cdp_url=cdp_url, + timeout=timeout, disable_resources=disable_resources, + wait_selector=wait_selector, + cookies=cookies, + network_idle=network_idle, wait_selector_state=wait_selector_state, ) - return _content_translator( - Convertor._extract_content( - page, - css_selector=css_selector, - extraction_type=extraction_type, - main_content_only=main_content_only, - ), - page, - ) + return results[0] @staticmethod async def bulk_fetch( @@ -383,18 +368,7 @@ class ScraplingMCPServer: ) as session: tasks = [session.fetch(url) for url in urls] responses = await gather(*tasks) - return [ - _content_translator( - Convertor._extract_content( - page, - css_selector=css_selector, - extraction_type=extraction_type, - main_content_only=main_content_only, - ), - page, - ) - for page in responses - ] + return [_translate_response(page, extraction_type, css_selector, main_content_only) for page in responses] @staticmethod async def stealthy_fetch( @@ -459,39 +433,34 @@ class ScraplingMCPServer: :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only. :param additional_args: Additional arguments to be passed to Playwright's context as additional settings, and it takes higher priority than Scrapling's settings. """ - page = await StealthyFetcher.async_fetch( - url, + results = await ScraplingMCPServer.bulk_stealthy_fetch( + urls=[url], + extraction_type=extraction_type, + css_selector=css_selector, + main_content_only=main_content_only, + headless=headless, + google_search=google_search, + real_chrome=real_chrome, wait=wait, proxy=proxy, + timezone_id=timezone_id, locale=locale, + extra_headers=extra_headers, + useragent=useragent, + hide_canvas=hide_canvas, cdp_url=cdp_url, timeout=timeout, - cookies=cookies, - headless=headless, - useragent=useragent, - timezone_id=timezone_id, - real_chrome=real_chrome, - hide_canvas=hide_canvas, - allow_webgl=allow_webgl, - network_idle=network_idle, - block_webrtc=block_webrtc, - wait_selector=wait_selector, - google_search=google_search, - extra_headers=extra_headers, - additional_args=additional_args, - solve_cloudflare=solve_cloudflare, disable_resources=disable_resources, + wait_selector=wait_selector, + cookies=cookies, + network_idle=network_idle, wait_selector_state=wait_selector_state, + block_webrtc=block_webrtc, + allow_webgl=allow_webgl, + solve_cloudflare=solve_cloudflare, + additional_args=additional_args, ) - return _content_translator( - Convertor._extract_content( - page, - css_selector=css_selector, - extraction_type=extraction_type, - main_content_only=main_content_only, - ), - page, - ) + return results[0] @staticmethod async def bulk_stealthy_fetch( @@ -581,18 +550,7 @@ class ScraplingMCPServer: ) as session: tasks = [session.fetch(url) for url in urls] responses = await gather(*tasks) - return [ - _content_translator( - Convertor._extract_content( - page, - css_selector=css_selector, - extraction_type=extraction_type, - main_content_only=main_content_only, - ), - page, - ) - for page in responses - ] + return [_translate_response(page, extraction_type, css_selector, main_content_only) for page in responses] def serve(self, http: bool, host: str, port: int): """Serve the MCP server.""" diff --git a/tests/ai/test_ai_mcp.py b/tests/ai/test_ai_mcp.py index b2c4a71..305edf5 100644 --- a/tests/ai/test_ai_mcp.py +++ b/tests/ai/test_ai_mcp.py @@ -16,9 +16,10 @@ class TestMCPServer: def server(self): return ScraplingMCPServer() - def test_get_tool(self, server, test_url): + @pytest.mark.asyncio + async def test_get_tool(self, server, test_url): """Test the get tool method""" - result = server.get(url=test_url, extraction_type="markdown") + result = await server.get(url=test_url, extraction_type="markdown") assert isinstance(result, ResponseModel) assert result.status == 200 assert result.url == test_url From c458ab65a2a0a5dc1c5e1d53b29f273ea29b4734 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Sat, 28 Mar 2026 00:16:14 +0200 Subject: [PATCH 17/33] feat(mcp): Add three new tools to control browser sessions Now you can open a browser, keep using it for other requests as you want, and close it when you want. --- scrapling/core/ai.py | 374 ++++++++++++++++++++++++++++++++++------ tests/ai/test_ai_mcp.py | 121 ++++++++++++- 2 files changed, 441 insertions(+), 54 deletions(-) diff --git a/scrapling/core/ai.py b/scrapling/core/ai.py index 0975a92..85ceefd 100644 --- a/scrapling/core/ai.py +++ b/scrapling/core/ai.py @@ -1,4 +1,7 @@ +from uuid import uuid4 from asyncio import gather +from datetime import datetime, timezone +from dataclasses import dataclass, field from mcp.server.fastmcp import FastMCP from pydantic import BaseModel, Field @@ -13,6 +16,7 @@ from scrapling.fetchers import ( ) from scrapling.core._types import ( Optional, + Literal, Tuple, Mapping, Dict, @@ -24,6 +28,8 @@ from scrapling.core._types import ( SelectorWaitStates, ) +SessionType = Literal["dynamic", "stealthy"] + class ResponseModel(BaseModel): """Request's response information structure.""" @@ -33,6 +39,35 @@ class ResponseModel(BaseModel): url: str = Field(description="The URL given by the user that resulted in this response.") +class SessionInfo(BaseModel): + """Information about an open browser session.""" + + session_id: str = Field(description="The unique identifier of the session.") + session_type: SessionType = Field(description="The type of the session: 'dynamic' or 'stealthy'.") + created_at: str = Field(description="ISO timestamp of when the session was created.") + is_alive: bool = Field(description="Whether the session is still alive and usable.") + + +class SessionCreatedModel(SessionInfo): + """Response returned when a new session is created.""" + + message: str = Field(description="A confirmation message.") + + +class SessionClosedModel(BaseModel): + """Response returned when a session is closed.""" + + session_id: str = Field(description="The unique identifier of the closed session.") + message: str = Field(description="A confirmation message.") + + +@dataclass +class _SessionEntry: + session: Any # AsyncDynamicSession | AsyncStealthySession + session_type: SessionType + created_at: str = field(default_factory=lambda: datetime.now(timezone.utc).isoformat()) + + def _translate_response( page: _ScraplingResponse, extraction_type: extraction_types, @@ -65,7 +100,177 @@ def _normalize_credentials(credentials: Optional[Dict[str, str]]) -> Optional[Tu return username, password +def _build_fetch_kwargs(**params) -> Dict[str, Any]: + """Build kwargs dict for session.fetch() from request-level parameters.""" + kwargs: Dict[str, Any] = {} + # These are the params that session.fetch() accepts as per-request overrides + request_level_keys = ( + "wait", + "timeout", + "google_search", + "extra_headers", + "disable_resources", + "wait_selector", + "wait_selector_state", + "network_idle", + "proxy", + "solve_cloudflare", + ) + for key in request_level_keys: + if key in params and params[key] is not None: + kwargs[key] = params[key] + return kwargs + + class ScraplingMCPServer: + def __init__(self): + self._sessions: Dict[str, _SessionEntry] = {} + + def _get_session(self, session_id: str, expected_type: SessionType) -> _SessionEntry: + """Look up a session by ID and validate its type.""" + entry = self._sessions.get(session_id) + if entry is None: + raise ValueError(f"Session '{session_id}' not found. Use list_sessions to see active sessions.") + if not entry.session._is_alive: + raise ValueError(f"Session '{session_id}' is no longer alive. Open a new session.") + if entry.session_type != expected_type: + raise ValueError( + f"Session '{session_id}' is a '{entry.session_type}' session, but this tool requires a " + f"'{expected_type}' session. Use the matching fetch tool for your session type." + ) + return entry + + async def open_session( + self, + session_type: SessionType, + headless: bool = True, + google_search: bool = True, + real_chrome: bool = False, + wait: int | float = 0, + proxy: Optional[str | Dict[str, str]] = None, + timezone_id: str | None = None, + locale: str | None = None, + extra_headers: Optional[Dict[str, str]] = None, + useragent: Optional[str] = None, + cdp_url: Optional[str] = None, + timeout: int | float = 30000, + disable_resources: bool = False, + wait_selector: Optional[str] = None, + cookies: Sequence[SetCookieParam] | None = None, + network_idle: bool = False, + wait_selector_state: SelectorWaitStates = "attached", + max_pages: int = 5, + # Stealthy-only params (ignored for dynamic sessions) + hide_canvas: bool = False, + block_webrtc: bool = False, + allow_webgl: bool = True, + solve_cloudflare: bool = False, + additional_args: Optional[Dict] = None, + ) -> SessionCreatedModel: + """Open a persistent browser session that can be reused across multiple fetch calls. + This avoids the overhead of launching a new browser for each request. + Use close_session to close the session when done, and list_sessions to see all active sessions. + + :param session_type: The type of session to open. Use "dynamic" for standard Playwright browser, or "stealthy" for anti-bot bypass with fingerprint spoofing. + :param headless: Run the browser in headless/hidden (default), or headful/visible mode. + :param google_search: Enabled by default, Scrapling will set a Google referer header. + :param real_chrome: If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch an instance of your browser and use it. + :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the Response object. + :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only. + :param timezone_id: Changes the timezone of the browser. Defaults to the system timezone. + :param locale: Specify user locale, for example, `en-GB`, `de-DE`, etc. + :param extra_headers: A dictionary of extra headers to add to the request. + :param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it. + :param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers through CDP. + :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000. + :param disable_resources: Drop requests for unnecessary resources for a speed boost. + :param wait_selector: Wait for a specific CSS selector to be in a specific state. + :param cookies: Set cookies for the session. It should be in a dictionary format that Playwright accepts. + :param network_idle: Wait for the page until there are no network connections for at least 500 ms. + :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`. + :param max_pages: Maximum number of concurrent pages/tabs in the browser. Defaults to 5. Higher values allow more parallel fetches. + :param hide_canvas: (Stealthy only) Add random noise to canvas operations to prevent fingerprinting. + :param block_webrtc: (Stealthy only) Forces WebRTC to respect proxy settings to prevent local IP address leak. + :param allow_webgl: (Stealthy only) Enabled by default. Disabling WebGL is not recommended as many WAFs now check if WebGL is enabled. + :param solve_cloudflare: (Stealthy only) Solves all types of the Cloudflare's Turnstile/Interstitial challenges. + :param additional_args: (Stealthy only) Additional arguments to be passed to Playwright's context as additional settings. + """ + common_kwargs: Dict[str, Any] = dict( + wait=wait, + proxy=proxy, + locale=locale, + timeout=timeout, + cookies=cookies, + cdp_url=cdp_url, + headless=headless, + max_pages=max_pages, + useragent=useragent, + timezone_id=timezone_id, + real_chrome=real_chrome, + network_idle=network_idle, + wait_selector=wait_selector, + google_search=google_search, + extra_headers=extra_headers, + disable_resources=disable_resources, + wait_selector_state=wait_selector_state, + ) + + if session_type == "stealthy": + session = AsyncStealthySession( + **common_kwargs, + hide_canvas=hide_canvas, + block_webrtc=block_webrtc, + allow_webgl=allow_webgl, + solve_cloudflare=solve_cloudflare, + additional_args=additional_args, + ) + else: + session = AsyncDynamicSession(**common_kwargs) + + await session.start() + + session_id = uuid4().hex[:12] + entry = _SessionEntry(session=session, session_type=session_type) + self._sessions[session_id] = entry + + return SessionCreatedModel( + session_id=session_id, + session_type=session_type, + created_at=entry.created_at, + is_alive=True, + message=f"Session '{session_id}' ({session_type}) created successfully.", + ) + + async def close_session( + self, + session_id: str, + ) -> SessionClosedModel: + """Close a persistent browser session and free its resources. + + :param session_id: The unique identifier of the session to close. Use list_sessions to see active sessions. + """ + entry = self._sessions.pop(session_id, None) + if entry is None: + raise ValueError(f"Session '{session_id}' not found. Use list_sessions to see active sessions.") + + await entry.session.close() + return SessionClosedModel( + session_id=session_id, + message=f"Session '{session_id}' closed successfully.", + ) + + async def list_sessions(self) -> List[SessionInfo]: + """List all active browser sessions with their details.""" + return [ + SessionInfo( + session_id=sid, + session_type=entry.session_type, + created_at=entry.created_at, + is_alive=entry.session._is_alive, + ) + for sid, entry in self._sessions.items() + ] + @staticmethod async def get( url: str, @@ -217,8 +422,8 @@ class ScraplingMCPServer: responses = await gather(*tasks) return [_translate_response(page, extraction_type, css_selector, main_content_only) for page in responses] - @staticmethod async def fetch( + self, url: str, extraction_type: extraction_types = "markdown", css_selector: Optional[str] = None, @@ -239,10 +444,13 @@ class ScraplingMCPServer: cookies: Sequence[SetCookieParam] | None = None, network_idle: bool = False, wait_selector_state: SelectorWaitStates = "attached", + session_id: Optional[str] = None, ) -> ResponseModel: """Use playwright to open a browser to fetch a URL and return a structured output of the result. Note: This is only suitable for low-mid protection levels. Note: If the `css_selector` resolves to more than one element, all the elements will be returned. + Note: If a `session_id` is provided (from open_session), the browser session will be reused instead of creating a new one. + When using a session, browser-level params (headless, proxy, locale, etc.) are ignored — they were set at session creation time. :param url: The URL to request. :param extraction_type: The type of content to extract from the page. Defaults to "markdown". Options are: @@ -269,8 +477,9 @@ class ScraplingMCPServer: :param google_search: Enabled by default, Scrapling will set a Google referer header. :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by `google_search` takes priority over the referer set here if used together._ :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only. + :param session_id: Optional session ID from open_session. If provided, reuses the existing browser session instead of creating a new one. """ - results = await ScraplingMCPServer.bulk_fetch( + results = await self.bulk_fetch( urls=[url], extraction_type=extraction_type, css_selector=css_selector, @@ -291,11 +500,12 @@ class ScraplingMCPServer: cookies=cookies, network_idle=network_idle, wait_selector_state=wait_selector_state, + session_id=session_id, ) return results[0] - @staticmethod async def bulk_fetch( + self, urls: List[str], extraction_type: extraction_types = "markdown", css_selector: Optional[str] = None, @@ -316,10 +526,13 @@ class ScraplingMCPServer: cookies: Sequence[SetCookieParam] | None = None, network_idle: bool = False, wait_selector_state: SelectorWaitStates = "attached", + session_id: Optional[str] = None, ) -> List[ResponseModel]: """Use playwright to open a browser, then fetch a group of URLs at the same time, and for each page return a structured output of the result. Note: This is only suitable for low-mid protection levels. Note: If the `css_selector` resolves to more than one element, all the elements will be returned. + Note: If a `session_id` is provided (from open_session), the browser session will be reused instead of creating a new one. + When using a session, browser-level params (headless, proxy, locale, etc.) are ignored — they were set at session creation time. :param urls: A list of the URLs to request. :param extraction_type: The type of content to extract from the page. Defaults to "markdown". Options are: @@ -346,32 +559,50 @@ class ScraplingMCPServer: :param google_search: Enabled by default, Scrapling will set a Google referer header. :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by `google_search` takes priority over the referer set here if used together._ :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only. + :param session_id: Optional session ID from open_session. If provided, reuses the existing browser session instead of creating a new one. """ - async with AsyncDynamicSession( - wait=wait, - proxy=proxy, - locale=locale, - timeout=timeout, - cookies=cookies, - cdp_url=cdp_url, - headless=headless, - max_pages=len(urls), - useragent=useragent, - timezone_id=timezone_id, - real_chrome=real_chrome, - network_idle=network_idle, - wait_selector=wait_selector, - google_search=google_search, - extra_headers=extra_headers, - disable_resources=disable_resources, - wait_selector_state=wait_selector_state, - ) as session: - tasks = [session.fetch(url) for url in urls] + if session_id: + entry = self._get_session(session_id, "dynamic") + fetch_kwargs = _build_fetch_kwargs( + wait=wait, + timeout=timeout, + google_search=google_search, + extra_headers=extra_headers, + disable_resources=disable_resources, + wait_selector=wait_selector, + wait_selector_state=wait_selector_state, + network_idle=network_idle, + proxy=proxy, + ) + tasks = [entry.session.fetch(url, **fetch_kwargs) for url in urls] responses = await gather(*tasks) - return [_translate_response(page, extraction_type, css_selector, main_content_only) for page in responses] + else: + async with AsyncDynamicSession( + wait=wait, + proxy=proxy, + locale=locale, + timeout=timeout, + cookies=cookies, + cdp_url=cdp_url, + headless=headless, + max_pages=len(urls), + useragent=useragent, + timezone_id=timezone_id, + real_chrome=real_chrome, + network_idle=network_idle, + wait_selector=wait_selector, + google_search=google_search, + extra_headers=extra_headers, + disable_resources=disable_resources, + wait_selector_state=wait_selector_state, + ) as session: + tasks = [session.fetch(url) for url in urls] + responses = await gather(*tasks) + + return [_translate_response(page, extraction_type, css_selector, main_content_only) for page in responses] - @staticmethod async def stealthy_fetch( + self, url: str, extraction_type: extraction_types = "markdown", css_selector: Optional[str] = None, @@ -397,10 +628,13 @@ class ScraplingMCPServer: allow_webgl: bool = True, solve_cloudflare: bool = False, additional_args: Optional[Dict] = None, + session_id: Optional[str] = None, ) -> ResponseModel: """Use the stealthy fetcher to fetch a URL and return a structured output of the result. Note: This is the only suitable fetcher for high protection levels. Note: If the `css_selector` resolves to more than one element, all the elements will be returned. + Note: If a `session_id` is provided (from open_session), the browser session will be reused instead of creating a new one. + When using a session, browser-level params (headless, proxy, locale, etc.) are ignored — they were set at session creation time. :param url: The URL to request. :param extraction_type: The type of content to extract from the page. Defaults to "markdown". Options are: @@ -432,8 +666,9 @@ class ScraplingMCPServer: :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by `google_search` takes priority over the referer set here if used together._ :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only. :param additional_args: Additional arguments to be passed to Playwright's context as additional settings, and it takes higher priority than Scrapling's settings. + :param session_id: Optional session ID from open_session. If provided, reuses the existing browser session instead of creating a new one. """ - results = await ScraplingMCPServer.bulk_stealthy_fetch( + results = await self.bulk_stealthy_fetch( urls=[url], extraction_type=extraction_type, css_selector=css_selector, @@ -459,11 +694,12 @@ class ScraplingMCPServer: allow_webgl=allow_webgl, solve_cloudflare=solve_cloudflare, additional_args=additional_args, + session_id=session_id, ) return results[0] - @staticmethod async def bulk_stealthy_fetch( + self, urls: List[str], extraction_type: extraction_types = "markdown", css_selector: Optional[str] = None, @@ -489,10 +725,13 @@ class ScraplingMCPServer: allow_webgl: bool = True, solve_cloudflare: bool = False, additional_args: Optional[Dict] = None, + session_id: Optional[str] = None, ) -> List[ResponseModel]: """Use the stealthy fetcher to fetch a group of URLs at the same time, and for each page return a structured output of the result. Note: This is the only suitable fetcher for high protection levels. Note: If the `css_selector` resolves to more than one element, all the elements will be returned. + Note: If a `session_id` is provided (from open_session), the browser session will be reused instead of creating a new one. + When using a session, browser-level params (headless, proxy, locale, etc.) are ignored — they were set at session creation time. :param urls: A list of the URLs to request. :param extraction_type: The type of content to extract from the page. Defaults to "markdown". Options are: @@ -524,45 +763,74 @@ class ScraplingMCPServer: :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by `google_search` takes priority over the referer set here if used together._ :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only. :param additional_args: Additional arguments to be passed to Playwright's context as additional settings, and it takes higher priority than Scrapling's settings. + :param session_id: Optional session ID from open_session. If provided, reuses the existing browser session instead of creating a new one. """ - async with AsyncStealthySession( - wait=wait, - proxy=proxy, - locale=locale, - cdp_url=cdp_url, - timeout=timeout, - cookies=cookies, - headless=headless, - useragent=useragent, - timezone_id=timezone_id, - real_chrome=real_chrome, - hide_canvas=hide_canvas, - allow_webgl=allow_webgl, - network_idle=network_idle, - block_webrtc=block_webrtc, - wait_selector=wait_selector, - google_search=google_search, - extra_headers=extra_headers, - additional_args=additional_args, - solve_cloudflare=solve_cloudflare, - disable_resources=disable_resources, - wait_selector_state=wait_selector_state, - ) as session: - tasks = [session.fetch(url) for url in urls] + if session_id: + entry = self._get_session(session_id, "stealthy") + fetch_kwargs = _build_fetch_kwargs( + wait=wait, + timeout=timeout, + google_search=google_search, + extra_headers=extra_headers, + disable_resources=disable_resources, + wait_selector=wait_selector, + wait_selector_state=wait_selector_state, + network_idle=network_idle, + proxy=proxy, + solve_cloudflare=solve_cloudflare, + ) + tasks = [entry.session.fetch(url, **fetch_kwargs) for url in urls] responses = await gather(*tasks) - return [_translate_response(page, extraction_type, css_selector, main_content_only) for page in responses] + else: + async with AsyncStealthySession( + wait=wait, + proxy=proxy, + locale=locale, + cdp_url=cdp_url, + timeout=timeout, + cookies=cookies, + headless=headless, + useragent=useragent, + timezone_id=timezone_id, + real_chrome=real_chrome, + hide_canvas=hide_canvas, + allow_webgl=allow_webgl, + network_idle=network_idle, + block_webrtc=block_webrtc, + wait_selector=wait_selector, + google_search=google_search, + extra_headers=extra_headers, + additional_args=additional_args, + solve_cloudflare=solve_cloudflare, + disable_resources=disable_resources, + wait_selector_state=wait_selector_state, + ) as session: + tasks = [session.fetch(url) for url in urls] + responses = await gather(*tasks) + + return [_translate_response(page, extraction_type, css_selector, main_content_only) for page in responses] def serve(self, http: bool, host: str, port: int): """Serve the MCP server.""" server = FastMCP(name="Scrapling", host=host, port=port) + # Session management tools + server.add_tool(self.open_session, title="open_session", structured_output=True) + server.add_tool(self.close_session, title="close_session", structured_output=True) + server.add_tool(self.list_sessions, title="list_sessions", structured_output=True) + # HTTP tools server.add_tool(self.get, title="get", description=self.get.__doc__, structured_output=True) server.add_tool(self.bulk_get, title="bulk_get", description=self.bulk_get.__doc__, structured_output=True) + # Dynamic browser tools server.add_tool(self.fetch, title="fetch", description=self.fetch.__doc__, structured_output=True) server.add_tool( self.bulk_fetch, title="bulk_fetch", description=self.bulk_fetch.__doc__, structured_output=True ) + # Stealthy browser tools server.add_tool( - self.stealthy_fetch, title="stealthy_fetch", description=self.stealthy_fetch.__doc__, structured_output=True + self.stealthy_fetch, + title="stealthy_fetch", + description=self.stealthy_fetch.__doc__, + structured_output=True, ) server.add_tool( self.bulk_stealthy_fetch, diff --git a/tests/ai/test_ai_mcp.py b/tests/ai/test_ai_mcp.py index 305edf5..d897bb5 100644 --- a/tests/ai/test_ai_mcp.py +++ b/tests/ai/test_ai_mcp.py @@ -1,7 +1,14 @@ import pytest import pytest_httpbin -from scrapling.core.ai import ScraplingMCPServer, ResponseModel, _normalize_credentials +from scrapling.core.ai import ( + ScraplingMCPServer, + ResponseModel, + SessionInfo, + SessionCreatedModel, + SessionClosedModel, + _normalize_credentials, +) @pytest_httpbin.use_class_based_httpbin @@ -59,6 +66,118 @@ class TestMCPServer: assert all(isinstance(r, ResponseModel) for r in result) +@pytest_httpbin.use_class_based_httpbin +class TestSessionManagement: + """Test persistent browser session management""" + + @pytest.fixture(scope="class") + def test_url(self, httpbin): + return f"{httpbin.url}/html" + + @pytest.fixture + def server(self): + return ScraplingMCPServer() + + @pytest.mark.asyncio + async def test_open_and_close_session(self, server): + """Test opening and closing a dynamic session""" + result = await server.open_session(session_type="dynamic", headless=True) + assert isinstance(result, SessionCreatedModel) + assert result.session_type == "dynamic" + assert result.is_alive is True + session_id = result.session_id + + # Close the session + closed = await server.close_session(session_id) + assert isinstance(closed, SessionClosedModel) + assert closed.session_id == session_id + + @pytest.mark.asyncio + async def test_list_sessions(self, server): + """Test listing sessions""" + # Initially empty + sessions = await server.list_sessions() + assert sessions == [] + + # Open a session + result = await server.open_session(session_type="dynamic", headless=True) + session_id = result.session_id + + # List should show it + sessions = await server.list_sessions() + assert len(sessions) == 1 + assert isinstance(sessions[0], SessionInfo) + assert sessions[0].session_id == session_id + assert sessions[0].session_type == "dynamic" + assert sessions[0].is_alive is True + + # Cleanup + await server.close_session(session_id) + + @pytest.mark.asyncio + async def test_fetch_with_session(self, server, test_url): + """Test fetching with a persistent dynamic session""" + result = await server.open_session(session_type="dynamic", headless=True) + session_id = result.session_id + + # Fetch using the session + response = await server.fetch(url=test_url, session_id=session_id) + assert isinstance(response, ResponseModel) + assert response.status == 200 + + # Fetch again with the same session (reuse) + response2 = await server.fetch(url=test_url, session_id=session_id) + assert isinstance(response2, ResponseModel) + assert response2.status == 200 + + await server.close_session(session_id) + + @pytest.mark.asyncio + async def test_bulk_fetch_with_session(self, server, test_url): + """Test bulk fetching with a persistent dynamic session""" + result = await server.open_session(session_type="dynamic", headless=True, max_pages=5) + session_id = result.session_id + + responses = await server.bulk_fetch(urls=[test_url, test_url], session_id=session_id) + assert len(responses) == 2 + assert all(isinstance(r, ResponseModel) for r in responses) + + await server.close_session(session_id) + + @pytest.mark.asyncio + async def test_session_type_mismatch(self, server, test_url): + """Test that using a dynamic session with stealthy_fetch raises an error""" + result = await server.open_session(session_type="dynamic", headless=True) + session_id = result.session_id + + with pytest.raises(ValueError, match="'dynamic' session"): + await server.stealthy_fetch(url=test_url, session_id=session_id) + + await server.close_session(session_id) + + @pytest.mark.asyncio + async def test_close_nonexistent_session(self, server): + """Test closing a session that doesn't exist""" + with pytest.raises(ValueError, match="not found"): + await server.close_session("nonexistent") + + @pytest.mark.asyncio + async def test_fetch_with_nonexistent_session(self, server, test_url): + """Test fetching with a session ID that doesn't exist""" + with pytest.raises(ValueError, match="not found"): + await server.fetch(url=test_url, session_id="nonexistent") + + @pytest.mark.asyncio + async def test_fetch_with_closed_session(self, server, test_url): + """Test fetching with a session that has been closed""" + result = await server.open_session(session_type="dynamic", headless=True) + session_id = result.session_id + await server.close_session(session_id) + + with pytest.raises(ValueError, match="not found"): + await server.fetch(url=test_url, session_id=session_id) + + class TestNormalizeCredentials: """Test the _normalize_credentials helper""" From 0f6dcccf5f819deace2cac0879bae1ee84fc8399 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Sat, 28 Mar 2026 17:19:42 +0200 Subject: [PATCH 18/33] fix(mcp): remove unneeded code and fix type hint for mypy --- scrapling/core/ai.py | 80 ++++++++++++++++++-------------------------- 1 file changed, 33 insertions(+), 47 deletions(-) diff --git a/scrapling/core/ai.py b/scrapling/core/ai.py index 85ceefd..2fabf43 100644 --- a/scrapling/core/ai.py +++ b/scrapling/core/ai.py @@ -17,6 +17,7 @@ from scrapling.fetchers import ( from scrapling.core._types import ( Optional, Literal, + Union, Tuple, Mapping, Dict, @@ -100,28 +101,6 @@ def _normalize_credentials(credentials: Optional[Dict[str, str]]) -> Optional[Tu return username, password -def _build_fetch_kwargs(**params) -> Dict[str, Any]: - """Build kwargs dict for session.fetch() from request-level parameters.""" - kwargs: Dict[str, Any] = {} - # These are the params that session.fetch() accepts as per-request overrides - request_level_keys = ( - "wait", - "timeout", - "google_search", - "extra_headers", - "disable_resources", - "wait_selector", - "wait_selector_state", - "network_idle", - "proxy", - "solve_cloudflare", - ) - for key in request_level_keys: - if key in params and params[key] is not None: - kwargs[key] = params[key] - return kwargs - - class ScraplingMCPServer: def __init__(self): self._sessions: Dict[str, _SessionEntry] = {} @@ -215,6 +194,7 @@ class ScraplingMCPServer: wait_selector_state=wait_selector_state, ) + session: Union[AsyncDynamicSession, AsyncStealthySession] if session_type == "stealthy": session = AsyncStealthySession( **common_kwargs, @@ -563,18 +543,21 @@ class ScraplingMCPServer: """ if session_id: entry = self._get_session(session_id, "dynamic") - fetch_kwargs = _build_fetch_kwargs( - wait=wait, - timeout=timeout, - google_search=google_search, - extra_headers=extra_headers, - disable_resources=disable_resources, - wait_selector=wait_selector, - wait_selector_state=wait_selector_state, - network_idle=network_idle, - proxy=proxy, - ) - tasks = [entry.session.fetch(url, **fetch_kwargs) for url in urls] + tasks = [ + entry.session.fetch( + url, + wait=wait, + timeout=timeout, + google_search=google_search, + extra_headers=extra_headers, + disable_resources=disable_resources, + wait_selector=wait_selector, + wait_selector_state=wait_selector_state, + network_idle=network_idle, + proxy=proxy, + ) + for url in urls + ] responses = await gather(*tasks) else: async with AsyncDynamicSession( @@ -767,19 +750,22 @@ class ScraplingMCPServer: """ if session_id: entry = self._get_session(session_id, "stealthy") - fetch_kwargs = _build_fetch_kwargs( - wait=wait, - timeout=timeout, - google_search=google_search, - extra_headers=extra_headers, - disable_resources=disable_resources, - wait_selector=wait_selector, - wait_selector_state=wait_selector_state, - network_idle=network_idle, - proxy=proxy, - solve_cloudflare=solve_cloudflare, - ) - tasks = [entry.session.fetch(url, **fetch_kwargs) for url in urls] + tasks = [ + entry.session.fetch( + url, + wait=wait, + timeout=timeout, + google_search=google_search, + extra_headers=extra_headers, + disable_resources=disable_resources, + wait_selector=wait_selector, + wait_selector_state=wait_selector_state, + network_idle=network_idle, + proxy=proxy, + solve_cloudflare=solve_cloudflare, + ) + for url in urls + ] responses = await gather(*tasks) else: async with AsyncStealthySession( From 68f7c5c36f043855db16c48a7679e7a9ed33e6ef Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Sun, 29 Mar 2026 22:53:54 +0200 Subject: [PATCH 19/33] feat(browser sessions): Collect XHR requests done while loading the page Solves #159 --- scrapling/engines/_browsers/_base.py | 37 +++++++++++++++-- scrapling/engines/_browsers/_controllers.py | 38 ++++++++++++++--- scrapling/engines/_browsers/_stealth.py | 46 +++++++++++++++------ scrapling/engines/_browsers/_types.py | 1 + scrapling/engines/_browsers/_validators.py | 3 ++ scrapling/engines/toolbelt/convertor.py | 45 +++++++++++++------- scrapling/engines/toolbelt/custom.py | 1 + 7 files changed, 135 insertions(+), 36 deletions(-) diff --git a/scrapling/engines/_browsers/_base.py b/scrapling/engines/_browsers/_base.py index 501db76..15371f7 100644 --- a/scrapling/engines/_browsers/_base.py +++ b/scrapling/engines/_browsers/_base.py @@ -1,4 +1,5 @@ from time import time +from re import search as re_search from asyncio import sleep as asyncio_sleep, Lock from contextlib import contextmanager, asynccontextmanager @@ -146,11 +147,18 @@ class SyncSession: self._wait_for_networkidle(page) @staticmethod - def _create_response_handler(page_info: PageInfo[Page], response_container: List) -> Callable: - """Create a response handler that captures the final navigation response. + def _create_response_handler( + page_info: PageInfo[Page], + response_container: List, + xhr_pattern: Optional[str] = None, + xhr_container: Optional[List] = None, + ) -> Callable: + """Create a response handler that captures the final navigation response and optionally XHR/fetch responses. :param page_info: The PageInfo object containing the page :param response_container: A list to store the final response (mutable container) + :param xhr_pattern: Optional regex pattern to match XHR/fetch response URLs + :param xhr_container: Optional list to store captured XHR/fetch responses :return: A callback function for page.on("response", ...) """ @@ -161,6 +169,13 @@ class SyncSession: and finished_response.request.frame == page_info.page.main_frame ): response_container[0] = finished_response + elif ( + xhr_pattern + and xhr_container is not None + and finished_response.request.resource_type in ("xhr", "fetch") + and re_search(xhr_pattern, finished_response.url) + ): + xhr_container.append(finished_response) return handle_response @@ -317,11 +332,18 @@ class AsyncSession: await self._wait_for_networkidle(page) @staticmethod - def _create_response_handler(page_info: PageInfo[AsyncPage], response_container: List) -> Callable: - """Create an async response handler that captures the final navigation response. + def _create_response_handler( + page_info: PageInfo[AsyncPage], + response_container: List, + xhr_pattern: Optional[str] = None, + xhr_container: Optional[List] = None, + ) -> Callable: + """Create an async response handler that captures the final navigation response and optionally XHR/fetch responses. :param page_info: The PageInfo object containing the page :param response_container: A list to store the final response (mutable container) + :param xhr_pattern: Optional regex pattern to match XHR/fetch response URLs + :param xhr_container: Optional list to store captured XHR/fetch responses :return: A callback function for page.on("response", ...) """ @@ -332,6 +354,13 @@ class AsyncSession: and finished_response.request.frame == page_info.page.main_frame ): response_container[0] = finished_response + elif ( + xhr_pattern + and xhr_container is not None + and finished_response.request.resource_type in ("xhr", "fetch") + and re_search(xhr_pattern, finished_response.url) + ): + xhr_container.append(finished_response) return handle_response diff --git a/scrapling/engines/_browsers/_controllers.py b/scrapling/engines/_browsers/_controllers.py index fa6e8b6..f788819 100644 --- a/scrapling/engines/_browsers/_controllers.py +++ b/scrapling/engines/_browsers/_controllers.py @@ -139,9 +139,17 @@ class DynamicSession(SyncSession, DynamicSessionMixin): with self._page_generator( params.timeout, params.extra_headers, params.disable_resources, proxy, params.blocked_domains ) as page_info: - final_response = [None] + final_response, xhr_captured = [None], [] page = page_info.page - page.on("response", self._create_response_handler(page_info, final_response)) + page.on( + "response", + self._create_response_handler( + page_info, + final_response, + xhr_pattern=self._config.capture_xhr, + xhr_container=xhr_captured, + ), + ) try: first_response = page.goto(url, referer=referer) @@ -167,7 +175,12 @@ class DynamicSession(SyncSession, DynamicSessionMixin): page.wait_for_timeout(params.wait) response = ResponseFactory.from_playwright_response( - page, first_response, final_response[0], params.selector_config, meta={"proxy": proxy} + page, + first_response, + final_response[0], + params.selector_config, + meta={"proxy": proxy}, + xhr_captured=xhr_captured, ) return response @@ -306,9 +319,17 @@ class AsyncDynamicSession(AsyncSession, DynamicSessionMixin): async with self._page_generator( params.timeout, params.extra_headers, params.disable_resources, proxy, params.blocked_domains ) as page_info: - final_response = [None] + final_response, xhr_captured = [None], [] page = page_info.page - page.on("response", self._create_response_handler(page_info, final_response)) + page.on( + "response", + self._create_response_handler( + page_info, + final_response, + xhr_pattern=self._config.capture_xhr, + xhr_container=xhr_captured, + ), + ) try: first_response = await page.goto(url, referer=referer) @@ -334,7 +355,12 @@ class AsyncDynamicSession(AsyncSession, DynamicSessionMixin): await page.wait_for_timeout(params.wait) response = await ResponseFactory.from_async_playwright_response( - page, first_response, final_response[0], params.selector_config, meta={"proxy": proxy} + page, + first_response, + final_response[0], + params.selector_config, + meta={"proxy": proxy}, + xhr_captured=xhr_captured, ) return response diff --git a/scrapling/engines/_browsers/_stealth.py b/scrapling/engines/_browsers/_stealth.py index 8fb248f..ef3a445 100644 --- a/scrapling/engines/_browsers/_stealth.py +++ b/scrapling/engines/_browsers/_stealth.py @@ -3,12 +3,8 @@ from re import compile as re_compile from time import sleep as time_sleep from asyncio import sleep as asyncio_sleep -from playwright.sync_api import Locator, Page, BrowserContext -from playwright.async_api import ( - Page as async_Page, - Locator as AsyncLocator, - BrowserContext as AsyncBrowserContext, -) +from playwright.sync_api import Locator, Page +from playwright.async_api import Page as async_Page, Locator as AsyncLocator from patchright.sync_api import sync_playwright from patchright.async_api import async_playwright @@ -226,9 +222,17 @@ class StealthySession(SyncSession, StealthySessionMixin): with self._page_generator( params.timeout, params.extra_headers, params.disable_resources, proxy, params.blocked_domains ) as page_info: - final_response = [None] + final_response, xhr_captured = [None], [] page = page_info.page - page.on("response", self._create_response_handler(page_info, final_response)) + page.on( + "response", + self._create_response_handler( + page_info, + final_response, + xhr_pattern=self._config.capture_xhr, + xhr_container=xhr_captured, + ), + ) try: first_response = page.goto(url, referer=referer) @@ -259,7 +263,12 @@ class StealthySession(SyncSession, StealthySessionMixin): page.wait_for_timeout(params.wait) response = ResponseFactory.from_playwright_response( - page, first_response, final_response[0], params.selector_config, meta={"proxy": proxy} + page, + first_response, + final_response[0], + params.selector_config, + meta={"proxy": proxy}, + xhr_captured=xhr_captured, ) return response @@ -480,9 +489,17 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin): async with self._page_generator( params.timeout, params.extra_headers, params.disable_resources, proxy, params.blocked_domains ) as page_info: - final_response = [None] + final_response, xhr_captured = [None], [] page = page_info.page - page.on("response", self._create_response_handler(page_info, final_response)) + page.on( + "response", + self._create_response_handler( + page_info, + final_response, + xhr_pattern=self._config.capture_xhr, + xhr_container=xhr_captured, + ), + ) try: first_response = await page.goto(url, referer=referer) @@ -513,7 +530,12 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin): await page.wait_for_timeout(params.wait) response = await ResponseFactory.from_async_playwright_response( - page, first_response, final_response[0], params.selector_config, meta={"proxy": proxy} + page, + first_response, + final_response[0], + params.selector_config, + meta={"proxy": proxy}, + xhr_captured=xhr_captured, ) return response diff --git a/scrapling/engines/_browsers/_types.py b/scrapling/engines/_browsers/_types.py index 30d6af1..f4f31cd 100644 --- a/scrapling/engines/_browsers/_types.py +++ b/scrapling/engines/_browsers/_types.py @@ -89,6 +89,7 @@ class PlaywrightSession(TypedDict, total=False): blocked_domains: Optional[Set[str]] retries: int retry_delay: int | float + capture_xhr: str | None class PlaywrightFetchParams(TypedDict, total=False): diff --git a/scrapling/engines/_browsers/_validators.py b/scrapling/engines/_browsers/_validators.py index e2424f4..dfe7310 100644 --- a/scrapling/engines/_browsers/_validators.py +++ b/scrapling/engines/_browsers/_validators.py @@ -87,6 +87,7 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False, weakref=True): blocked_domains: Optional[Set[str]] = None retries: RetriesCount = 3 retry_delay: Seconds = 1 + capture_xhr: str | None = None def __post_init__(self): # pragma: no cover """Custom validation after msgspec validation""" @@ -112,6 +113,8 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False, weakref=True): self.selector_config = {} if not self.additional_args: self.additional_args = {} + if not self.capture_xhr: + self.capture_xhr = None if self.init_script is not None: validation_msg = _is_invalid_file_path(self.init_script) diff --git a/scrapling/engines/toolbelt/convertor.py b/scrapling/engines/toolbelt/convertor.py index 8606003..8d624e0 100644 --- a/scrapling/engines/toolbelt/convertor.py +++ b/scrapling/engines/toolbelt/convertor.py @@ -8,7 +8,7 @@ from playwright.async_api import Page as AsyncPage, Response as AsyncResponse from scrapling.core.utils import log from .custom import Response, StatusText -from scrapling.core._types import Dict, Optional +from scrapling.core._types import Dict, List, Optional __CHARSET_RE__ = re_compile(r"charset=([\w-]+)") @@ -81,11 +81,13 @@ class ResponseFactory: @classmethod def from_playwright_response( cls, - page: SyncPage, + page: Optional[SyncPage], first_response: SyncResponse, final_response: Optional[SyncResponse], parser_arguments: Dict, meta: Optional[Dict] = None, + xhr_captured: Optional[List[SyncResponse]] = None, + collect_history: bool = True, ) -> Response: """ Transforms a Playwright response into an internal `Response` object, encapsulating @@ -102,7 +104,8 @@ class ResponseFactory: :param parser_arguments: A dictionary containing additional arguments needed for parsing or further customization of the returned `Response`. These arguments are dynamically unpacked into the `Response` object. :param meta: Additional meta data to be saved with the response. - + :param xhr_captured: Optional list of captured Playwright XHR/fetch responses to convert and attach to the returned Response. + :param collect_history: Optional boolean indicating whether to collect redirections history or not. :return: A fully populated `Response` object containing the page's URL, content, status, headers, cookies, and other derived metadata. :rtype: Response """ @@ -115,9 +118,9 @@ class ResponseFactory: # PlayWright API sometimes give empty status text for some reason! status_text = final_response.status_text or StatusText.get(final_response.status) - history = cls._process_response_history(first_response, parser_arguments) + history = cls._process_response_history(first_response, parser_arguments) if collect_history else [] try: - if "html" in final_response.all_headers().get("content-type", ""): + if page and "html" in final_response.all_headers().get("content-type", ""): page_content = cls._get_page_content(page).encode("utf-8") else: page_content = final_response.body() @@ -125,14 +128,14 @@ class ResponseFactory: log.error(f"Error getting page content: {e}") page_content = b"" - return Response( + response = Response( **{ - "url": page.url, + "url": page.url if page else first_response.url, "content": page_content, "status": final_response.status, "reason": status_text, "encoding": encoding, - "cookies": tuple(dict(cookie) for cookie in page.context.cookies()), + "cookies": tuple(dict(cookie) for cookie in page.context.cookies()) if page else {}, "headers": first_response.all_headers(), "request_headers": first_response.request.all_headers(), "history": history, @@ -140,6 +143,11 @@ class ResponseFactory: **parser_arguments, } ) + if xhr_captured: + response.captured_xhr = [ + cls.from_playwright_response(None, p, None, {}, collect_history=False) for p in xhr_captured + ] + return response @classmethod async def _async_process_response_history( @@ -219,11 +227,13 @@ class ResponseFactory: @classmethod async def from_async_playwright_response( cls, - page: AsyncPage, + page: Optional[AsyncPage], first_response: AsyncResponse, final_response: Optional[AsyncResponse], parser_arguments: Dict, meta: Optional[Dict] = None, + xhr_captured: Optional[List[AsyncResponse]] = None, + collect_history: bool = True, ) -> Response: """ Transforms a Playwright response into an internal `Response` object, encapsulating @@ -240,6 +250,8 @@ class ResponseFactory: :param parser_arguments: A dictionary containing additional arguments needed for parsing or further customization of the returned `Response`. These arguments are dynamically unpacked into the `Response` object. :param meta: Additional meta data to be saved with the response. + :param xhr_captured: Optional list of captured async Playwright XHR/fetch responses to convert and attach to the returned Response. + :param collect_history: Optional boolean indicating whether to collect redirections history or not. :return: A fully populated `Response` object containing the page's URL, content, status, headers, cookies, and other derived metadata. :rtype: Response @@ -253,9 +265,9 @@ class ResponseFactory: # PlayWright API sometimes give empty status text for some reason! status_text = final_response.status_text or StatusText.get(final_response.status) - history = await cls._async_process_response_history(first_response, parser_arguments) + history = await cls._async_process_response_history(first_response, parser_arguments) if collect_history else [] try: - if "html" in (await final_response.all_headers()).get("content-type", ""): + if page and "html" in (await final_response.all_headers()).get("content-type", ""): page_content = (await cls._get_async_page_content(page)).encode("utf-8") else: page_content = await final_response.body() @@ -263,14 +275,14 @@ class ResponseFactory: log.error(f"Error getting page content in async: {e}") page_content = b"" - return Response( + response = Response( **{ - "url": page.url, + "url": page.url if page else first_response.url, "content": page_content, "status": final_response.status, "reason": status_text, "encoding": encoding, - "cookies": tuple(dict(cookie) for cookie in await page.context.cookies()), + "cookies": tuple(dict(cookie) for cookie in await page.context.cookies()) if page else {}, "headers": await first_response.all_headers(), "request_headers": await first_response.request.all_headers(), "history": history, @@ -278,6 +290,11 @@ class ResponseFactory: **parser_arguments, } ) + if xhr_captured: + response.captured_xhr = [ + await cls.from_async_playwright_response(None, p, None, {}, collect_history=False) for p in xhr_captured + ] + return response @staticmethod def from_http_request(response: CurlResponse, parser_arguments: Dict, meta: Optional[Dict] = None) -> Response: diff --git a/scrapling/engines/toolbelt/custom.py b/scrapling/engines/toolbelt/custom.py index 58a7485..483a32b 100644 --- a/scrapling/engines/toolbelt/custom.py +++ b/scrapling/engines/toolbelt/custom.py @@ -67,6 +67,7 @@ class Response(Selector): self.meta: Dict[str, Any] = meta or {} self.request: Optional["Request"] = None # Will be set by crawler + self.captured_xhr: List["Response"] = [] @property def body(self) -> bytes: From 5c450a3b52cc559c94e248506cb91c50659aad3c Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Sun, 29 Mar 2026 23:01:02 +0200 Subject: [PATCH 20/33] fix: improve type hints for the static checkers --- scrapling/engines/_browsers/_base.py | 9 +++++---- scrapling/engines/_browsers/_controllers.py | 8 +++++--- scrapling/engines/_browsers/_stealth.py | 8 +++++--- 3 files changed, 15 insertions(+), 10 deletions(-) diff --git a/scrapling/engines/_browsers/_base.py b/scrapling/engines/_browsers/_base.py index 15371f7..fa95419 100644 --- a/scrapling/engines/_browsers/_base.py +++ b/scrapling/engines/_browsers/_base.py @@ -28,6 +28,7 @@ from scrapling.engines.toolbelt.navigation import ( ) from scrapling.core._types import ( Any, + Awaitable, Dict, List, Set, @@ -152,7 +153,7 @@ class SyncSession: response_container: List, xhr_pattern: Optional[str] = None, xhr_container: Optional[List] = None, - ) -> Callable: + ) -> Callable[[SyncPlaywrightResponse], None]: """Create a response handler that captures the final navigation response and optionally XHR/fetch responses. :param page_info: The PageInfo object containing the page @@ -162,7 +163,7 @@ class SyncSession: :return: A callback function for page.on("response", ...) """ - def handle_response(finished_response: SyncPlaywrightResponse): + def handle_response(finished_response: SyncPlaywrightResponse) -> None: if ( finished_response.request.resource_type == "document" and finished_response.request.is_navigation_request() @@ -337,7 +338,7 @@ class AsyncSession: response_container: List, xhr_pattern: Optional[str] = None, xhr_container: Optional[List] = None, - ) -> Callable: + ) -> Callable[[AsyncPlaywrightResponse], Awaitable[None]]: """Create an async response handler that captures the final navigation response and optionally XHR/fetch responses. :param page_info: The PageInfo object containing the page @@ -347,7 +348,7 @@ class AsyncSession: :return: A callback function for page.on("response", ...) """ - async def handle_response(finished_response: AsyncPlaywrightResponse): + async def handle_response(finished_response: AsyncPlaywrightResponse) -> None: if ( finished_response.request.resource_type == "document" and finished_response.request.is_navigation_request() diff --git a/scrapling/engines/_browsers/_controllers.py b/scrapling/engines/_browsers/_controllers.py index f788819..ce4b643 100644 --- a/scrapling/engines/_browsers/_controllers.py +++ b/scrapling/engines/_browsers/_controllers.py @@ -11,7 +11,7 @@ from playwright.async_api import ( ) from scrapling.core.utils import log -from scrapling.core._types import Optional, ProxyType, Unpack +from scrapling.core._types import Optional, List, ProxyType, Unpack from scrapling.engines.toolbelt.proxy_rotation import is_proxy_error from scrapling.engines.toolbelt.convertor import Response, ResponseFactory from scrapling.engines._browsers._types import PlaywrightSession, PlaywrightFetchParams @@ -139,7 +139,8 @@ class DynamicSession(SyncSession, DynamicSessionMixin): with self._page_generator( params.timeout, params.extra_headers, params.disable_resources, proxy, params.blocked_domains ) as page_info: - final_response, xhr_captured = [None], [] + final_response: List = [None] + xhr_captured: List = [] page = page_info.page page.on( "response", @@ -319,7 +320,8 @@ class AsyncDynamicSession(AsyncSession, DynamicSessionMixin): async with self._page_generator( params.timeout, params.extra_headers, params.disable_resources, proxy, params.blocked_domains ) as page_info: - final_response, xhr_captured = [None], [] + final_response: List = [None] + xhr_captured: List = [] page = page_info.page page.on( "response", diff --git a/scrapling/engines/_browsers/_stealth.py b/scrapling/engines/_browsers/_stealth.py index ef3a445..f06c62e 100644 --- a/scrapling/engines/_browsers/_stealth.py +++ b/scrapling/engines/_browsers/_stealth.py @@ -9,7 +9,7 @@ from patchright.sync_api import sync_playwright from patchright.async_api import async_playwright from scrapling.core.utils import log -from scrapling.core._types import Any, Optional, ProxyType, Unpack +from scrapling.core._types import Any, List, Optional, ProxyType, Unpack from scrapling.engines.toolbelt.proxy_rotation import is_proxy_error from scrapling.engines.toolbelt.convertor import Response, ResponseFactory from scrapling.engines._browsers._types import StealthSession, StealthFetchParams @@ -222,7 +222,8 @@ class StealthySession(SyncSession, StealthySessionMixin): with self._page_generator( params.timeout, params.extra_headers, params.disable_resources, proxy, params.blocked_domains ) as page_info: - final_response, xhr_captured = [None], [] + final_response: List = [None] + xhr_captured: List = [] page = page_info.page page.on( "response", @@ -489,7 +490,8 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin): async with self._page_generator( params.timeout, params.extra_headers, params.disable_resources, proxy, params.blocked_domains ) as page_info: - final_response, xhr_captured = [None], [] + final_response: List = [None] + xhr_captured: List = [] page = page_info.page page.on( "response", From 61cda587be068cfbf14c44f9ae1c14bb61cac94a Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Sun, 29 Mar 2026 23:12:56 +0200 Subject: [PATCH 21/33] docs: update pages with the XHR feature --- docs/fetching/choosing.md | 1 + docs/fetching/dynamic.md | 19 +++++++++++++++++++ docs/fetching/stealthy.md | 3 ++- docs/spiders/sessions.md | 6 ++++++ scrapling/engines/toolbelt/custom.py | 13 ++++++++++++- 5 files changed, 40 insertions(+), 2 deletions(-) diff --git a/docs/fetching/choosing.md b/docs/fetching/choosing.md index dcba9b9..b9f3ee4 100644 --- a/docs/fetching/choosing.md +++ b/docs/fetching/choosing.md @@ -77,6 +77,7 @@ The `Response` object is the same as the [Selector](../parsing/main_classes.md#s >>> page.body # Raw response body as bytes >>> page.encoding # Response encoding >>> page.meta # Response metadata dictionary (e.g., proxy used). Mainly helpful with the spiders system. +>>> page.captured_xhr # List of captured XHR/fetch responses (when capture_xhr is enabled on a browser session) ``` All fetchers return the `Response` object. diff --git a/docs/fetching/dynamic.md b/docs/fetching/dynamic.md index 1d33a59..0c444ff 100644 --- a/docs/fetching/dynamic.md +++ b/docs/fetching/dynamic.md @@ -91,6 +91,7 @@ Scrapling provides many options with this fetcher and its session classes. To ma | proxy_rotator | A `ProxyRotator` instance for automatic proxy rotation. Cannot be combined with `proxy`. | ✔️ | | retries | Number of retry attempts for failed requests. Defaults to 3. | ✔️ | | retry_delay | Seconds to wait between retry attempts. Defaults to 1. | ✔️ | +| capture_xhr | Pass a regex URL pattern string to capture XHR/fetch requests matching it during page load. Captured responses are available via `response.captured_xhr`. Defaults to `None` (disabled). | ✔️ | In session classes, all these arguments can be set globally for the session. Still, you can configure each request individually by passing some of the arguments here that can be configured on the browser tab level like: `google_search`, `timeout`, `wait`, `page_action`, `extra_headers`, `disable_resources`, `wait_selector`, `wait_selector_state`, `network_idle`, `load_dom`, `blocked_domains`, `proxy`, and `selector_config`. @@ -217,6 +218,24 @@ The states the fetcher can wait for can be any of the following ([source](https: - `visible`: wait for an element to have a non-empty bounding box and no `visibility:hidden`. Note that an element without any content or with `display:none` has an empty bounding box and is not considered visible. - `hidden`: wait for an element to be either detached from the DOM, or have an empty bounding box, or `visibility:hidden`. This is opposite to the `'visible'` option. +### Capturing XHR/Fetch Requests + +Many SPAs load data through background API calls (XHR/fetch). You can capture these requests by passing a regex URL pattern to `capture_xhr` at the session level: + +```python +from scrapling.fetchers import DynamicSession + +with DynamicSession(capture_xhr=r"https://api\.example\.com/.*", headless=True) as session: + page = session.fetch('https://example.com') + + # Access captured XHR responses + for xhr in page.captured_xhr: + print(xhr.url, xhr.status) + print(xhr.body) # Raw response body as bytes +``` + +Each item in `captured_xhr` is a full `Response` object with the same properties (`.url`, `.status`, `.headers`, `.body`, etc.). When `capture_xhr` is not set or is `None`, `captured_xhr` is an empty list. + ### Some Stealth Features ```python diff --git a/docs/fetching/stealthy.md b/docs/fetching/stealthy.md index 4fd888e..4fe3477 100644 --- a/docs/fetching/stealthy.md +++ b/docs/fetching/stealthy.md @@ -72,12 +72,13 @@ Scrapling provides many options with this fetcher and its session classes. Befor | proxy_rotator | A `ProxyRotator` instance for automatic proxy rotation. Cannot be combined with `proxy`. | ✔️ | | retries | Number of retry attempts for failed requests. Defaults to 3. | ✔️ | | retry_delay | Seconds to wait between retry attempts. Defaults to 1. | ✔️ | +| capture_xhr | Pass a regex URL pattern string to capture XHR/fetch requests matching it during page load. Captured responses are available via `response.captured_xhr`. Defaults to `None` (disabled). | ✔️ | In session classes, all these arguments can be set globally for the session. Still, you can configure each request individually by passing some of the arguments here that can be configured on the browser tab level like: `google_search`, `timeout`, `wait`, `page_action`, `extra_headers`, `disable_resources`, `wait_selector`, `wait_selector_state`, `network_idle`, `load_dom`, `solve_cloudflare`, `blocked_domains`, `proxy`, and `selector_config`. !!! note "Notes:" - 1. It's basically the same arguments as [DynamicFetcher](dynamic.md#introduction) class, but with these additional arguments: `solve_cloudflare`, `block_webrtc`, `hide_canvas`, and `allow_webgl`. + 1. It's basically the same arguments as [DynamicFetcher](dynamic.md#introduction) class, but with these additional arguments: `solve_cloudflare`, `block_webrtc`, `hide_canvas`, and `allow_webgl`. The `capture_xhr` argument is shared with `DynamicFetcher`. 2. The `disable_resources` option made requests ~25% faster in my tests for some websites and can help save your proxy usage, but be careful with it, as it can cause some websites to never finish loading. 3. The `google_search` argument is enabled by default for all requests, setting the referer to `https://www.google.com/`. If used together with `extra_headers`, it takes priority over the referer set there. 4. If you didn't set a user agent and enabled headless mode, the fetcher will generate a real user agent for the same browser version and use it. If you didn't set a user agent and didn't enable headless mode, the fetcher will use the browser's default user agent, which is the same as in standard browsers in the latest versions. diff --git a/docs/spiders/sessions.md b/docs/spiders/sessions.md index d922ee1..1d1d759 100644 --- a/docs/spiders/sessions.md +++ b/docs/spiders/sessions.md @@ -74,9 +74,11 @@ class ProductSpider(Spider): manager.add("http", FetcherSession()) # Stealth browser for protected product pages + # capture_xhr captures background API calls matching the regex manager.add("stealth", AsyncStealthySession( headless=True, network_idle=True, + capture_xhr=r"https://api\.shop\.example\.com/.*", )) async def parse(self, response: Response): @@ -89,6 +91,10 @@ class ProductSpider(Spider): yield response.follow(next_page) async def parse_product(self, response: Response): + # Access captured XHR/fetch API calls (if capture_xhr was set on the session) + for xhr in response.captured_xhr: + self.logger.info(f"Captured API call: {xhr.url} ({xhr.status})") + yield { "name": response.css("h1::text").get(""), "price": response.css(".price::text").get(""), diff --git a/scrapling/engines/toolbelt/custom.py b/scrapling/engines/toolbelt/custom.py index 483a32b..48e3e43 100644 --- a/scrapling/engines/toolbelt/custom.py +++ b/scrapling/engines/toolbelt/custom.py @@ -26,7 +26,18 @@ if TYPE_CHECKING: class Response(Selector): - """This class is returned by all engines as a way to unify the response type between different libraries.""" + """This class is returned by all engines as a way to unify the response type between different libraries. + + :param status: HTTP status code. + :param reason: HTTP status message. + :param cookies: Response cookies. + :param headers: Response headers. + :param request_headers: Request headers sent with the request. + :param history: List of redirect responses, if any. + :param meta: Metadata dictionary (e.g., proxy used). + :param request: Associated spider Request object (set by crawler, in the spiders framework). + :param captured_xhr: List of captured XHR/fetch ``Response`` objects. Populated when ``capture_xhr`` is set on a browser session. + """ def __init__( self, From 8e89a730149e511f1e424c058abf45e2114dd52e Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Sun, 29 Mar 2026 23:14:52 +0200 Subject: [PATCH 22/33] build(docs): Pump up Zensical version to the latest --- docs/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index 7e8e342..f218ea6 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,4 +1,4 @@ -zensical>=0.0.27 +zensical>=0.0.30 mkdocstrings>=1.0.3 mkdocstrings-python>=2.0.3 griffe-inherited-docstrings>=1.1.3 From 7f552bed73f0aca26692372d50452444782619fb Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Sun, 29 Mar 2026 23:38:54 +0200 Subject: [PATCH 23/33] docs: Add docs for the new MCP tools --- docs/ai/mcp-server.md | 47 ++++++++++++++++++++++++++++++-- docs/api-reference/mcp-server.md | 18 +++++++++++- 2 files changed, 61 insertions(+), 4 deletions(-) diff --git a/docs/ai/mcp-server.md b/docs/ai/mcp-server.md index bf59d66..c87b6cf 100644 --- a/docs/ai/mcp-server.md +++ b/docs/ai/mcp-server.md @@ -6,20 +6,25 @@ The **Scrapling MCP Server** is a new feature that brings Scrapling's powerful W ## Features -The Scrapling MCP Server provides six powerful tools for web scraping: +The Scrapling MCP Server provides nine powerful tools for web scraping: ### 🚀 Basic HTTP Scraping - **`get`**: Fast HTTP requests with browser fingerprint impersonation, generating real browser headers matching the TLS version, HTTP/3, and more! - **`bulk_get`**: An async version of the above tool that allows scraping of multiple URLs at the same time! -### 🌐 Dynamic Content Scraping +### 🌐 Dynamic Content Scraping - **`fetch`**: Rapidly fetch dynamic content with Chromium/Chrome browser with complete control over the request/browser, and more! - **`bulk_fetch`**: An async version of the above tool that allows scraping of multiple URLs in different browser tabs at the same time! ### 🔒 Stealth Scraping -- **`stealthy_fetch`**: Uses our Stealthy browser to bypass Cloudflare Turnstile/Interstitial and other anti-bot systems with complete control over the request/browser! +- **`stealthy_fetch`**: Uses our Stealthy browser to bypass Cloudflare Turnstile/Interstitial and other anti-bot systems with complete control over the request/browser! - **`bulk_stealthy_fetch`**: An async version of the above tool that allows stealth scraping of multiple URLs in different browser tabs at the same time! +### 🔌 Session Management +- **`open_session`**: Create a persistent browser session (dynamic or stealthy) that stays open across multiple fetch calls, avoiding the overhead of launching a new browser each time. +- **`close_session`**: Close a persistent browser session and free its resources. +- **`list_sessions`**: List all active browser sessions with their details. + ### Key Capabilities - **Smart Content Extraction**: Convert web pages/elements to Markdown, HTML, or extract a clean version of the text content - **CSS Selector Support**: Use the Scrapling engine to target specific elements with precision before handing the content to the AI @@ -27,6 +32,7 @@ The Scrapling MCP Server provides six powerful tools for web scraping: - **Proxy Support**: Use proxies for anonymity and geo-targeting - **Browser Impersonation**: Mimic real browsers with TLS fingerprinting, real browser headers matching that version, and more - **Parallel Processing**: Scrape multiple URLs concurrently for efficiency +- **Session Persistence**: Reuse browser sessions across multiple requests for better performance #### But why use Scrapling MCP Server instead of other available tools? @@ -252,6 +258,34 @@ We will gradually go from simple prompts to more complex ones. We will use Claud https://www.arnotts.ie/furniture/bedroom/bed-frames/ ``` +7. **Using Persistent Sessions** + + When scraping multiple pages from the same site, use a persistent browser session to avoid the overhead of launching a new browser for each request: + ``` + Open a stealthy browser session with 5 pages maximum pool, then use it to scrape the main details in bulk from the first 5 product pages on https://shop.example.com. Close the session when you're done. + ``` + Claude will use `open_session` to create a persistent browser, pass the `session_id` to `bulk_stealthy_fetch` call while opening all pages at the same time, and then call `close_session` at the end. This is significantly faster than launching a new browser for each page. + + !!! danger + + When using persistent sessions, always remember to close the session after you finish or it will stay open! + + +8. **Using Persistent Session on a long flow** + + Another long test example that makes Clause think: + + ``` + Use Scrapling MCP to do the following in this order: + + 1. Open a stealthy browser session with headless mode off. + 2. Go to this page and collect the number of stars: https://github.com/D4Vinci/Scrapling + 3. From the README, get the URL that shows the number of downloads and go to it. + 4. Get the number of downloads and the top 3 countries from the graph. + 5. Prepare a report with the results. + 6. Close the browser. + ``` + And so on, you get the idea. Your creativity is the key here. ## Best Practices @@ -278,6 +312,13 @@ Here is some technical advice for you. - Use `main_content_only=true` to avoid navigation/ads - Choose an appropriate `extraction_type` for your use case +### 5. Use Sessions for Multiple Requests +- Use `open_session` to create a persistent browser session when scraping multiple pages +- Pass the `session_id` to `fetch` or `stealthy_fetch` calls to reuse the same browser +- Always close sessions with `close_session` when done to free resources +- Use `list_sessions` to check which sessions are still active +- A `session_id` from a dynamic session can only be used with `fetch`/`bulk_fetch`, and a stealthy session can only be used with `stealthy_fetch`/`bulk_stealthy_fetch` + ## Legal and Ethical Considerations ⚠️ **Important Guidelines:** diff --git a/docs/api-reference/mcp-server.md b/docs/api-reference/mcp-server.md index 03cb102..045018a 100644 --- a/docs/api-reference/mcp-server.md +++ b/docs/api-reference/mcp-server.md @@ -5,7 +5,7 @@ search: # MCP Server API Reference -The **Scrapling MCP Server** provides six powerful tools for web scraping through the Model Context Protocol (MCP). This server integrates Scrapling's capabilities directly into AI chatbots and agents, allowing conversational web scraping with advanced anti-bot bypass features. +The **Scrapling MCP Server** provides nine powerful tools for web scraping through the Model Context Protocol (MCP). This server integrates Scrapling's capabilities directly into AI chatbots and agents, allowing conversational web scraping with advanced anti-bot bypass features. You can start the MCP server by running: @@ -30,6 +30,22 @@ The standardized response structure that's returned by all MCP server tools: handler: python :docstring: +## Session Models + +Model classes for session management: + +## ::: scrapling.core.ai.SessionInfo + handler: python + :docstring: + +## ::: scrapling.core.ai.SessionCreatedModel + handler: python + :docstring: + +## ::: scrapling.core.ai.SessionClosedModel + handler: python + :docstring: + ## MCP Server Class The main MCP server class that provides all web scraping tools: From bcd39d57c083a2f8e0d39935b45ac1abf9dd2ea7 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Sun, 29 Mar 2026 23:46:25 +0200 Subject: [PATCH 24/33] docs: update the agent skill with the new features Before I forget lol --- agent-skill/Scrapling-Skill.zip | Bin 81922 -> 81696 bytes agent-skill/Scrapling-Skill/SKILL.md | 12 +++-- .../references/fetching/choosing.md | 1 + .../references/fetching/dynamic.md | 19 +++++++ .../references/fetching/stealthy.md | 1 + .../Scrapling-Skill/references/mcp-server.md | 51 ++++++++++++++++-- 6 files changed, 76 insertions(+), 8 deletions(-) diff --git a/agent-skill/Scrapling-Skill.zip b/agent-skill/Scrapling-Skill.zip index cb75c55f86dd5bb4acbb3347b8f1d45202977041..987ddbdb54bfb379641ce654a9c7ae5492f23de4 100644 GIT binary patch delta 26199 zcmZsBV{{;0(`{^HV%xSS&cvSBw%xI98xuR3*qqpz*tUJ2_tSkZdaYV#ovN-~r%(5f z-n&kqKV<$V1ft>(a0m>Le_hM5UI~bd@Wbjumj7k-5>}w#MLA~wyyiyZ&cr>+FnOJ6OF_qyQij+vNe_3 zM(M_^^Y(;R?^p8g(k>$)T<(Jle9vvN??<#MTXMc*-} zSNR595=?2iGB+5%u`w8-;-*Zi>qLcz&~lJ+&>wi#+eA=Hx*} z%u%)MH09y-+_wn*LT4Y5Gpcq9fG-_YX=lP}50B|i2UB5$5Cs>ZUgo%rH}|11wrsIU z9MUO0w|{zaS7`yj&}6+*qu9}e*-ZIk(m4Z1%$!3E4{S;h987iQL0mY{+fx6^4{?QU zG^Ns-!YaH3h42XRbwupd?BJUX|GZW@mMzipQA`L2eIa5P=Q87LkG9g;tn9I_npRj)i?U1n;ImHmlF1c})z=Sa3Eb4Q+ohLS!niXATE` zyx)8bt*+w5syx1~U0o8yI;mT)Q{5tv;ZUMyw#c*JFVoBv6cLZ(|%oSTHdGDBA1kcVgK0F20_gZjTd}(*fnyB8(IDb8@0GY6cI{-GQ zZGjpZwH)YbwVm?S@|^0yuUglZ_Q7EyAS_p*PnD+bW~)E7D4Y<8if=xO^0!WY{*vbx9)6a}9&8S0zS$j82tE zolI?=l!cNXrVA8Kh{>ce!|6btqcEZG17;Sv&yPL_g87QN^CHY+#>Wh^YNP993ZQ|E z)x<)cYZ7lwNxH+w9-4O*l6ogk+hWDT-j1mj`P~_O)Z}rTwV5==lr$zhv}z7q^-Z?u zP6cenwjf4^R$hyDf<-9pFy(bY`>G7&^*G@fDen6mgvqK6{B(r>rydLhvE)4-{)>h1 zjFr=dd}SHWgOH8nQaKhqMq7e44qzll0D7`|mZ*jbXkC7zf<5k-$mXA&q2lt&3T-MJ zAf1lb;ZJ=3&K(NZ@CtPrPRB94ui~~2xanR#W z*euxh1m0LWg(+E0Y&jZO@Nn*tJy#N1C7HYQViua`py^ zKL5e3p>cU~QrWrp=O~u?j0?F&6J&Gz&f@OnA;!iKC0)H5xer%%aez!n_X^o?=>Cd* zK@aaI8+n05M3cTa(Cm$mVl1lq+B&1J3x9!OJmAm!Y8`CGM>vDT!Z~>1?RY)XvUv+p zpS$bmu#%6B2N$(>zl`->0Sw~8Z^+^AfnZurcA8)W;L9JU!n0l$EhP324my>xMEEI2 za}fTdD|jGW?WbV4wOt3l**lFyJaNv(#ZAN&H3~XL_IudrN<)V^N=u7~M(rWU>h8oh zj^Fs^wc8M6hMb>vF>p2Ni}T}(&W`GEV8(i94WC@Eg>v+G#29CwyIwN zn~TG1+~m6@Fpd`Q10eC(w}eyTksRYX{aIHtW7|QWCeDYhTr#lB4O1&OG{M(`0q32c z^^RSQZt1piS$-F#y^gK`L#XF`{4G``4ek zbO3fza24x^lzN!vlyxwmY zQV71npW<470O5bWQw#?pi2UKVw%e|CT=;BAxIpU@1;|NZ6w<#d;$x`Gc@-tA%9TNx zHFTQx*5z8&H=!o3ANS&CVx@*UV!?%$AyMdy=kBB4vHm|(XiiFszOfTFSVh2t-NLA8 zVCL$;~2<_k2vG?0+UeA&C)+Im8Um{E5espVDG*A zqjl(OHcL!|f;>VkLf=vfHVt9LU6W)Utt!3o0_?-yIq*toM3WUPk;V4pf`fG&9#bAf{%jr?*Fn|XT^F5?t~^f%3#7#d7A{IdcKbS zVeYaY1*pLu68G^V+a&fk+kC02PfSvyB(*#cTMHo<;aKPj<^E9BkIP}>A$_ti(5u%M231svk1+(r z+3lh3ZVGG^T%K~0XFtN@k$Nt~=jZ$1kGD5kNDsY#@YLQx@UHzlMKg8Ji&}Na6x!M~ z0_f{+J=5t1AIG((yfEsC|U9`-|siX=MA;!Vh-tCpF#e&hU^oOjSF|J)5(4_4K0kk2`Le7B)qd!%qAg+{yAMQ1=tbb zST$HmCL^ux6XWoHi=dP7?B5@o5yZeu48kuDD{hSJkaD~J6(kfN+3|a3=_n`mSfb3# z>k8vf=|#n2Qw@$BU5S7#7HH{Y#aNT}e8v(*^hu5KPLs+^w%d>14fJk^A9&*S+ZoTNpypXbBu#C%%^SV7=3uds?kXSeD$s*X#GYq}V}y`7K%P z`_Y-8P=0%A_IddLVi{_DWuVOdB|^GrEcu9vI!VvB^oZI;wQKojyO|-Y2v8ebh7uVQpZ8)pbM-08t4Vj>8?al3Qoyb1!&g;nsSxD1;O{!>Hp_l?ohRYkQZz%L_pBUe z#=6LO@q<28o37&9l{Rw`UFj8ro3Z04sn9d7bL)K7&2ei7c{PkQ!owG0a)0~Dl~u)~ zWyNM{fO+p#6#bdM{8!Oem5eB9G(YXR+rF*F4d;2KVh7CcE`~w21<-OWSUnFDq%T98 z&5o38CM9I;PexR)W(iN3QiF5$YD=79oY?Rhjp1s-MkVnuJNuQ>h4ldC_O&S6yyr%C zY>Zdg8NgIfCA_mS>-!0FttD9OHl4PyB}a5V!6F4Pg*^2{)S~LHc!`&?T<~@GsyaXLBB0@=JfAMr-tZ?Y&cccJ;%aMt#6KO~$-s;^$!$ z6>Ja;!r+37Um8j9vaLw~9?6!}wG|2DXFP+?vV~XthJ2-NS3SIUltlvMmn4U~0jY5t z9!6BtUG<}Kf`JtdC*8i_AtkZjQ^*I&9iRn5ql<8KW(8ZV&i$*&Qzw?3klF|D!boZO zpy~`h0mUECFkv`@_b^BxAckB?$;=i1!&Y7X;iJtgJU9sd(8+KA*uTdA%~t;vg=Q6n z|L^F(QR5_Gdz7T8V&o)!dzhpPRx0TF^g&CZn88LowqUsbnEdZ8<(P-)a#+UgY>;!w2 z|KF-G8XdfB{~xF{Z5UAGMu*5Wod2gtR51p@zd11)$@)vs|IGmbZurlh2`z~NhyRdt zzkhb~U*m?(S>l?6+vjxIaU!kw6b3X~;(7IC@o>uQWM#5#Ir|(fV|A1bm|TOqFnb_W zXrV@jLVU&h*z~@@Ex%_#=l7{VyZ!^dukO zpO{QFy9>Dz7nP2bnrga*ze_)wt#1FO&Jeo}tutmuZndF(?WHiXa#-8w5v_iI`Fo82 zx`G*NuU`7gmlZqPJ0C;hkF)7!N?T}ZTa6rtdNceTjv7-8?L&X`;xNEb!Az;LSsmR@ zzs}3G+E2};VeiOdtu--Ii8Z5>)8p|tVDWRgL`Xo8aqool#Hy=_ zhJFeCL=U{W*QLZT!x^v(jLMTQlO5WwtnF658?R1Qpwm69OOUp}8os(BeI(%5%Dh)@ zD^>r|Dxusj2obABC{B7lqJ3=chh6gH=-RVk{L%MTOuHD8S>RG_*5WU)tS5q{oq-VQEsDRC!ZWndUU^Az8rKdYGFO2aj1=|uZu}S< zD#AU%*99gmZ&0%X%JRf*T z!Xx-`adY)~Tqd9qe!Kkx<$&dZ)DtPpWXGRvr>Ue42X7GrCpvDR|4}3l=e)f#?Ov8G z#MpOaOCW_K`XQbT?FH!*mpA2inOzfy>l+bBFb`7|Q6_|QHYorF zY~3n9L_+W6$G`>M9aGFrGjzQ}n$>{=fScarA434Xm*78%k~^1cQp#y5=N)*hBp?i{RBIKC z6W0|#E-G8J{o2*e9Mh@~iK;jy8xOl)WL99PIt6Dt2d62LJ2p$=rxA{K{4dEk{a=zu zu{x3LTZ$8NRoU!new;dc9qWBRwRrXOK0iL*o(_O)A?uTASCu-3df= z8wcC>u8)5u3yxS+i3xct&IE}naa+gxRhJejl{P%UZ8e91{H^N6HkV?fxq-Lqj?fIK z+QkOq64T8E;Z}ZwG?S`l%=k~SMmZmMs&V9t?$kl3I41H-T?$BrOTp!Zujn8C5NJ`fsh*;Plwd>N9Tulh4!RDLyq zI!-WHMe|t9iHk~^Iv^?R^@R6Z*TGo^zN7$Qi`!Nau5~m)ecs|d&L`7f%2y&_z3bYW z>075)U0!;Ts+~lgLIUlde7-$bh$hz;BFrOR8ylqqpUE~Aln{@A(9Eo+PnD36koYn~a1m$= zyY$ehzv&!{tF5BSGdo}fF>if0R(~Jq`I-hk@IKHp5`M-#0$``;*P?K*h#cYvQt%>pa-|Y*n0CvDGVZdf##DGw;U@qR|pW#4nY*Xi%Fw3Ugx@G59k>`hLV#`!2qooGc0)6<~C$#+MF1xT%va$>Dbt^GuytD?;JCPt@Bz`$%i0S z=4&mN5qM>vi=2eZO>F@scw5ukE&Fc# zEfkujPHbBlO7RqvxcDy>6Y^yj5!Qf=23vQsz}GJ)_QuVUe*8vFzS z%m(xi&#w$60IE;M$=CWPUy-M3?q_Zj6vgrzumos@4&uO~ifVVIYYS{_7iU7*(l!Rg zeC~1Uzh+n_Yn=P)V}Dr6Sz(!i(ySwkYKaOvvNDQB6tq zP$oX5LNQ?!B2>$&qzIlQKf_&O^)nERqv zWOMRPmypr9+K=ROGl`D%h?;PMeKp8$O_t~_K*K1gJkJ9hCw1y|jRfZM4iPb=s%^FH zD#~xgAIIXR5MV6Su!JNlGb6D=WIxT zXD8OrM}efp@yT->WZJ2Dxv?6j8Y`yvE8{j5GK0?&IRp^V4oY-7*{nmgS*QJqGW*tU z1YFA>5hnSEGq7^7j*)`GdzB^Kbg>;M_1gd5%lbJ7sPR3)Lz=psAp~Y7jW6=Pt{qvu zg5`DZhs}S(bp=N&&sRN_(c2@_#i+T&9I2BOM-)Oa=?U4=o7ckWozMCG=^kw(%ziy7 z0a^h{!1NN^)&e7>ZLkT6kL;%g|2bpN1mK53DzJjXDP`(9C?aV{c0W!mScB6ROlJz> zU4HZ!f`2_SlPyxHBg<|w&{{vM~F-~&^nV9lrH9YZ=P{(M1ZilYf#wL+W2R!&aj zSP1yej@>d`Bu1drq?r|2(*_nmwtP5l#ev9#o&r$#omZJJkZ1JpPg zUstU#Qevx6!&ZUvD$m-42Ss|hCUoE|RyH|e!LsWvY$x4Us1>Wu9H;)w2UxvL*=W~A zrU_#>Sfj`1A?{kylaRNV7L$~Wcf_mwXW3G+=#tP6`8JSn48A7}+$GCHM2vj76*fOl z>%j!+)L!UV9vx&&U_^S6B_Jui4EJtNKv64T5i1exUYXyg2NvqFYdP?5epY=85=pUTR< z2&f)&Z|W2&mw@gRvWS6YiJo>)%|n9x7KU{BhqUE8pWz*9Y?ifQ9rgu+HGl_AycG4d zZ3x>efA+w;!!sjGG?3xdf{B4P-_rHg@>$G{e(;$HnjRiSFeH_Y+@(iZ!qOpfnhIac zt)sM|)n)OpE}mdHKoV#&q^x}6L6>>c98qcG{b8-s+YK5sel+155JOy$!-KvIvp`|| zKp#zTN3(gwRw}4cq9k~w&b_GXkZIx=Izsg~Z6DXF)qx|eXC_cBrU(7QQg3h$Yb-k* z62)}z%lL%!^=X6js$&~({_~P?6FXjsS0ztn%R_F!fu&m6eNhyuAJE(eMH@@OGZsG} zz^}BbWFTTgD#Q<6SS5ih%*HOc}iE?-K*^a3}$B*fB*J5-{lEku@Gt)XI@n zGEHQJXXb^j00IkhO__R$Gmjyn0Jy>xvR#a@Exq{SEN|azv{@2Ymr199*+)oAiuw5n zt{e7Lche{Zmc5J|rJ$muCkh5>rY*jzKS{KH%O9x)XY%AZD6dfWAL|GgwotYriydA# zn?rw3upyEg(ArN1D6Mp1*wqvmGS(jPfkevkd#!)omXf{H0h-9vi2ge&w^f|+^EGzN zG)tE3E1rR%gCI>5BphiiRytchn1ADrdK61vV&je%XR!+F(H)Dz5{U9-IBmEY<61a7 z>h%DiFk4xBj_?QZpWkQ3EKpQ!YsGpUUGNpVmVib5v&)oC68_C{!=n$`}z5jx?la)i!#Fy~(#hP=OYd@FE83J?S#=d^OV6qvw87yJkVtz+5k9uF53uB13mjTv)G zeTucmRyLLmtl-r^P>G-T2*O?CF6b3XgI|0l?$Ff-(jE?(?k-m*Rq{wJTOK`F`}^s`5GpR1Xk_1qqH&&ICxQQ&dR?;kR8{sO%o@;EO7zk0L-@B4Hq z+g9Wdx=wR!h2fvv1dfR3R}~rY5TIr40mb2S+1E_6j^!$c!$7kWETvFv69v?IH)bpc-mIg;tk`vO7W}&0I|yL zcGXzNbVYFNsy$6=(lPP=WTQHm8`b$!quy@@H|*cAAm=d=e#}))!YYYLyWf1pI;4`NA5g=r zfm&4cDM<>;^NCjuQH}Pas-OE*0k!1IkN0WDh#Sit9Y0Xjb|T4S=WRgFIUx;~GrT3y zl(=z*4_h{~ZxKr^3znkEp$-JNutE96y>?;Gk?bM3ouZ>_$PcnZ}V11ant!p zD}uaY@e_{WNBBwyExmIyMm!>d$#2Rk95T7iTV05mvm9$AtJxHPfBNP4Ie;YMm2FTS z^2li%YGlX;`Q*xD(vtc4+p5dW?)n`fR0SKXD+gPS9wTiroMXv%p^-|Z2^@kZO}hOg z?q#$A{hZ{{8XpX(7QPd@;QI}F1ao0|d;B2m^ZR8!^jtOfevzR0`!W=sH}n^tv7H=+ z+g+hkmL~xxdc%RY4w5*ib->2i^8B@T%9wT9$<~D_Y7*Ns~Xbw=3)n_t&K#A-r4 zu2hQ1EI63|wJ|A~^lvdC3dAX*(RR1uC`&PKcw|L3jWhSAxIVXxCZJ8e3k1O8M|@P4 zIW4}beC)*SWa+hs1qbJiz?+#5*l)x=XTkel_o#a8Q4G1!?u9Q~(%d$V$=}r5{ z=0c{tuw8U=-LCk-5a{357egA4lc zkmd&3TrIMiTY-(=SM$fatPDE|hmaf|eBFACV7LBu=F`?ESjZ=HP*{oEtBVEQhu3EM z0r`GeKM0?C)&@?MhaU6-Ss1m9F^1+X#2LF_m(DLd?zj@~1}G9!GJlG5y3*!)M@+EP zFOZKbJ37v=5kTbeVAtF(4+9#^C*qmyWxMr+<<7IhJG)xGkSt6DkKMuISLk1UQB>k zb#$t$eoHqvkAJKl1wEauBS5Fchp|e|njJpJHoK>>s!fZrBC5UTyM6dr1y}c9-Y+Jj zla$PAkBJQf&XD9u*Lt@@n z*DIrpg}io41bC#o{5irCnSIX7HWgBhyYJ2(Xh8cmUer%-@zy4NM zQ5*b@KUS#GVAu0^u_n-HfcbjKDz+X^Qn}CmH8P|8%-as_UFQeyGAjb#$#k< z+nwcvzF=2In|-8KlZ3yVwjEQ7R60b#!Q%STApGG6s>=QdzYm0?88`YY#xAaAgc-^k z7D#$HuKb218+fb^b)VU$>8vl=fz1iH_HvVR=Ml(lz!9Z8$8ufd#ol;T%D^e_Tx$6elGV0R(0nfC)FCEC`5nkc+-IEe?2zVKeZh<<|>Xsxy@y)zJu(4V83lp8Q zsqS8DH{_~IVQ9gyVqn(0ctbx#t%S|0DDV=CF5!uU9P~*n>gnN8(QWI$s7xoR3~l_) zby@@&|F5+KI_@i~E}!Lm9Zd{CXkGK7CBx-RcQZo)jM|83U`EQ}3wAs%b~7uXqIK0- z!mA|UMJ4IFl%I8i^Z(?z@=kH#UuuPN^C+6|u~UI(w|7VnyJ2(G@fL#&NROr)13>!K zTQpL5u~K}>2>l^)d;GqB=xjLQjW~6WsL1|I0qX_%A;bF!sp9lqDRn^QQz^oUhEW_6!40|>pV5tr zuOrx@a35AVB}<3$D&c4hejoGJ^=abM-=Dn0W}fGFeDB7~S#IuSEdYhQ;A-lkRqZ}4 zhgmDfgFW$-JCWilLwA%1RETHT=g^eFU&t1w&Pnlf)R!~Kv^3OqrW6#S4&{>5*XE5hs_)L($Bf;r8Y?D7 z651=0<saC;v)sS(!<%XDilNTLKo@3-TXWzCy5wG3kFc~@rm<5Q%gb4m&sqx4UsW($kd zg;1wP&Zp=W_kF6hDV2LV)lDre^oYS_?0g))aJ-!B*nBf~*EPxRh7zcz*$1#-VG&ASio$n}%yf+j!~W zedIorK3(O!uOkuvt|5?h!apZ3spXf4bM>~TpMw^RV^ zh5R9fwihF!xW++my)Mntw?ZT($&2q}1_ zOIrn3{sLmnz=;i3A0H~x%96UZ5`>8PeDpVjq07eW1_NBI@-is6zF$I%S%Gb-QHoja zUMl{Ztwqo~bQrv2*NQG{ITY^@T;8X$(ADCTEV2CdTi@H= z$;R}{8#waUtsuk2(N&Pxwo001m_{uzXX&ZHd(<2A#n+ww7bO6p&OT1g?j>So_E}Li znH@`fk3`hqFU|XUvkwF zKQc>w>iTKCvywKy0q@n=`WwI)XTL9Z4p)^MKdXbVxvs`%v+#HO!!ei02nyKc@2USs z`M1uFzmA5XtRJn*?9KPyTGHQkj>h3IA@W9v4i}PvvNn6o{(Xcl?HIJ!BcE!buIbF zh>)+i5{sqHe;vqF{fcB@+M> zs8i8n=zoGmAboz9^sTJL@$Ypva*J6{hY0hZh=ty_{T6S%V6kR;h3^~?f(a3TB3B-|MbDKIeXe>4dO+v%allJAz@e@MVLgcosCn>P3pkj-Z@Cg=}uG zYTPbLfG)E4@i39!gV0LXd(0j}gKS(UxbhPU%Mj_{bEbrZ&DVRtpbiROVYIC>N_u`E z`IdMYq}@rm>#AYRXlunU*%>PvNU6q>#Po;->`hY4EIuz+|4Ya>wFm7fDX9bh1~DQ1 z(PwmxLN0~d$R~Iu2Wi3WeV9@aJrqD2iCSf-Te#}W44$W!TkAiPGi%zJY8EDYMi}Iu zNJ6jgtWP~a0TRwlQu^M^^}&n-GFp;52A9p=;osPI8qKUaZY7G+4=vix?3`g_) zN6b(7N6dHhr(eQ|*z=$gwN{TKm~xEc2RGRr%~|E)fDnV49f-h1W1aTT4%>xCq}had z&p-gO8i8P(;A?Xh|4B-Nqso%MA&N<2Pw12^>#zfo0V$a%Qv4p9kf`OMsJgA2;*>?E zEFA^>>3N6U!k__1nn}J-cz{vqbay}241T%22{EM(Q(7?6VMcc|9U>xD39t5IG`^jy z);u+qqfMmMtK41xk6fXUf*dz1Yi0RPlLxb%kJ{j5a&^e~zMG8(c^bcylwa_Y%NIJP zoQMXPPKCx3Fos&sYPmRR)>TtQS(M>tzWruLv9h>-w>k>mMvpwMNdOu%TW+Kg@}erq z@Apv#=zAQBTP=VlD=+@$5Js-1H0%5IEnXf{#W+8v#@dsOE+TuUWGsOhR89zvnEW)_ z+0Jeh6sA(0u)x5Nhly01DR+|j!)s_B_*_1cb$PP#h0S!zR8i8&{xMx79<4#HezoEP zVqHirkuOR^iCC9-LckC4YcuTq!{yR^ggRC#G=1K*LSu@n!@~GW+Q@f;PmuwUhC?(q zM2KvM;x|ddy|)krbOva!kF}X&QsACh?J|g`5niHn7G+tf5Scqx?bvf2+O44ZRu_Gj zyL5Dj$V<$MhbHScl%1^ACHisN$I09y{OpG@F*q|kr(9ZOLBKzhcFNO@)!hlHMlYT| z1>;*8#{Sh${#g8^W;tTK#N1@F>sGahqFnpGJ|*@u5#9K5MxVK!Gd{7*;NP?DrP?bD$T@`a_Krr`r|&T3f;Y6$925 z<_e`9{xd_PJz`e(*9*MYt1{t~wtEHMV04$`P7S_g`vT5MAS7tbWWQP8a*p!=GWp@< zjx$E=z$K8jp|kD2izvc=JQ*t^;}7o}>r&1@v%HjX#Y+{B(G-6 zXrol8+=8vli%Y^3_dn;&2L`^lY~rU;%?8^+h}j$6&*r%9P~YdOCivXi;f#bcT+xcV zlU~Dm0MBuh=0)-0Yv8{9@Eh33PP}&IB7bM}J3nl--KP21 zJ0JImGsmr);7i2#Arcpq#;Tvlyx8^QU&ZSZ7CgaU;$E&PhwwEYR>sYO+`fp^x$@kM zVi#*@5=Gi(B46Z>OOzaO^(*!Dl%cL?tA8VSE=Nu4J0nT@t$fD~v%yzL$oXeNvb`4&3z}4P;tWJDj)!JuGzZ(hGF> zH6h~1M&ErvEK&m`V%(skQ17WC=I=hYCLruWpUC%9Lvg7lu|2nf#>o)2@GqM7%QMK> z#NT4bbO*e^R*pqb_yBTw&4&j05Or;q&1O=Y`ce=RU?vg{j9QT4r<~ySzL{b@6?#3^ zU!;lC-hCu5`v!r47HN=_u4gWScSV%04Lh=oAmS04`+-G0#r3QcU+I=bs-e}kM?geI zf}%fbg&a>^P^1r3zWjDuwWsWt#^0B|OjK9{*HFu3sa7p=445$Dx@tj7p3j^(7QjY{ z-9ge9LAR0aqAqN_hokHOMwFsm_OC%UrtRQRw;FMuH0G&BH(2tg$4rFbGnjr&6jF29 zhSe?fX(6aZj-Jo?IA~Tn)!W0qGGH%=3l%Z8O*LXYxto|S=l7Rzd1qgq`w%u|CZ>MM zPF0rH_G$U0?*!V!Ad}KuSb~EocXXFD9`LEk{ELo1-NR8M+b}S z6TGzHrsjgW&v!rGsaIbNm@g6kmcA@m_LZ)x!1W{*x}f=K<1p4pmr$HTv#JD zd=|fpvY6=Jc_5aPpn!<~0erQ)2 zUiFwohF$t6-gVbVQ>Tu)ZI!ZATYbZD_qKHc-)7seL|7(u>(m+h4lsZ+zTIshlR;jm z$R#f5j**6a<`q%hBIZMeH6Lm=v1d!0wrvfysl6n{J&8AWRxae@k}fK^onII!b|&~R zd{9{4h;x0MiWR-X(<+69qimL?hf>nB4M|}kz<82xK;Z=|nUzR_#vJrRIR{8E2x#8exZ`x~> zAf0GZb5iJZLfEcG)`8$*Fp=~+E6|ossn-&wOxO=8vLUS0N|})fgnPa+Kbd64-6%SH?1M*l&TlhvU9q3k zt8QMhwK$($n%Ac-5-uO>?#^h6*y=j0()lAv-j;2*ZyG(TUAcf83Gm=y$YG2fM-Q2u$^c-0f) z)u|bIxqz8B$f-E{pe>nG5{=BQnut9|2iKN_#;#M>tidZU9*_13XHdp~GlCHiFj6s8 zo=J;c3m9k>>ai$mUie)fJj-9=G<+nAQHJk<9TKp^N%jXI+R2e0=U^Sa9%3H;!Vz2u zGVMUw&^qj!i`o1~pfcZ%fWDtjHp?(dwM1#TNuTwhBRor1<2?fnH12E)9fOtE46F{bjn0e0*@+kD-HTt`RAhJ>+2DdiG-)S>ptF~dnM z0O)eXme8Dv->+*5ea1d0^yF92BT#P8CSD74t_XW zPcHdBrc%<}27U+T;jGLczwIhzm$ki14c$bThU0)}t3Tl6?==X`(jm+usAjIt0jV{( zDFa+z>9U)j4|RN1`Yi+D_L*h((zbbF^mXMFy7L*lie0}7DtAnZ0_a=@y3Gb#b7*`w zLljmCNjA9lxXkS>Vc9h6a1gl)RTW7<8>Sqb{@hHxUYKEuz^1v~m>rLMJg36;&I`a1 z=V;*3JZ@FB(;eD>q4{UGKU!wm13bi{ywHbK8eGE|AE$MJ+4kKd5}XMH?}+p*40G}b zT{lwP#j9_#;Ss@{{e3rgQwm2|5Ry+stxwzf)y|Q5qj|dtj?Gr1RIX7d&t9@eWt6z8 z3LB$Oh^G`{J>eMr7~M1KZxpTAVb)ristX2Nn9t9(jvj&3x1KnWP@-1J3>aH+}x-V^;men=+A04B2V1e@!<%gql=DwsM@7!)lpe|Du zAnSod5Ft>TXrr}~g%frm0c_?=vu&1$?nUFOIcVAl67xz9mHA$fc`9^s^O133o?k#* zAFH|+y4kL`oOI7#t<1V85Z*O*7_Fkpku))-z3++0J(^r8hJ2zOAYRsQC0Uffn#6eh zeGIoj?diIg7i{iI+lY8gMuJAetA-V#@MsE2+z;+u-+>~T;L-;3> zWf9iU>2)MbpkDb(feDcxCT)0QU!hv!-BzpgQfRQIWAw!dy+1uncEO&?xAV!NHj56E z+ofTqzYgPn=ey`gb$3Z(c%`{e?8g7We++lysKecJ>2DFMK4|il(BM*z9dc(phEB^1 z0Ga>sHu7YjQig2|FY5Q2r{u{DXxh1qt#sn1MJ}=)y&k+gns{3+5_#*y|{DP|uYG z8$kLLey(%`v^eilu&0d&#fVkDFO+3(f`HR__Pfl-p(J#qpH|nlHDhc{M>Eb7)GDB{ zC?8MImXU?+nk61pa&g|Ba7jxkt2IqpFKg6IQanznb)l#jkHpn%hqobw-qNi9BRymblo+UJ)woQ52qezIZdkU1GF7B zqZKTN_;3%)3*Z{e)=I8}8-yXV{#R9B0adrMHOx6U6qn-e?heJ>-QC@x*ukyP9`sP$ zrNt@kR-hF3;;zNL$Pc&g|K5Gcnw3npWU^w8Aw(HIXTIv1eL!D$RY|ellpw zrvAe5CA3VELIso24i>qG@{4!e4Bo%kXj0Vw_GL#Y(~>%Wd_HY2!8)r!dS90L)?wf4 z*rqNC>#PuU6xWI5TZK(8U_W^<9&~B^9zz~q(kw#5r11wlYdIF703AVde@A5mw(*=n zfpUCh%Dk)zFk^mSHCBF>_~Y=nKxT>;K**OSjWvgHb^h1asvTMu;p|^momwQg6i*|4 z5$p*BHCjI$sN3udlR`+^Xa%4PwA(bl0D3pG9S#Y~W14?7EMQ8`jUKnT3xkv+Pv1HkreJ)LkcO&Gb>(pn_*R^~xrY@DG%7)Ncu@TPPm6OgXT^3=am zY@^en)vt%-d2-1JYiY#vhb^`BvF$<&PXyP&kE4sfZ?Dkme$zcfka8G5 zs`tM65-C9J^E~(W<23)~$in7|H3~Ie9ON{KDhKKgXRoKp>T}9br>`Jnz1?uzYXV@c za{tyv3q*hE=|VqXdZu>rZdvRZesnmu5Yg%eHvx6frb@--cB$zWKF7jz1%4ER|LDmO zAD^ozRl1u!zvKDh0&SJ}X=O!!K(yWDCuPjSn&Y2j(Bn4z!xFwxs37q2I&Bs4Df`W( zTgc(sC3~%qmHd*%#xux}6a<;WY&PK)!(&92Z_j<2y77^r33@zRY#iGV>=ww6g3JE$ zqeVGj$Bprm=BM$yoDDP={0muNx^En&Rht^t_JF1t>J>Bt`m|cz(|(vsG}pv9XY} z2Q{GZ{B!WwqomQw*J5;q{6#X$340a<-79&<`{V2*3QAQzn-|AyC(ygmnHeEAlL`AC z1}>KkUhK}LIGLbJhr0Hi5y&)yn+MH$u<3XU83T^I0aMvcp0k6DaI6e8KyztxZHRq96*&N!I>+d;sKV`JuQvM*e~UM)VFI{u~^) zF+m=tR(aRX5&r!hpP)t+)dEkL`7tw0-5*P=6xs=wJeQ+*%oy7I3s%+?vE$k@oIm!; zgr=M2i7g%S!n)cxv?5ssOy(zJ`MLfZ=#3LBft??{ECW;0{nTMCOZ=f8nD(hkd?w!z z*zNY_4K{)}$w`0Du*>p7A{S z+vgj(^2k0dtIy#uoxFK^kNh&vWX6C1@DTK0yo1+qv@XPGB&dB=t+`@;%P5?AGrg9x zvKvAQx|I}L0X3oWC#xWJm!q&hH+GJk6Tfm*vNdc~2v`yPW3SM3JHALc725Me?aar~ zi$#Khf_n>6uv!%FMdPQX!Ui;%Mp*J2R0GRHzI?bKATbbEWx=e7=!gxmv0-{Rr^^UOuZdM#czZBr9WluB$TD zHag|0QWp%@fsOt%o9V-&-HOq7MjP4QqQoss1wa)-MTw-DiG2ioDaGdD48&VTgUtfu z;{uMDBAzK>vAykFQmk9|#eGk}7@ahaRA?n{zNuGI7{#hMYW&?I<$T&9o;E2wDJiV~ zp1JvimC{jtPN~|?;RlBQoEDmdO+n}5Fxi^MF9&d7nG?9Zr z90CPubH-BRHPRB$!)#_dD7e{mEEv3TNjRUQHiV=_~W#PvlnSisQ3$BCXqAT;! zEDM32YJ{dV)n;6L_hR|MXER-0Q;ep$hrXT7Y0rcLwPV5llC3c1>T zF77ru(xmmhXX<9mIndBk`Xk)y%uR5WkQ@LH1(o)qlK-zdKICQQo<#>RgD75RY_pg0 zdK?kjcPEtbd)}oNP_+lxhUB5r*2A(IDX2}?Fvb`r7@wqS!*W>@h-QTCpAk{5*MGf6 zIj6nL8j#JVWd#d~cNDblLizUBnkOsJzy$`}%$SV%3ZbeuYh$TNHvh!TVi~Zm8Ba)+ zLud3Yty9nNq)cF)68 zC!n6veK0m2)BtL1F+|9nD$sSXKuxE}M=*_QG0FGVP0MyYgIJOh*qT*!tXK-v`)ZS` z6Z9zEBUrS~sGqovN;&jnBBl|A=ya;tA852|OKX0*YkWaK4EJz@;gRXk2g`qJM% zXVCMaTu{{Jw~I;`yR6%3ZfZYSX}>yUtNr>N+}6!B2uYRMsLNOn=%A4-krzx41^WrA z1`LegP=gU%3lD&eDND38l*G)&^64e>HNwp$dY|W>K!+H*g2{-{4Ohxf*HFtaqMu5z zLrbX3$Br?n;HU_j__?%R-i#g7?5i_=wLt{%hHOaks#E`*@u@)vOk<6*sq$V4GM@wHjm4Qma&W z&8=SB$x_X%63R$JUX$R=1oW%}fDv7X3;m)1B?M~-5Y;YFB`QH88TI(=BE8|CLixr) z*Q&zIhU~6RGDsyZu?s?jP6i%onuZbsWUxrP9)g&aFF)6Ca^fblSM%C5zyxU<*I~|ybb-V61pB&Od^;R21 zZVrWw`%|gj@cj-{S_DQ}UP!dy8eu_ytUC_$0ASSoWNpi9U6T*h>=SJsAM~Jdbd?q$ zU!F@aAc$ht7U;G+sLz4!DX{{>_FYUtb#Qt;a@q@uZXQ#2cBOSiTS&*W!dwzA_&UkX zCw!kinj5UKGaNe;J#EJJVZ%{>d&m()WzA<6d+kARNJLs~N?~Rr0lXKdA>c-xri4Y# zOw1NY&vN!axcnk*Vf6*3AEj9Up2DyvmJA=QOS%kG(+uTx{1aR z%w33I13H2p>~ogKm=H!?v*x=9=!)Q;+7z)UEz~t@3HOS`MDHnRM1nWRZO6hE9C;x9 z45iz1zj;W)e5H|OV7d(uRZGaejiQe{gyXkBQ`ykar$r}Zo95V(pR(~4@TdQDuxC+2 z<$DSq5nY;P!wE%OCPBIFYT$dZ)c-^aZ&PJ0u`vXFwj{aBfQGuJqw369D9&pC@w+sR z9jQ@YgwI)VPyoHgITGrxp1^KcU$4A5vTRC5IyN9Sj#XGa6YTmV(=h5o!VKISJaFnr z7ciy0n1dNUf5KGkr@!@DL07?dli(eh7Vo(}grpOq4nryameUG3tm{3{Ps!Y0>lt(C zHUM=hP89jpLHiw73H_)r<`Q;51?0^GA0In+h!iUk}1n3!n9mcFI z7F)dREbHU??bgxNS?koNjnmuTi(x)v{ZU~ztSav!$$yG`yZnx4Q6@j}kXcG38;{X7 zT)=tqG?bD{#@34G_z|%S)E`-`Y#r-2L!XBaZPSZK=G@aOdWc%S*t;H`M5 zReYd*4b&g~Z9ID2vcN8i4{&f5er&i=j25P7NNu8B;0;*uO0faf49Qq*lsvRN6U5lY!%lJ|+odPXx9rC>(@|wPAs5495~>RQ+JFr%k2Iu+8gb zeL^!y1$r?HQhHMd5L*lnxqC-m_{_uXH9j7^;W{mp>DxXHAm_=b4hQVu;wJiYeN=(D4Qh`84s+4%clq z-88W+cmpFzta(_hO$$5p>=M0rN@kmxa3uZkAc{}~(@9ZiAR9U}G}2lgYy$S{2J8T| z>bBe^UM)z(rVqyKlw8RJ5U+cie*1xA*q0{5>Yjh45M4j(|O(=nMmEqq+ac#hZ{KN5b=TB-GPsSQA% z!EXc2qqWKbERlOk6J*UMgI(oot9U&T=j!yRSRGImLV{=Bg6GCew_-#dP#%ymv<)x$ z+Rh>DfcM6bfMc#lo2Li#2R8iG$0;JA=DD9^X8_AS(CrLDs2P*5S)>=|@$J~cM9$bP zvfH9kxa!1xDG9=aJ(G_XZn)&TMKXkO@_e_Z+P}I7UCF*Y>*+G7# z(>+u6^Rp+|jf-1_kk#zdM%s3jxNckyS=y|JHPL(6jPqx1q0m}cU_uUa0of;rQ*k#_pCTGqv09wnWQ(^2Q5mFqv_7LA8p3OqQcRfh9 zx4=`8kP`l;onF_bo+IUihmRn9WxlCez3PqHiHk3|e~Ga}gSX%yv-_6{T>>8S! zz09#VaE?Ob!$O@vw>=X^tOPEj39?VjylBKQIt72U*<3K5v>Q=ea6^8z`8UX3s7Er9 zI0$>3YgR6fUB6Cd12(#=$1t9ZT6fS8P2xiwT3aUcH=$pCE9-9}3k=E%2>;w|A4QSz zHK-ZU=4Y7ocS(On|2R@gF&)1xqESAJ*dn5#Di!x}f^YJ@p$vnGX-#O5XtQELVQdOx zS~I$@^F}{~6-aS%6#Fh?3o{w+@61qP@e+<$1WbY*~kd0g4~PB_SC%;}j?j>|I2ZVm5UgcT`%V(=Va!!UYfp z|DU$_cPB1kv)RHIzaSTTXGtX{>Jh%r26gg?X<>Xj>FnW{`xRTMpPf(9t(d9gh&nNa ztBqgu*E*l08#8RcN+PJFcfxHhjn_xoijOQE58|yvk_kUd;mf}rP4GKyp54YPLU2;F zv;>~+zwI8{W?jw@BZ=N41y}ZUQICTB4eTz!@i`gD1g<&BSCFjg*&P3(%Fd&{`lo8% z>qmlF4 zXDDjgeOYS+qE#LwcuGrH6*hNo1{3tZ<)9C8Cc-bn?m>R}MCJzna{GGq6V(|+c@;lR zRJo8B8X07K>mnMfd`%gPN31*|6XZ%*Hi!IWxT0zHC%{I6WIr=edE=IfO#7`sXmm1c z@W=V4RfNE2vzfs_}F zR7&~d$`HF=dRm5Cuy$Pq53YftUIj0v2Ho7kB){c>kD7`%lkODV(SZVlwWFeahpnhP zyb}c&LY>F(Qi(}VKFrqo%|N##|2ao`Ys}xTcPo;LLx9xe%X?#KY+_jk$zSlyOOl*I zzL6KzHz$t^d0ijth!$rmKfQM}LNeI?uC{Esd!W9&o3*Tbc4O^f{H^CStVP1-_fYG5 z%~V9EEw_l14Y(!L(k`SXP4rJ?fDZ z=hJqm({b?nAYs6;%B(9qX!)is-EV@r(#;=Bh5^|=gl*p%WrJ`filO7%RHB~oOh3yd z%GwaCrpVn$!KzSgJ&4FYHlJhzDnoxIz~dC6vAV0?*xLjq%rLJGbA1z%sNWTae~rv7 zP+}XNc?bi}{e>5oYfQShJ@{Vt%dI|K{jkOFZ?V;C!wq25xnXa{_%=2HTsm>tPuf?h}ZIHM~cj0)wFYYw^$N0zw zNF8{7mVC5a4~mvf3y5~5+*UZy|S-F2TZi%MU> zZRi$qC|vcovfrbyC}TKc@}lw8jEN_krko`^xRu<{jte~%gHg8lZ=eG6W~i^lzBdSG z>Q7CDFUC*2gtPMy^z!bF1uxzo?*^?C9-ZpaZKS>>+9u@{{~^pGd3cg=(Xc)A5>%#L zSZF;`KvyI7gaL)iqk27iE#MXi?D{#Pl&dYT`P`XJhV~eJ9kj#xwx33)z)G3RsCf{(O=Av1H^bP26Hlv~6tMc8 zzeg;KQ`tGwoSmGH5#89b&)0ATE!2DExQ?%?KE0EtT9h1`fuLr4&U&$Dkvo6=c$+Ri z%Z_e&lzVbvEOOisTC9i0cYRL5kTCLJWfm>ZQ0953L20zkrsQ>=P0*ITYb%Qdp9S{3 zz4J|41>b^Hn7M^`m&14j1y$fj^Dxe&131m~7k!KEAe|sp1_z!WQ_Bo$Vv3)^`JB>~q5C^^;M{)2Elug?u)cRsF(@!05YVKq+MEW4Dl(kTec|Eo-dn`=F#3D=Zn`W->`c&W;L_7myEr^`=FJa*w;D5%eH zWUR=Jjn;g$BpYUdCsLexJq9=z5!`l@k_5i7(%bQS%IgFY zAAw-}Kb5F`wp02eWalhO(4nUfrLNS)>H!GX@S>r;+D?Xe9F^48WKU|Mw@Frj0|qpIWShXY zvd5Z6A-B?NYfDh-#aNff*Y+ICHOgn!W-o^d}4--FN?o|3idQI ze$3V)d^0Y;cr+fYV$JpG*s;(FM5WxiW7mR1h#(UespaO}no|=KZfB3Igf(kk@*5waVp3xRC%OBNKSZp>O z`0L5N{;X+vV!fT))ecN7v6d_1oE`A1={U_JLD>&HnyWFg!3{S&!>+GGncf3p&d(L!e#` z_3jDB$cAvViB4nuU(H5T8EU9DJi5GPQ}$O3FGaYc`b zYW04*mQz+Xo+zk;nDe&W(;Bq8oI6KwF3NCpk)!dP9(Qfu_eKzIk4py~%ZRt(v4Gni z1i;!lzBrQ;YM-Q>8`g^PxeLVLoW9=RRmg|P=J<3M&b8z z5|N^_%JHtVK_&FYnH;ARS9`z5iDOJcXJ#b5lE8Qetn+~h-#7BQ3K4MtBj^ zDE-rx?37f&qtOJN8URGrc0EA*_(^PSgGU!>KbCQ}2GzMyDj zI)pD12A=wvW+{K-g#TtppSblVGVacg=0WpL;oFmPQ)U=umnM6fPQSHF0%Lv2`O=oe zpM$*6k8?L_G4*dx(c{<$r`E{3dE1a>2dgALo5dJB+|B$E)_w$qCCy=%;!srX8E3et z{4%#JvBW;ppZcoRk;CgB_IyB)|16)0KziM9=ja5x>8S)L?N(;&9F&GME4c$p_bS|eUATq)*%c0z(hu(aK?uoU#@t% zWkIQZ%YeXS``sM$IXDq}THMF(lkS6{wrYt*FYC<)Za^c!qMiWSgu$2!1I-sr^UUOQON&W5Ioo8XLZxqf?;GkOc4;&EP5#&&xTag4GN*= z0EqsBGQk001g3F7a`^!`Nds&c5K#fZTSzWD;0dy7fvXsq?%)F$T%keo9%Bb(f>C31ZZ5=x2znB(= z@GHFH`vB1u|1weG0cZl{HU41&>3^+|X#Rz~#1>ToVp#r#AK(Pk40x5O zIWXYED+64}u>b%msUe2w^#<%fxwwDi4T%;2C2SI|2 zzg&Sq?)xkEB*7j6h!GnA4KgkSfPq*Gy|l`9?k~2pFaQ}+A_QQjc!^MI`0672|2<^? z-4BH!T=W0wLnL7U3CC;n(tnSm7av-~e;eYE{*Sm9&i+ID;$ch5@#_gva%~W zIy&=HmHA&_f&Sp|iZY;JXh8p5o4{TP@C>j!aD!I6f6L#xLEIV`)Q@B^Y2FdHX$JTa z|HW==pl?3i!6A{c6$iF%e;YI%2@qg3Fp=g@- zZS1&)O5p=3%0NPi1%G)Z2}hGOZNo?XuNAbWJEV&L6(BX?qZNYw+pJwuJu4U}4X9NT z2Nn~Y;!BOye@YVU5KdAw7Cr2L1pjCDUkXX=*t!3C?td*r1IG&bzXJ3jq$DvirX=kl zIKVIMO=r?pWZzSb{AR}F$l_)vWvPk-RnzrO{Y@KM8SYv>r#|d>F)MQ>V5Z`9rtjRv zzAdl$@Xix~>XX2mAn~G31;m|tt8y=lN!M|K9l`pFIhz8=pwW~wKIR)})Y%M{0VdMC zb@8c;g0wUIDplL`LJvBK#dulTDiL&rH2~zKY_6HK&frbiVgqsmRL#O6Hz$d4fpm=| zy-fRxwdg{qwwTx1u|UL9{ToJQ%}GH?7EMg6tmsD%ba6CPtk$N~tWZLNy80&LRg ziY`m8xPM8AAr>jrSz#!_L-ec}-SzKOGv+kvKVSFvMnfk+K~{0UBuRmVNGV4&`vE4M z#-in^hwY7hsZJ&gltKM^c$lGgSXdVQltwHmIQL3FND$Vlz-*~h zesUWN8$oY{moy-hEcaQS8PziTh2s`qOymBR%si{(q|=0f!->Opruz0^VDhnKV#Byv zRS?uefVkzdZ$G{5{rSWdi4CIAfd=3SqL?&Igtk+is8OZH75HMqZ%Emz2Pz#aZEyBL z$f#!{ePS`iHW(ny?o-tpwH_R!X@1u99 zwWC-&-;MR9#lW-CF)-#fJ7LOu+k|&N6zv`KQ0QH1Y#Rav+gZvhonPiYW83IRn6p<`k_vg@%DYsLsMbqZxB_5LlHQl#w zNpyF~k_$3ksupnO{Z?(Dwx?LBV2mZA;@vK+at=wg-d1DOxT!O?&X`1RTsPaj2>;JTsT()?-Cnu$ zO2}3c3j&Bp=(2J;2hrQZ2vGj-V@jwE7W*1W7Bpl@B3On)-QKIoKmIMOWM2C1?g~N< zn`ys7uEfKW(OIt%%YXo?sx+Hg2|b|h=6K$W$^}*~Hc(=EAOGU{w3F^la6=oUN^^5?@&|e z^G3DoYFz1Y1>OJxy0LrQdc+Amc-V+jMbj3lMDhlPyH+G}oHLSbHpuX$U}PE6oM_UK zPH!yIzIRDSJlMNsh1l+WaP`=mxi%?ge$I{Q8k!hg-)(dKUef{S&N6N36Df55)x;*q zqo|nF$rRcWzExAfJN7&tk;MYtd5n?}xVkb#kJCE_j%+|w4GBi)ff&3h&D1e8i+gnJ zC&5S6q2MT$Dr4gBAFmoZ2goe$)yZG{uNtc&kmGAkLik_HK~3ERbHIP)ExS=cf~{}r zN<=FvKKOX<$t8SiB8 zMW)parhuB@mR3`hyymgidxWt#2UL#o*~_&#=1Qk%TgA5^1K&5zIq2ZWsw%OO@ZFxw zo{kHX_(6;f$93VqZ%XdZ1Fy4zI7<9|IqG>PBwB}7wda#HV;#`7T~lsn_2#kT9M!Mv z_TAsW)8&j*fv8SUrco*@-B8(x$iIld=&4(;UjS4z=mh$@Z}5t`Vuh)a{>qM&C3)s3 zKk3WW2zSiz6Yp`g$P{X#6Ew7lBNF6R7VPmC1h==yx~qsGsS-mIxgvdvoblg9S}5}! zjL`*z9jvFr$AcHX!pW%~xX-$_NiwU0^DV~cQFk+@#S%PYL9}^VAY5GV)oVg?oh)hS zCIA@-yN@Zp21Tk|;VLseY1HL;PpokB!fOy}n;5(V_F)h74`!*3oInwktzNm}%cwMc z^1v`fwk>IAf9x4jN(U(6?qy}M`x~^bwgG?k0*|Zcj@*x`MaK|SV)%tTl*p_YA77iW zG;xuJUAc@l&LN(g;>&`_jIT`ViDzqY7y(fPpfvX(xQny__@|-8lbE-Rn&q{6*8s zM&f~}tAxrdNk8I+!u$SAB)V#fD6tkEB#tDEi?ea}QC~}A!*m`SXA`C3t7|QeGxm|i zDUKGw-NQQm*tJ)?*Pa@f7!&4ea0YI$^? znbvTdJV7PGPdd(x_82tQ{p-)1&6#=WaOQ7~!5XRncDN`xnO|eJq$eAp{W3JW)T%sJ zoXp=(p0}O2RI(zMf;@(zT8W5|G-$ZL_XCCl1@&4~omZW|25!um@ z-2|ftG=7t;c@_LJ=yMIjg~CV6qExxl2 zhlfjaW~^pr2;SUdvzidf7+B`)b3HjXh5amT?8wr}DFL>k)k(KA3vtMKv6;RKv#PnC zdvd>qIifR*iLa@w*20IguHLZ+{DJ%Atg_#D%{129bM{C9XV_$=Jt6>PXz(}Hj(oA= z_F=odsZG;=I`$LKtRYYI!Ow>$j*)C&%aA7tGZa09XPSkGBtMvO)}4jo(8%ROX$D zxo@-?aPl_&(v;eZaaaNJQpMAL1UFc2Dk+PQH4}Nx<~3%m{Mnh6A_=Wep#9#LGpFHj zUHWV!9JahMGH9}t8LAdh#An4RA`q#k|8TnX@q*fD;u{!K*l7T0pxzmmPAg1{Yzh`N zAq8`m844 zVjf2sBoml*Pyx^TE`7*O$?YWNj*oE52HZ1caACKVvdk-LRU%; z5}tuEk_Z<_qr5A|aav-t#2PvNXthtd@t-DdnQGx;r7n*gPf}*`2T%G2{m0#ZZIjRu zgarcPphMJXtx4Y}0P%m-(kx(r5&v0PnmjKsei9M_VpD0f3HJXEX^JUD z!~c(<34Q1n>c1QDNqTFPe^(3!`1j(hwskLKH^o!G>dL6rG$rXJRa<$nr>y7uTO70) z57rzvO~68lQ^pci7s;XqNp#dlSQYB~zSi?PZyRQt80+imPnFfk+!}Jk6K6t&4luOR zRLbOY9&=#54EWU4x>nw#$u@P(c`$H))psbjtY!(G=q!n)xG-x1GFep2I?x}}DzrKfM^($y>Ins{9m#Z8%J`YX ze2l_mP#P;;Hsf6Cu1Sxhy`RC2YGh2%;y!z)VNiw4FQ}Fy{-7x^qoGxc6*K$rkyf=K zhiO+PZD9X3mLeStV9!8VZC@{@(V{j=J-}x$7mYgCxkt&2noOJBL0nN&TRS&Z4jxr1 zw5J$u(a!QoOka~XEueIwslh*_qZC^G{Buyy+&9XJar5NoZ?*aWaWWo|sBF==6&bTd zl2T8iqApa_GT`Jy#$kg$XiT4APd*1Di;0rjenips`sOnOP<`roJHc(BlcTWJogfK` ziT2?Y^z*=MaJZe<9$p<-O3}=MPds?(cd2fsLh%#a5f*&*L#m7E|D%+;HA+#5=kXiS z-3Bf=mLHh1d&lv$yJE?e3$4yb*Q(Wv&VCR1`gbBlmda!M7#bZ_f68pdcJ%F_wE=0+ zR(?#p+NClzAYp$b&5e|ISzDbXwOupc)eJUgFjs~)KZ-#OWU!kYVNzMnsr4qRN%BML zgh*1EEpX1*DGe~SBfJY6_D;kae*|@Mk%#26i*)Ns?$1(4k>rJtoK&4Dax4-lVY9}n zXE6;8oS+aR8#p9B#=M4@pR|&CPDoydbli_YG(ma+`0;RsI6eVn@nIEZ#M)9ggNj8M z7IGaV0K4XR>&{>rdbZ-N*rw@yjVEm2q!JOQotE_;T}@0)>G$hiKd8q{9{W>S;xY={J(?AYQEY{HOaqu)MkqV>ewt$F6HE;Cpepi}{?aFJ03ot`6wtS0HD$I`&@z%*)hP~`u>7_EWMJ4 z6X1ba^9lI6+*wT$q-Db2)XgpZMbigr{vrKBq}6jfc3588%#xhf5d%%-U7<<#-hb^n!i~U^ZW^o z=N!tZrB&DzK~`0Y8UlukpzSjglltqG>3~i&!k2aHz(%N$u#=A%)HXqsI>)QtB&Yg| zw5UHor(#oIM@6I*$6wR5KCrK}EzQtB^#a}@HL(p25uZ5Ze5MtQb%34$XU$VuDnaNQ z1L8I4e1bENqEq>_5J$yoq}UpcA|L#5;$v9c5ZiApUvR6;=J|5L{CDC*_^9h04!|!# z$RVVlwWwS2B+Pg{La6QY>M>X-yrhzv=-$ExK(uXY;ZWoSE_Xhk@iW7uirc?kv-RTo z7`s5tk199PAG}M2q1(&>4rZ|Vg&|=k%ID3OnC*T9b>J4VBuEK{ue}2N=oz0i((}yY zLai>T>2z#AMSAeh0%{zsk^M6*kOFT2DVz_HKv0EbrmKCq`FsAkU_&x~boNc}`3E>h zCiPZL9tbYLu(U`>wiSuo;RNCH<8|Y0MLeK5ns01%6t{vu45g_0g&;3HXk)6F#6P(Mr`H$oKEnJAQm{2 zocy;4hsh1Yf@j45P!8hyH4Od4#Vu&Sm19N)&wE^c^vyd%Z4xnB_HiDqt48K~H#61J zM$q-%^Gf+(B*xuK4dL|fY_;fF=zR2Ypgg}lx`!BhV_?2Kjw}>7wM<> z3AI_h8K2;VEWKGZwn z!-0U9PK0z>+(M^9!aG)zYocwm({Zla8bq#O!PJLf)HptLAS~?6D!)IZyo0RfM%ld= zYIbbbCE-aweGsoBvuvYEd8?#2awJ{Ek!X)LQBys<9RQ{Dxtm@-gFp-nYHF!G|vM`#0FuUFJ*VEt3lQ1MM5Kn4lrYWwP(r#zb>F7P_;6}lC$ z+;7H;dp>h;J7#r9gAlWYp{8StVVU%|x#_qGV%4Sns|8NjcDmkYO026Eim$9Yb7QK2 z;cw$j!ZHFZj37uTEh5|<@6CjMV9_DP?H70gs%2PCb@ISk??7|FNAU>>Miay;>(s)J z9UstwlREI6o8ukAs}w~8T+u^W6eIEyNbt+Ub^`%_y~57(s~QtD39_K+bCwj^%S=5E zs~`&01C43;06*35ujtn>3XjGo9YON2jCnc_3K?TaFST57xsBi&I%j_s`J6!bNOB)Q zx#IAzS4c@(lF)l)HsqwY9l(5O9R28}QQr?u@pefB_EVR=4WU!jE^nf$Owmwd9V^V!? z4*S5Zl!zgswG1;L!o%Qx+6y#h12|%>K=nde|By7}wDbaPv|i!|ydbh^Z_s7A6+Qai zxw`K6PjL<+oL5s?pk@D9kGI&8B`85$!yO2`)7}bjzy%M?s8N5DSb&J@bS)QvDy)js z;AcT)3`okHnOqvY#ka#+-zovC_w2*=(_45TW&Zb^8&u>%yJ@+UdWDu^h&Za1V?jz4 zjG||mz##f4H8vzBqP%XQgR4jh&Dmt3mq>~XxE8`O2Wryl2EDvXQK)i|;v+F8i5e#j zc{1y^TOxDF2(XoOem}usZ_O)!?CXl_`pEXm9MbocBD(QskSLdDXAX1!u7H`)c~5O6 zg|=kcE1Ldlt&y-n)x}YC!3OYy&%P9djAinlT|%EvM8{&g;fyVsV)ls39`_V;ae+ED zZK$YRYTAJf<#p)PhCK{_JIxhp8zH@f)?%+9;|K-|k2x3>A*_TwE{__lxuEFM*Xq@~IC`Hbni`;2*)O$S3bc+0%yz5}$Y?j?^Up-$X(~w3z zCB)=|<$wp=8K4_X2&w4Pq`>_|qS)D^dKu1AIJH~}*x>9_L4!-NY5!-=g_}71$QajO zq)hd^Rn5bfpA32FJLGr(Bq_y&GrZq}wp-~+w^d`1oG3Yl`*@=#W38uPlxFQ)E1@D_ z)kv?V9m&3}D@?+mzfenZ{|I~q$b5r1Z?R(Pzu_tM#LvoYv(wyfW%v7m`3ynuuQtRvvl1%!pAeh zWx5!*TWKotVEpn{plAWrl{g?+*6NeY@4NlHF4h`(VMy_feimd&=H)%5r@J)L`s#NY=t(K>>ztlq6RN#N8f}TQkPt-JSekFr{RYDu%hCLk9IR zRBZ$ul?kT#gv8N(x`)v#mqOzQ3Ivok*X9`Xh~BgotTCcOUu8hg&+v<nlas)9(LZQzK-07cRlL^@)2T%=D(;MgVj2^`b0@BFS6Gl{12 z{4U+Qt8}W5KE4y7E}$K4Cw=bADjQ~wR{iG2Z7_?mcNctJgs0U(13N|g=s8Y$-BZ!PL^S=@WaO;(&#m$@f~YDL_gYEbQubU1l(uc z4(^z=+1K8;IFT0&n4UD5X6nlE=gUfx;V4u7br;K%)GlxAKm-K8<9@H%Cit?G$ptZ$4yAQx+^L#hMSgu7wODy zAofl2wQ0(?8gidH@Ut5E&NjPgzMRFiGzyX7b z22CVrl@%9CfAd^Mp8+3Fc$Tz87wuUSTU?>upUuqlo%o>jE?{x3t7oC^j&pO>l(26= z)DWg}Bo^Ns_`#6giZ!o^eTq zK7^cZ$SMT38&BK-wJBp6cK6Q{W=xK6wA&-2lY6#2lY-*>4a(QHRl~fDpK?5S$7Q@e zcHJmmv^R+hq=tx0xk+Tdp)MRSw|$%eX;KUbsG@N^7PxZ*>ji)COgqjxAKv}q&YvgJ zbEI^f=v-!DC`W8k?bi*@qlG1q8{Dbdgl^}T)ylZ8SvSUhQ2&m?105~D8@se)fZHZq z3|Y!_9F|{RtPQ&jFVQD0n_uVOP3QUXYlFu1!Oy$Y)PArt5}zg@dZih>KH@+HAbW9$ zIeR~RL2e`OLywleI`qC_R4p8poCJDCRf-vX4C8C28W2h2?rt51jN<4Qdg;H(J_zmQ zkVl6(;*8_a(Pb3DvrLt{vHf~{pR>hm(#dYsoS*YD1cbd+9^M~e1}HSq?Z*n)_Z3#` zd9F9J>dL|GI&2%FpBwW1*u}B{U=PxnFxpb)H+UbW`pfE=WxipK4Pe=*3aqY{B*i{? zjN)`SAb5C~iMEfArl+rgtkt*-)WSXnkdbiOW93{}^B#M+pCt5Qu}CcGkQO@qC=R7_ zR}OL>Fq#+BIXXw92$~+2B>v6EM+?PUOXn_(xMtakrkzT9a8_Aguc9IZSYeId&or5a zvN}j1B=IL8)NRs;Dg^>7ZnsWEkebUAaKCW8K^L#ryHnwFbI|as5WV9D-;MA{s+)Wn z|-Y`Uk!)B3__ zO94Us6_Lbqi42zAg|I&zz&(%6wKg;#1Yxrn+sc+Oxks6*?-9o_)Ne-IC5C3+1e&3q*naB*2XGkf!~jiGKFS1}^LqHA zpeWF_(R@*MQmD3{HYHt?Ab6|gKTAdXJ2c?(1ndkn8JXl!h-LKCyMS??5TxpGL`>^I zK2fz@T$cHKL~9-c_9eqqAVtlZ?Q;6CWJdSfKE@*SUQ7_XKQ%Xe*IpUd#0Xr=(cJH@DRA+fx<^e?Fl)a5#70> zH_)3td?6zM_qsmxAZEK|v%G{d`-SibccnawDS{+Iy)ze6n8JhcT^ZMraFB?|izdCA zKMt<#F$|a9f}#KLVELH2Ap}2$5B0_49{L!NU$bLU#Jd!)16x8B-c~G`Nzn}@u=)Kl zTl6U}(*g${HQdWtmaWGPO9Xn$1Gj+Eqf2%UoA$j0pj0&>ay$6NIco4%8D*=##tWPq zS5)Fvjj2=ySa#E1TvMoz9|Umq?cPZ@e4pX)&D-?|6$HH2R1 z4?P8gqzn9t9mZ~ER4wlBl!Aomoe!iadOk2aAX{>jk{xkE<$MOS5{EGMOI?xM)sd+O z@e(D>YWyfD!`t)gAb+O=pa7e!Ic73bZI2NHc=^hM$o$~L9j5|S^7d$VELAphW_y=w z>LVB%o#*ktppjcLX5yDjNyCBp%jfoR^uvqJGrhC^Q5b)WH zkl{bWX{NLIRAmnhLRn;L#T?3Nb%iXacNw&m+*n?V{a#IPMfLm&w=6mg7u752q;oU? zTUUlKzQDNU)Q_-@9|9lG@J zUxnsoS_8fi|Jj7m=mg`r!UO^`R!TyU#Y>_!#Y&?U0EPx(Fi4x@2gXf8WdHx{c7Vo% zTBT{j0Fx)dJ`e(sUu&$4|CWF1ZQad8TnT62Q4Q7beIOh2 zFrk6wKp{6EioCuq8ZZ8uE-$cn`kU_&cNg<8$R>vPIzvvzTHLS<36*&>h(m2!>Dp_aBYVF>S zwrS@};yvG+N_2qP1y4SuUrp=U-ChZ2+bzwPa_p`b2gA#o^QDzA9)8~1Y!2oP?fmra z%M`QnhkVXu990uOd@33#3r^8$!>v3~^)NYV?j2LKYRO(^g%c+k+BY!Q*^#aVX&enH z?DmC#g)KGfq9(2aeYcddj>ZJ}MEtl5jt(|?HQb_XdAa;wRuM*vd>j!A^7QYf3HDYNP|_5bC!&#Jt*6@+;D2!2yG@vb(ix!6p7 zr5#%pH9PbAYt3VmFknWApCXjjoelG$?!OA8k|;n>2kd!d8#U=nt%4YPKa^`G_|zC^ z+!Fjpv@9`4b}>m@33TmcU5g%YK4w~*Q>kXv-SyH!Pw8F9uO?s%Cn`A4EOufG<&y3n zDFDUx_mS?j2KT10_C1Ofz(1H1diLC1ycqa=yXR9PU9NgdzQ3KA^6zg5Aa@G7MH@JG z!(#oP@`TJyez4JoXFGGd_j`<(o7wb29nL@oh0$d}=;U5?X__Q78!XrMaXxj*KJe)8 z1L2HqRdr11nxVRQ|8NefOIaT8X+tEquLn$frp@Y?H;xDoNTU=v;MLvz((3VXqee6j z8r>T1;G0M9__){3@Sv!)GNGtsv+|27tC(@G>|?|zS#>eG7!N12*7OQz>Wp4AzN6dpD|6eJW7~y zpmO`_o1CJ;{(w+Fch)P*UxKe4Jh#Q{=S4%$g60^9DDKokMEi3u_@QsJ^xnMcJyFecrQSA=v?L=Js68wI4c#tss zQ6wu;glTGyQ8%$ccLpVLdq)hLUR(r_=j4c$D;1}tRAldvuS`m62$VD6Ta6I)XuMs* zf_!>f;Q@5Y!d+Ic`^N8nLeYLb3Vyxxe&>9XnGda4(K#ZJrLLNR@r!2`PQhxX11e&I zMo7H#L*ey#M0o8~#91cLAztmFsg$OdCg*5Ju)z0wl&j&QDdUB6)%xsV`$~TmFjhp= zlrqW`0rQ{f^n9SIIxS9Mf9YpZn+*x0~;*KC#S2p;sn|fynoxVjj7jl~3w6R%cFp zDwLGFNoLz1__9OFUZ`!q>NSDI@6-!G#c2aBf$9e136h?CLD+uRJ3w4{Ax@3S{+gZy zWVMH&+)eA-^oDJ%UvH3`YbgZ^|KV$~5i@(eNUj zM|-fn&{q=T>e2G?$gi4O!eTWwGGY8QJ-a-ubb)IJu%g)ChBI7(v739^gHe10nqUpX zG-t$6Ptw7EbyPO(6(v9xIrw;*3Lk=Lr|bXD9!3Fg+QPr}6Aa4`?&S5L0EW)je@3GU z3LmSrEhb6&`AQ6kzX{UmB0g}_u&K2*=9B1(jSiwvL$;-ok)LaMT0%ZugMOAj?RNPK zs=bYwPp8S}mS0zE%%MsR?Hqu6OG@fNF~FvzKKhOCkjNx)ns|H8@}p|n6qP9n1Zz-l8Fpm-Vn zG;}U2cVLU4Wn3#-e?I#{5!A%rTnPJd5IVC4cG`jA(o9D4hw}rp6NSxQRS+&^&n-?QZ+xOf1zOPUy}yvRpa&%aQZkVw`8;>Pkt0Hpx0`opz!Hfd~H^7aO_jhZ1q^3+m@x|igd*is4LGo zr(qerzR*J5H!5nRxb72|&lmm}FjGra!w>}>cADiVek{-`VaC>Owoeqp8=^a#kESCJ zG~ZS6u`+OK=16eLA+7!io}R(aU~BI65PeCQz26fB=uT}q;{E*ywBpjwF#9P+Z1Q{T zzUxh0xw@jL`5F>UU-yqW1@Ki&Y%_2XbSdg-Bf{}<&=#74@vg}N~~Z{Rd{ji4E$+D ze-W>{BvF)&vPg`MGVp4%{pK7yK0p|ocPx5jJyjdgZHzO@8f&<@0(adazKx;SmHciG zopc5PtFaA%;kOkf(&*dGPkwpGjQ)4Pq9c_KsFas^{3sN|nCOqMWJURy%^}T{*(JAb zIK6SK^P%I&N%+3}gp3*sq3t{`EWyEsS6kvDE^Wxh-V|kMh z?-yCCqwnO&AWwGq=!DYXMZALC_rrcn!!nh3roTBdPE!k*UhwyU%lh=H%ln++dVIF8Au}vtA*8$8$D(|5sr0@}(dO>Tdc#)zu zU{1kLT>uRt=-WNu2DC^4{FS=H87cIFQTDFgd6T%u#&QQ9bgx4Kx{Wr{Om%#fO*5K4 zmC2}?r`JZ->mP`{5YDCaKq*`Ke{>sRE;HtKL;}BmXETUNPGW&8xN~WpJngftJ5u=zU-Z#667$#jf7SXt$$q zsp`02mJyamK_t3*-#5@gX4C#`G!UjoUnw$*PlwAjYM!TKXpnkb{N08?=>dYIP;O}( z?dE zC2SckwYz7PL40pid0adcISuM>&dR(Nix$6gZ3BFXPc`KOiZp2kmB2uQ z;>r@K;nOaZ5&(<`AVvpfujXJO?O|iz0R?&DBDYvaK07~RYe%85OV%eLgXT%zG;fRrbK{e0P zM=23JOu@6{XE4sc!}Erg$Rfc;VTyHtJ+P}Z)0}+Q#_o}B0(@E(!l)>d$ZU*`Xn~9d zT4gcGudw<9$lN(VkJ90Tp{H4(%7UHYvYdr&%=)Yoq!LYQjSE5c2|CnBJ7GKxr;^?c zU0Pu^zOQmJ{NpK5_8$v6{e+vsv32>5TYBEsjo+rcO+K2``J&?NA9{+@~IgE)8zUu!;R%JM)805j?oJy+v+& z=sTbLS2vp-Si*z=oSm~Ff5YEAU&~1|Jj3t4y@6yaUO8{Nl-57d*ITTAq^`_R{^rmi8M=_|S3Gb720c^ymO?ARzE&Ab3bBfP@f53=QPLz>2vxtK9;=ar#OSQ`bO` z@S`dwxqo=?(1>|K_C11SxlwoHpgWYV%5N_Lp!~i`7a*8>5BzDS|EDL$fbJ#JGlwB| zoI`WFrIFT?s$w4dXRU{Z71(&NEzsH!LIvIP`pd2ezkaot$3>)qF>d+Udri@d@(3gz zja0}PJ9vf^G!BE(25o9-qaZW`Wc?zBlu2Y?pULyQsvugl1`~ag{+pcUwcp+GtNaZ% zpz)K0k0SgK52y3h=-Tg5u%*g$fmv%t#g=s?`l%P7uvYgp$Wo(j6zQZ%*gel4I6 z%FsR=!-z3wK{&nMot#PXl3`h;dYL|NfQ1-KUGQ@FFtFk&F#?K?1RwvX>DpGz$)6Mn z^Ji?q=>0tfl*or-CM`c?iHbsL^+_N$CPknui|ODMvSdPSpk4t<7sV(qU^vqf*Nh1_c;l!NFcTnJ50R8=hX1BNChlX=$+w<14=M$84v zz0e8p^7xew`AO&2ii1Np$Juc?dXW;O-%=fwBE>Tp8>vICTs9&{e1Ot74;1nfakd51 zhc2Dh(U#?FpNG#6hnZRGG9J8Hz)C4jj0Lh!ku6(p^CxVZ`sWub|8W?M!NW3)Efd9M z+~T=`vtJ`;A8sW?rZ=UP^nNI1p72yT)LCM;mP!v>>x%8Rg@aWh2_w}LXHCBOiXkz< z53j_@Sr)uJ&CqnvdNZ{=U>=%3sNDld(?TT~b++e~O58q=v-d~WyNI2GfOj+A=+{y& z>Aa!eUSl9xnhWeX6`;$LI2}0h95`(voL4sDKARaRBV!Z~g9x4cA{`DR4l_FoC@1xX zE;#mL@klj*cH) zQH#Ui{qvur*VDHeS=k4a4`rUFx_Q_SQrpeE22kf8Z6ZMz6}WFb!rj4Vm=FFo&!}M!s8&+gBuJn?XieK0FKc;ecrHfXX1t9MJ? zaICMHi)~FfBW{Jcy_A<2Sy-PlP2O@ufSuM+I_bz~MF`AAC_=BHF&Ic$K>wa>JIN<+ z+=@8@h`JFN02pjB$RdIpO)(ZE_vrXooC5x+Ix3=o-YU6BZ<7U|0?Z_I6?quQc6Uf& zx@9`R9LLNMeu{PCXvR4480i$QIBy6PO~OEr9`I@`1WnEg`Jb139^#fwEX!~&3vf@|+^cCl__AV10p#m11DQrSz%oSbN$-NF%^61} zp}J;#%168@GuL}dvq5XbpA<}H{XE~~sFqN78s?fa^KK3{IxBlC@>g})W1wj&TpxzP zm4W*NFU&+6sn+GfDeo-HyM%J$xE=A7+b?W?r@w%I&k{`7KLn7xgkLG0`Z-*Yv!zW0 zsTQjF0WMVJ9sC25`HVWwEj-5bWS>?vGPEOaEX32sV^=97u&A%iQWg_L|Na{vmn)`y z-sMmdRUvI3H=faLUFc$N#2Gg~cFFuZI_ve&w`40;i-$X!<*|LWn0iN=QtyYfhcSIEmy&TYk_X0M5}(Q#Q;TKnj^97f8T3-RF}msHUoC0q1h z6sTZT>(2+(Tha`9Gz>+wlT?XeeO@D&2aVXd`h)t36+CG>EGu(wM5LAjc3TGj+)@Qe zsit5>9E(JNCKsZAtEy5uVqTO`e_I)|dCILXbHv~#+V>a0qHcfU^L2TJwm5I1^7BX* z>79N9|AXd(p)uF${-Sw(MWDaw(f=f9L;bz~0{A3ru)lfDjT|qT86^ugyi%&&EnkgL7FQ`GV?k zuZC3p6h97LUesAR$CitU{OXJt++tEC>vMiDWNU0OfLTbXQaA)Lv3{Y z0BzH3^|hG#fMd&F?hDip zA)aa~j64c-=)I%p#OuO=Q>oop;&BF7KYwn66IWR~Q&;U|L}uTt&xu_-8J;MGBj`lx z)v)P;MD0z=U+PBrb?$OXa;|6TukINU)2PE^| zD0*$sqec0HF13?F#J3CJLup5P$!{3~P*79d_9XHi2LN!f-Y*J%y??2F5b09>zUzav zuD%@BAZUFcdj`wP!Rxu{J%E70;;WU z+d2sZcXuf6?(XhRNpVUk(jvu6kW#$RQV0}laVNMtMT!@P;_fa*AN0HT-QNHG$v8>Q z-fPY+XRl=BthHwmZMpqOzmi|urd{>Z;u$K$=N{hPEP`RH`qm^t1esa=qMkq0T-JQ` zJ}i77+gj798RR7~NX)~g!z9QhP;h0s4l`$$Cb6*8)@2&Y8g~%X_-(5$pe--Q+k^07 zx(DkkTtoAO0F(#4iKFwKQam~}ZnK%Z(V>*xNrym&$$RZgTX|xLXbZ0W;!>CbYXZna z1 zwxgc?Lar{4+$zensHE1*2GS-KpT0NJ(XBXlEu!EAEf4tav4y@t&H#1&I!+rdQCSaCoINuy<@Ns zJ^Fc01x(#dU7cRBFq(66UvH(?bwLuG&>2lJoqrL^6y9zbk_jyZknyXD50=eUYlJEozB~_ zHERbN+S7Y|@{D%9P`S3=*okG)(?Q~nK%p!wM>C&B`yw1cFw{Wu(tykpb5!L>fkL5c z^(iKok+*bjuGIUgQzoQi-VbL4Lp$PH7yA6#bP1qO(gtnEo_j;=WHvL&tT;I!-Qla(c-_o!A@7rKVcFW`B^vlKRmF@*;i13w=4u)-8YUzF{qhP(&a0pA>sBr8XQISE zIw{eyh|~@p+T0)iFO16M*q}_a~4Xa3axLiVHNGYkkr=eOy2p5-amT( z+KBhN>opS%J7O zkO2S&(zrHEinx~S{{+&ZXh7=6pW%P+*h3h`^oGgd0Xd*1v>o<#569#C0A$CNT*>1u zCU;XhpW!R`eZ6o#hu7BAhzy%ORa&F0$%brQV004F@VbSMxP%;@*{ktpnLGNH6O!JO zRq!XJo;vpn2+F)w63-6>^x)>y(T(HO(Of%mstevaLww5>+4cOv1nUapP2;{w7_(R2 ziEc8~kyi82lYw&#vg(>&l*i7$-aVxO*>q*FG2A}-_1UVqH@nIzcAJj@OIME}QlCgQ z#Zck7T57?P@|12bfoSn(bzN}rMo3+&&QdOfsQ_-J)4I&5?jnfC`w{-x?K@kxA+K8J z!CIa8H`@qV+hJ7y@J=D3gnHei*mHG;4E|qGwP?hf16oQ{J(M>lM7*2|v}&mMNdkCw zZ}N9xm!(^>@kgrcS(-~SbwS}vLS>(-b9lMb^Q~-f&RGsGRy68(lfpDIw(ULbC%9Y; zo!2%aeKrnZm$Oj3x404q7rK;%{q65565ph;zA!DYatayjoE1V2Jjeub$|Ej{%GO;H`iOAOOYI0xXOZ2y>fW3;j1TIv`lvro?155 z-Xl(5PtI#Lns}654sS$RsgLv*-v`>>zHh7J>h*iFT9>qtkNFD?K>-yKJ9|imlCNDSB zoh=r}I(xXvcnFoB7fi4Qw<*6%tzU!BvEm6Mt1uRXj+WEJcp!*^+g*z%Q1#;5q3p^j z8|%Dy*Ki_CxpHYiVNixrA+EYrJl%L%?{Wb^f=$WlSvGeexM`gEpa;~aNCS&EXn=Q* zKdlN^2&qGK)nL)F?}XmonuECkA>0uh}$Ig3k;vEQH5`)h+sEHN{S z1o|=9p&x%xzsRKDRdd6oiQTG&j=@3baXh0=s9KXYG-mpu3Ty`Gh)Lwy!xIR*26?t% z2x2g@mSRu6>>i+7nldH!E00VPPf`$rg#S%jUu_N2P-d={|LrBuoku+={N4R)2m zb9Dsb*A(S^GG3HxwVF3=P_TcAI_^?HQ)#Dfa8PjS#loVqEoqS(NJ< zHZY_;AS6Gb5H12&caVRc_v9zmDdUT>WmyLJ3S6_H0eaKrL#RocxMD)DX=&SCSsD-4 zVMwEF6en`v4-|IWWhX_cfAR`MV4capY_o1Jw|@m@PQRxQxuq5Zfsbjn1uxmHc43>El{?jJ8ypoF3Yx)4YkqrdZRT*ZGHjb582VfNQ=f_x{E{MpVmpjlTLm3;PF#m z<)e3ypIoDuOTDe?-nX0BNnI8W59HBS#k@J8Rf|?v*6DTr3PS=;gAc#W76(!#63q_Y9eLfK?o7v?uYUBhU*=E0~$}2aW_zjo7r4@f24V?i z<68t_^f1~a``0scg-N*z>@x38X0pd|o4b)44n;^th#?FHBO?py3qElCC3Q_0siicw|y*)RM*7yyeD z5f85p`D>w2R|aO3#%GO=U5JZTbRi zww5H`TNy*3I4{*+5x`l54&SP9+?HaR2jg?-zRBMRQ&nDJ$2tfK8kidB=(x_U>8Fcx zLroAs{q`NS>tH~m`&#!pOo0d!8n5A=&b+!Jt6YC^h^?s!XvwmuP?uppjx4K-I0HT5hEMzGCSq+Y z*{!%Z?oD~7e+x~D-ouC*t6|e9A6y?3fn|fTprN%kph6i5?;Ef8c;}Zq4#DHfF7WI{ zUyDAD@6Ac|aQ_x3eNU!TGz6L;Hr|NewrOr!f_( z9eQL9?jM4cbcn}Bj3N7NW(2{A&b>wFa;n&<_D!^ z>>mhLt*nE9*(W}F{K|L4yxeS;$p8p5*JrE5L-32WEf@H#%b`^h^>*qs zZZ%34`}#!J+;aD&PByydW8#kHqz1a;puT5*Vc0lyUFI{?7DNiAM?@WE z)q?kUYhL8%$*Ef4LfTP{-R+Qi8=+OKR!(qD3Eld@^=QXrDD`F^ayp^wZ6H|~)B;n1 zCj6=LRA{)@-KyI`YgHkgm2g^oyKEHId+X~JQ{Y55xuv>=kK*Tez#H7NZ@WhKmPg<+ie7(GA z?zArv_4dX5xeD1-@=l2)0ck#ENA$I?Oq0#kqITR}L0+U55@@8^Z>T>pG9tmLQNnv| z_;wo-5})>)jR6g`o-sr-}iqfR2(tZh03BMQKwVW8A%L=!|QWEO^fe=)ZgRfudlC zrk?aU2E`9VVZqEOZQ?!&9K)S}5@e7; zIxO%s47|!W_QHzv_4|K@nfL@UXIh50MxMx}d zk1GqDyINu0WJ>!qYKayjOE@RHbO#Z;-R$ay46N#6tLFX+L(7@`tX{Yd7<=~{Rnv9T zM{oZ)LTPPu$&lY~wtte5KDLMxwArw5<$joZ7|Y^_syBdhQmv8lsMR9E#vGgFNo;ib0Xo_1^w={PC43^?H>9Lxx3%gHE`)VPHi#a2 zK#8@OK0^%6D_;z1d;eb3UH$St;P)4?5HlrHaeaiIml_705&3#18+EeZcms%YgvlK&suqeTpnQ|H8e=uOd=NJcu)y1? zvju5_)ZOlPBoZ0}O5a?P^MufU^c4$F_6JlviE-CdVRG@qi_LSRM4+_=ex(yE5t3cF zkJ#aA9g%iV{J|P|1plPeYb83FCDUG}B-|5KQhsFC8>+dzRH-RaWXa)UIuDe@ZZT3H z1Ec+Th#Jdbjjo8Fd5PRe6vnH6q z*|?M_M}SNv$)%2gg!81g!+$>>;o8=&m23y&w>4f`azzhi21?O=(9dqn3?F^mH3v?| zV%#&GfrJ;AeC@47y4O+4Q6V_!*Sh0#G5EBu9f2UBHfT8`ujP_WyjI)03de7xaL05? zor}+Ty>kxEpxwjBH ziV#R>2Tmw_7UK;uLsMJpa(QO6aiTl7kvgq;c=6oDtWow-X8hai`sy@@rSCH%T4lSZ z3xd=2XA3BJ#;m0Yf7pZBoJl&Ue~>}WvLUEazxb!bO9#I!+MQUF~>7E?o2*`J_fF%lw;PxJZB^&wVY^1Q1$GZL1x z%q;2RhS3mf-s#nL5<^PFbNP*)iM800rm}wZ&2?aAb8q@IXWfNiLoABd7XCDO?rEW^ zn7hJS18SjCow`M7j*PqgWNl;k#~(qeH0sqR!Iz2`ORCCyl{N`VXHy}~Z9Wlh2I~GN zt3p~(2xn+SM&A3{Pz8=xEpzvMDtT`CeU{>@T^S@rG!zKhy69fq9X{U_*9fX&-BLlg zpUH!X^mkSYC`7RO2+aGwx_bAg8uAdmti!`A@~L~ON;ed(i5P*yYnxxT`AIdd6=#rx zFwQ!fpjtvN(R#8kO4Fj8cka}u6s@@YUBTWY)Qjy~qJ{RRnmC6+l=Zi7g=>{~eWErV zif*`Jz@k(P&Y_0g3PnNN8)#7^ zfj)(>JLmpzCc(9u)xK@CmJ6h$LjMjq4k~V6LCX5ICSQ36^qh|u|6LwG=ZV}6(MGmb zJ$joRXy@-F%p^?R8~|`wU(RWjNU#`8b(45m!P_;?`r8++`Uh@t=H`h;92FRH_F~C7 z#LQZbdMD;aO4IABSXM~8*tvrgM_x!^1GIBfmfUD6Orp=!T z1wH7g@$Qo$%fBR~R4et@EV?5{E(o`s^~suF%D;cf1Sy$Or`%kigKA~EV3Wl33mNCP zY03}Dzf)Q4MZDj~a3P!OYs1+a!<~Zy%Cx7orj;1k=1{<`$%w)h-)wsA4)wZ5^ic^Rc;kg4siO|DNr(V)%8vM zCxBxrBo!PNvEQyUGaY)ZExtP!!wvoX#mIQrzv}XG-}U5^8b$C_?SWIiz0@vRw4Cf{ zz&o!}MJl)^uI^p`C->-R=~H2i{wvX~x=sg@V9M79b^VfeoSJQC`)4{5*XbTd&|u5p zmD*(~YBke-9UtH^ZqOMv9QaoBQS1dQW}~ zR#SHZZoOT{SeVR;Fl`hJA-cA{XIwv%(uw&pXn)w|yS#;e_3=)*q1US4f%}F$pphN^ z>&v7FDjU_rg`^Im7d1`S5l1rgx?=Q2RC=@_924p!(SaR#@DAAx!QIn?o8sACH*8;2 z;`frq+xv+tqGp?6%k9oeIjp;PIqa2S4zz(9T+%BymT?;XyUSHQ`o8kvlNd3!;kHyQ zivyjspLjV=y=~kX(MW-eV?4&Ezp?1yII=lF&j@})TkQLF9C_%JgCsK4GI4qOthE=L z(&oZM4(G&?_)YWmRz!sHXQ|Ml1$?S#eZ7bw9szi>r(8}WP&JO(oEl#*bc$C=yH8eVm!H(> zy+K_j4-P)TR8IOZ!c|+?>)tIYfKw~LC8gJ&L>?`(OaYo_?Ym)sGJ1|Amwd?3a$+^@ zqaF4N=RCi{ZD|hJ?>l#plTl;gCXr;0Je@?Q7##inS+)yJV)1>DtHsHu8A}i9<;}sp z3*|;I511oocHez1#o3$RkPW6AoS=K&V_t>B*V0bfp!UqQeV7OM4JGv40>aOl$%Il$E9LpZg#XD zUNbE^lf-BqkYM?gL{@72u?|Ga{fu4CdNF!T;W@XJHyz!>&?`EDd-sj=)aSLT_17-| zNy|;cO3U$Gv-!?yM+rVFtTWX*8_6t|Rzmav`gOlD4KVAT_w_>LsPY)jX6kJ>TT11a zYtEW-tfjY+*|b@ra^<||Wr?d(czf!f3rKW{9lB-m025fAr|KwGFVV*O1_k$sr&Ngn z?$s4SKFo!vLVq_}WV>DLBQ;InAnRJC--}?D_P&TwOkSD+q_3lKkGyf3l@iT7C+3#9 zlcY}X9A%S0Yl6gk!KCPU@JAbO22`eByCt!+M?(?PT;zO(R}A>x_3gKPy*KR#`h$ah zn-+{*|5z$`Yj;8Op_1hJ_nCsOH~jsJw+N=<$HRlUkx99{(Y$l@JM*cVy;-7jkanr;q@~4nuVEC7?+H#-SR5g6ryt z#U{THW9rXqQzW_ar%HFXPpq4BP^Ao(c%5p=IXAvy3>#pq$ItC@qGfv+=pSc#SZZLjR7E)S-M(;?W*#Yp0pJE7dZC>vUI7`O=8gNeCmMEyJ(n=0l+$1Nt%+>usV43m^XQ2}{BXEA(V;@Z^+4 z^Lgw*v8MKi9`D;Pa>T53?f{GiZ}a2D-#j~*|%v*qyy!&^%N=87-y`Hd7jbEe~u zO-P%qfabMyek>eR6?$LDVQ8068ZkQ4L-&k7Cs#ceV2Jj6)=SH>`Sj943M_>}CP4&M zayJPyx811dqLC8Y6hJ9I-1_Wt)5f+((h!7?!~&AC+NvbJ*d0q5Rs(!OG%o>mLaiOe>H%f6K>D0sxF@0l5CkD|m=WK>jGe|LX-GKm#y0*N~CZ(lh4bG}lv8 zQCI&X$oTJ=#XpG#8Z7@;!`<+KBJ<(URs(_v5dK$J9;VC&6i0Yal&=Y81DgF2g@=B?2V+FTqLJH%TDz zA3oSuJ|HbZ0yiMe#g70s%?FePZE*hq#F6un!4|oJ_>V%9I0S!j?#blwzykPztS~X5 zhra#b2V#M)c>aK3hx|aQ$6^8)tpJb=)W{245&{y%9r9DdN`!#;Ff}0{4lF_70U<8& zh@gq{k)nC5B8J^Q0x%^X0fceV(u^<*u7}bgULXd{Q}6*Gr1A#<%M}C~fv{Bn*ue;e zfchY1oquW~A5SQBA8V{Y*jE4WYuHfy!;~oe!2iqYAAW3G+Q0a*L>~C9ogV>iAi-Dv znAp2e{MF}r(FcB~a~4h0by0DMfs`FkZl?v@P{1Ck@SAp>F5B0$2wQg$f#|383Uk{6KwL=pjj|87kN z1^8#=4*|u0UA#Zb;DZV{b%*dKDzhc tLwE!~I^(~c8vnn*dMv>I^DXg*hwL97Fw&#L2|YZQ2>}2!R^Y?g{{YuKMgjl; diff --git a/agent-skill/Scrapling-Skill/SKILL.md b/agent-skill/Scrapling-Skill/SKILL.md index 85c54d9..5c187fe 100644 --- a/agent-skill/Scrapling-Skill/SKILL.md +++ b/agent-skill/Scrapling-Skill/SKILL.md @@ -346,19 +346,25 @@ async with FetcherSession(http3=True) as session: # `FetcherSession` is context async with AsyncStealthySession(max_pages=2) as session: tasks = [] urls = ['https://example.com/page1', 'https://example.com/page2'] - + for url in urls: task = session.fetch(url) tasks.append(task) - + print(session.get_pool_stats()) # Optional - The status of the browser tabs pool (busy/free/error) results = await asyncio.gather(*tasks) print(session.get_pool_stats()) + +# Capture XHR/fetch API calls during page load +async with AsyncDynamicSession(capture_xhr=r"https://api\.example\.com/.*") as session: + page = await session.fetch('https://example.com') + for xhr in page.captured_xhr: # Each is a full Response object + print(xhr.url, xhr.status, xhr.body) ``` ## References You already had a good glimpse of what the library can do. Use the references below to dig deeper when needed -- `references/mcp-server.md` — MCP server tools and capabilities +- `references/mcp-server.md` — MCP server tools, persistent session management, and capabilities - `references/parsing` — Everything you need for parsing HTML - `references/fetching` — Everything you need to fetch websites and session persistence - `references/spiders` — Everything you need to write spiders, proxy rotation, and advanced features. It follows a Scrapy-like format diff --git a/agent-skill/Scrapling-Skill/references/fetching/choosing.md b/agent-skill/Scrapling-Skill/references/fetching/choosing.md index 974b566..10ec7e8 100644 --- a/agent-skill/Scrapling-Skill/references/fetching/choosing.md +++ b/agent-skill/Scrapling-Skill/references/fetching/choosing.md @@ -71,6 +71,7 @@ The `Response` object is the same as the [Selector](parsing/main_classes.md#sele >>> page.body # Raw response body as bytes >>> page.encoding # Response encoding >>> page.meta # Response metadata dictionary (e.g., proxy used). Mainly helpful with the spiders system. +>>> page.captured_xhr # List of captured XHR/fetch responses (when capture_xhr is enabled on a browser session) ``` All fetchers return the `Response` object. diff --git a/agent-skill/Scrapling-Skill/references/fetching/dynamic.md b/agent-skill/Scrapling-Skill/references/fetching/dynamic.md index 1a4c96d..a5fa235 100644 --- a/agent-skill/Scrapling-Skill/references/fetching/dynamic.md +++ b/agent-skill/Scrapling-Skill/references/fetching/dynamic.md @@ -79,6 +79,7 @@ All arguments for `DynamicFetcher` and its session classes: | proxy_rotator | A `ProxyRotator` instance for automatic proxy rotation. Cannot be combined with `proxy`. | ✔️ | | retries | Number of retry attempts for failed requests. Defaults to 3. | ✔️ | | retry_delay | Seconds to wait between retry attempts. Defaults to 1. | ✔️ | +| capture_xhr | Pass a regex URL pattern string to capture XHR/fetch requests matching it during page load. Captured responses are available via `response.captured_xhr`. Defaults to `None` (disabled). | ✔️ | In session classes, all these arguments can be set globally for the session. Still, you can configure each request individually by passing some of the arguments here that can be configured on the browser tab level like: `google_search`, `timeout`, `wait`, `page_action`, `extra_headers`, `disable_resources`, `wait_selector`, `wait_selector_state`, `network_idle`, `load_dom`, `blocked_domains`, `proxy`, and `selector_config`. @@ -201,6 +202,24 @@ The states the fetcher can wait for can be any of the following ([source](https: - `visible`: wait for an element to have a non-empty bounding box and no `visibility:hidden`. Note that an element without any content or with `display:none` has an empty bounding box and is not considered visible. - `hidden`: wait for an element to be either detached from the DOM, or have an empty bounding box, or `visibility:hidden`. This is opposite to the `'visible'` option. +### Capturing XHR/Fetch Requests + +Many SPAs load data through background API calls (XHR/fetch). You can capture these requests by passing a regex URL pattern to `capture_xhr` at the session level: + +```python +from scrapling.fetchers import DynamicSession + +with DynamicSession(capture_xhr=r"https://api\.example\.com/.*", headless=True) as session: + page = session.fetch('https://example.com') + + # Access captured XHR responses + for xhr in page.captured_xhr: + print(xhr.url, xhr.status) + print(xhr.body) # Raw response body as bytes +``` + +Each item in `captured_xhr` is a full `Response` object with the same properties (`.url`, `.status`, `.headers`, `.body`, etc.). When `capture_xhr` is not set or is `None`, `captured_xhr` is an empty list. + ### Some Stealth Features ```python diff --git a/agent-skill/Scrapling-Skill/references/fetching/stealthy.md b/agent-skill/Scrapling-Skill/references/fetching/stealthy.md index 5708d61..4fc6e50 100644 --- a/agent-skill/Scrapling-Skill/references/fetching/stealthy.md +++ b/agent-skill/Scrapling-Skill/references/fetching/stealthy.md @@ -61,6 +61,7 @@ Scrapling provides many options with this fetcher and its session classes. Befor | proxy_rotator | A `ProxyRotator` instance for automatic proxy rotation. Cannot be combined with `proxy`. | ✔️ | | retries | Number of retry attempts for failed requests. Defaults to 3. | ✔️ | | retry_delay | Seconds to wait between retry attempts. Defaults to 1. | ✔️ | +| capture_xhr | Pass a regex URL pattern string to capture XHR/fetch requests matching it during page load. Captured responses are available via `response.captured_xhr`. Defaults to `None` (disabled). | ✔️ | In session classes, all these arguments can be set globally for the session. Still, you can configure each request individually by passing some of the arguments here that can be configured on the browser tab level like: `google_search`, `timeout`, `wait`, `page_action`, `extra_headers`, `disable_resources`, `wait_selector`, `wait_selector_state`, `network_idle`, `load_dom`, `solve_cloudflare`, `blocked_domains`, `proxy`, and `selector_config`. diff --git a/agent-skill/Scrapling-Skill/references/mcp-server.md b/agent-skill/Scrapling-Skill/references/mcp-server.md index fbe1b7d..1e18a04 100644 --- a/agent-skill/Scrapling-Skill/references/mcp-server.md +++ b/agent-skill/Scrapling-Skill/references/mcp-server.md @@ -1,8 +1,8 @@ # Scrapling MCP Server -The Scrapling MCP server exposes six web scraping tools over the MCP protocol. It supports CSS-selector-based content narrowing (reducing tokens by extracting only relevant elements before returning results) and three levels of scraping capability: plain HTTP, browser-rendered, and stealth (anti-bot bypass). +The Scrapling MCP server exposes nine web scraping tools over the MCP protocol. It supports CSS-selector-based content narrowing (reducing tokens by extracting only relevant elements before returning results), three levels of scraping capability (plain HTTP, browser-rendered, and stealth/anti-bot bypass), and persistent browser session management. -All tools return a `ResponseModel` with fields: `status` (int), `content` (list of strings), `url` (str). +All scraping tools return a `ResponseModel` with fields: `status` (int), `content` (list of strings), `url` (str). ## Tools @@ -66,10 +66,11 @@ Opens a Chromium browser via Playwright to render JavaScript. Suitable for dynam | `cookies` | list or null | null | Playwright-format cookies | | `timezone_id` | str or null | null | Browser timezone, e.g. `"America/New_York"` | | `locale` | str or null | null | Browser locale, e.g. `"en-GB"` | +| `session_id` | str or null | null | Reuse a persistent session from `open_session` instead of creating a new browser | ### `bulk_fetch` -- Browser fetch (multiple URLs) -Concurrent browser version of `fetch`. Same parameters except `url` is replaced by `urls` (list of strings). Each URL opens in a separate browser tab. Returns a list of `ResponseModel`. +Concurrent browser version of `fetch`. Same parameters (including `session_id`) except `url` is replaced by `urls` (list of strings). Each URL opens in a separate browser tab. Returns a list of `ResponseModel`. ### `stealthy_fetch` -- Stealth browser fetch (single URL) @@ -84,12 +85,51 @@ Anti-bot bypass fetcher with fingerprint spoofing. Use this for sites with Cloud | `block_webrtc` | bool | false | Force WebRTC to respect proxy settings (prevents IP leak) | | `allow_webgl` | bool | true | Keep WebGL enabled (disabling is detectable by WAFs) | | `additional_args` | dict or null | null | Extra Playwright context args (overrides Scrapling defaults) | +| `session_id` | str or null | null | Reuse a persistent stealthy session from `open_session` | All parameters from `fetch` are also accepted. ### `bulk_stealthy_fetch` -- Stealth browser fetch (multiple URLs) -Concurrent stealth version. Same parameters as `stealthy_fetch` except `url` is replaced by `urls` (list of strings). Returns a list of `ResponseModel`. +Concurrent stealth version. Same parameters (including `session_id`) as `stealthy_fetch` except `url` is replaced by `urls` (list of strings). Returns a list of `ResponseModel`. + +### `open_session` -- Create a persistent browser session + +Opens a browser session that stays alive across multiple fetch calls, avoiding the overhead of launching a new browser each time. Returns a `SessionCreatedModel` with `session_id`, `session_type`, `created_at`, `is_alive`, and `message`. + +**Key parameters:** + +| Parameter | Type | Default | Description | +|--------------------|-----------------------------|--------------|---------------------------------------------------------------------| +| `session_type` | `"dynamic"` / `"stealthy"` | required | Type of browser session to create | +| `headless` | bool | true | Run browser hidden or visible | +| `max_pages` | int | 5 | Max concurrent browser tabs (1-50) | +| `proxy` | str or dict or null | null | Proxy for all requests in this session | +| `timeout` | number | 30000 | Default timeout in ms | +| `solve_cloudflare` | bool | false | (Stealthy only) Auto-solve Cloudflare challenges | +| `hide_canvas` | bool | false | (Stealthy only) Canvas fingerprint noise | +| `block_webrtc` | bool | false | (Stealthy only) Block WebRTC IP leak | +| `allow_webgl` | bool | true | (Stealthy only) Keep WebGL enabled | + +Plus all other browser session parameters (`google_search`, `real_chrome`, `cdp_url`, `locale`, `timezone_id`, `useragent`, `extra_headers`, `cookies`, `disable_resources`, `network_idle`, `wait_selector`, `wait_selector_state`). + +A dynamic session can only be used with `fetch`/`bulk_fetch`. A stealthy session can only be used with `stealthy_fetch`/`bulk_stealthy_fetch`. + +### `close_session` -- Close a persistent browser session + +Closes a session and frees its browser resources. Always close sessions when done. + +| Parameter | Type | Default | Description | +|--------------|------|----------|----------------------------------| +| `session_id` | str | required | Session ID from `open_session` | + +Returns a `SessionClosedModel` with `session_id` and `message`. + +### `list_sessions` -- List active sessions + +Returns a list of `SessionInfo` objects, each with `session_id`, `session_type`, `created_at`, and `is_alive`. + +No parameters. ## Tool selection guide @@ -101,8 +141,9 @@ Concurrent stealth version. Same parameters as `stealthy_fetch` except `url` is | Multiple JS-rendered pages | `bulk_fetch` | | Cloudflare or strong anti-bot protection | `stealthy_fetch` (with `solve_cloudflare=true` for Turnstile) | | Multiple protected pages | `bulk_stealthy_fetch` | +| Multiple pages from the same site | `open_session` + `fetch`/`stealthy_fetch` with `session_id` | -Start with `get` (fastest, lowest resource cost). Escalate to `fetch` if content requires JS rendering. Escalate to `stealthy_fetch` only if blocked. +Start with `get` (fastest, lowest resource cost). Escalate to `fetch` if content requires JS rendering. Escalate to `stealthy_fetch` only if blocked. For multiple pages from the same site, use a persistent session to avoid browser launch overhead. ## Content extraction tips From 375951bd49c9c10e7241e313a76235b43285522f Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Mon, 30 Mar 2026 02:31:14 +0200 Subject: [PATCH 25/33] feat(mcp): Protect from Prompt Injection by removing hidden content Solves #214 as well --- scrapling/core/shell.py | 34 +++++++++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/scrapling/core/shell.py b/scrapling/core/shell.py index 80caa99..bed1c84 100644 --- a/scrapling/core/shell.py +++ b/scrapling/core/shell.py @@ -2,7 +2,7 @@ from sys import stderr from copy import deepcopy from functools import wraps -from re import sub as re_sub +from re import sub as re_sub, compile as re_compile from collections import namedtuple from shlex import split as shlex_split from inspect import signature, Parameter @@ -21,6 +21,7 @@ from logging import ( getLevelName, ) +from lxml.etree import XPath from orjson import loads as json_loads, JSONDecodeError from ._shell_signatures import Signatures_map @@ -67,6 +68,19 @@ Request = namedtuple( ], ) +# Precompiled for the prompt injection sanitizer +_HIDDEN_XPATH = XPath( + './/*[contains(@style,"display:none") or contains(@style,"display: none")' + ' or contains(@style,"visibility:hidden") or contains(@style,"visibility: hidden")' + ' or contains(@style,"opacity:0") or contains(@style,"opacity: 0")' + ' or contains(@style,"font-size:0") or contains(@style,"font-size: 0")' + ' or contains(@style,"height:0") or contains(@style,"height: 0")' + ' or contains(@style,"width:0") or contains(@style,"width: 0")]' + " | .//*[@aria-hidden='true']" + " | .//template" +) +_ZWC_PATTERN = re_compile(r"[\u200b\u200c\u200d\ufeff\u2060\u180e]") + # Suppress exit on error to handle parsing errors gracefully class NoExitArgumentParser(ArgumentParser): # pragma: no cover @@ -580,6 +594,23 @@ class Convertor: element.drop_tree() return Selector(root=clean_root, url=page.url) + @classmethod + def _sanitize_for_ai(cls, page: Selector) -> Selector: + """Strip hidden content that could be used for prompt injection. + + Removes CSS-hidden elements, aria-hidden elements,