From 35032d6f5fef4c8989262bf7497ce1f38d1af885 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Wed, 22 Apr 2026 15:31:18 +0200 Subject: [PATCH 01/20] fix: solving a bug with using `configure` on Fetcher --- scrapling/fetchers/requests.py | 55 ++++++++++++++++++++++++++++------ 1 file changed, 46 insertions(+), 9 deletions(-) diff --git a/scrapling/fetchers/requests.py b/scrapling/fetchers/requests.py index b559cd6..ec629a4 100644 --- a/scrapling/fetchers/requests.py +++ b/scrapling/fetchers/requests.py @@ -1,28 +1,65 @@ +from scrapling.core._types import Any, Awaitable, Unpack +from scrapling.engines._browsers._types import DataRequestParams, GetRequestParams from scrapling.engines.static import ( FetcherSession, FetcherClient as _FetcherClient, AsyncFetcherClient as _AsyncFetcherClient, ) -from scrapling.engines.toolbelt.custom import BaseFetcher +from scrapling.engines.toolbelt.custom import BaseFetcher, Response + +__all__ = ["Fetcher", "AsyncFetcher", "FetcherSession"] __FetcherClientInstance__ = _FetcherClient() __AsyncFetcherClientInstance__ = _AsyncFetcherClient() +def _merge_selector_config(cls: type[BaseFetcher], kwargs: Any) -> Any: + """Merge class-level parser arguments into per-request ``selector_config``. + + Values from ``Fetcher.configure(...)`` act as the base; any explicit + ``selector_config`` passed on the call overrides them. + """ + selector_config = kwargs.get("selector_config") or {} + kwargs["selector_config"] = {**cls._generate_parser_arguments(), **selector_config} + return kwargs + + class Fetcher(BaseFetcher): """A basic `Fetcher` class type that can only do basic GET, POST, PUT, and DELETE HTTP requests based on `curl_cffi`.""" - get = __FetcherClientInstance__.get - post = __FetcherClientInstance__.post - put = __FetcherClientInstance__.put - delete = __FetcherClientInstance__.delete + @classmethod + def get(cls, url: str, **kwargs: Unpack[GetRequestParams]) -> Response: + return __FetcherClientInstance__.get(url, **_merge_selector_config(cls, kwargs)) + + @classmethod + def post(cls, url: str, **kwargs: Unpack[DataRequestParams]) -> Response: + return __FetcherClientInstance__.post(url, **_merge_selector_config(cls, kwargs)) + + @classmethod + def put(cls, url: str, **kwargs: Unpack[DataRequestParams]) -> Response: + return __FetcherClientInstance__.put(url, **_merge_selector_config(cls, kwargs)) + + @classmethod + def delete(cls, url: str, **kwargs: Unpack[DataRequestParams]) -> Response: + return __FetcherClientInstance__.delete(url, **_merge_selector_config(cls, kwargs)) class AsyncFetcher(BaseFetcher): """A basic `Fetcher` class type that can only do basic GET, POST, PUT, and DELETE HTTP requests based on `curl_cffi`.""" - get = __AsyncFetcherClientInstance__.get - post = __AsyncFetcherClientInstance__.post - put = __AsyncFetcherClientInstance__.put - delete = __AsyncFetcherClientInstance__.delete + @classmethod + def get(cls, url: str, **kwargs: Unpack[GetRequestParams]) -> Awaitable[Response]: + return __AsyncFetcherClientInstance__.get(url, **_merge_selector_config(cls, kwargs)) + + @classmethod + def post(cls, url: str, **kwargs: Unpack[DataRequestParams]) -> Awaitable[Response]: + return __AsyncFetcherClientInstance__.post(url, **_merge_selector_config(cls, kwargs)) + + @classmethod + def put(cls, url: str, **kwargs: Unpack[DataRequestParams]) -> Awaitable[Response]: + return __AsyncFetcherClientInstance__.put(url, **_merge_selector_config(cls, kwargs)) + + @classmethod + def delete(cls, url: str, **kwargs: Unpack[DataRequestParams]) -> Awaitable[Response]: + return __AsyncFetcherClientInstance__.delete(url, **_merge_selector_config(cls, kwargs)) From b626b4d585c17f4cc7ed0586ffc3a7815a8f4107 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Wed, 22 Apr 2026 15:35:15 +0200 Subject: [PATCH 02/20] test: adding new tests for the `configure` function --- tests/fetchers/async/test_requests.py | 36 +++++++++++++++++++++++++++ tests/fetchers/sync/test_requests.py | 32 ++++++++++++++++++++++++ 2 files changed, 68 insertions(+) diff --git a/tests/fetchers/async/test_requests.py b/tests/fetchers/async/test_requests.py index 51417cd..f0bccd1 100644 --- a/tests/fetchers/async/test_requests.py +++ b/tests/fetchers/async/test_requests.py @@ -6,6 +6,17 @@ from scrapling.fetchers import AsyncFetcher AsyncFetcher.adaptive = True +@pytest.fixture +def _reset_async_fetcher_config(): + """Snapshot and restore the mutable class-level parser config around a test.""" + snapshot = {k: getattr(AsyncFetcher, k) for k in AsyncFetcher.parser_keywords} + try: + yield + finally: + for k, v in snapshot.items(): + setattr(AsyncFetcher, k, v) + + @pytest_httpbin.use_class_based_httpbin @pytest.mark.asyncio class TestAsyncFetcher: @@ -124,3 +135,28 @@ class TestAsyncFetcher: timeout=None, ) ).status == 200 + + async def test_configure_propagates_to_response( + self, fetcher, urls, _reset_async_fetcher_config + ): + """`AsyncFetcher.configure()` must reach the Response's Selector on the HTTP path.""" + AsyncFetcher.configure(adaptive=False, adaptive_domain="") + baseline = await fetcher.get(urls["html_url"]) + assert baseline._storage is None + + AsyncFetcher.configure(adaptive=True, adaptive_domain="configured.test") + configured = await fetcher.get(urls["html_url"]) + assert configured._storage is not None + assert configured.url == "configured.test" + + async def test_selector_config_overrides_configure( + self, fetcher, urls, _reset_async_fetcher_config + ): + """A per-request ``selector_config`` overrides the class-level configure().""" + AsyncFetcher.configure(adaptive=True, adaptive_domain="from-configure.test") + response = await fetcher.get( + urls["html_url"], + selector_config={"adaptive_domain": "from-request.test"}, + ) + assert response._storage is not None + assert response.url == "from-request.test" diff --git a/tests/fetchers/sync/test_requests.py b/tests/fetchers/sync/test_requests.py index 225932e..20682a1 100644 --- a/tests/fetchers/sync/test_requests.py +++ b/tests/fetchers/sync/test_requests.py @@ -6,6 +6,17 @@ from scrapling import Fetcher Fetcher.adaptive = True +@pytest.fixture +def _reset_fetcher_config(): + """Snapshot and restore the mutable class-level parser config around a test.""" + snapshot = {k: getattr(Fetcher, k) for k in Fetcher.parser_keywords} + try: + yield + finally: + for k, v in snapshot.items(): + setattr(Fetcher, k, v) + + @pytest_httpbin.use_class_based_httpbin class TestFetcher: @pytest.fixture(scope="class") @@ -119,3 +130,24 @@ class TestFetcher: ).status == 200 ) + + def test_configure_propagates_to_response(self, fetcher, _reset_fetcher_config): + """`Fetcher.configure()` must reach the Response's Selector on the HTTP path.""" + Fetcher.configure(adaptive=False, adaptive_domain="") + baseline = fetcher.get(self.html_url) + assert baseline._storage is None + + Fetcher.configure(adaptive=True, adaptive_domain="configured.test") + configured = fetcher.get(self.html_url) + assert configured._storage is not None + assert configured.url == "configured.test" + + def test_selector_config_overrides_configure(self, fetcher, _reset_fetcher_config): + """A per-request ``selector_config`` overrides the class-level configure().""" + Fetcher.configure(adaptive=True, adaptive_domain="from-configure.test") + response = fetcher.get( + self.html_url, + selector_config={"adaptive_domain": "from-request.test"}, + ) + assert response._storage is not None + assert response.url == "from-request.test" From 9af644ef84808466390544613c84cd01a3dbcd41 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Wed, 22 Apr 2026 16:52:59 +0200 Subject: [PATCH 03/20] docs: improving the code copy-paste experience and use less tokens for the agent skill --- .../references/fetching/choosing.md | 40 ++--- .../references/fetching/dynamic.md | 2 +- .../references/fetching/static.md | 146 ++++++++--------- .../references/fetching/stealthy.md | 2 +- .../references/parsing/adaptive.md | 41 +++-- .../references/parsing/main_classes.md | 24 +-- .../references/parsing/selection.md | 4 +- docs/cli/interactive-shell.md | 119 +++++++------- docs/development/scrapling_custom_types.md | 11 +- docs/fetching/choosing.md | 40 ++--- docs/fetching/dynamic.md | 2 +- docs/fetching/static.md | 148 +++++++++--------- docs/fetching/stealthy.md | 2 +- docs/overview.md | 58 ++++--- docs/parsing/adaptive.md | 41 +++-- docs/parsing/main_classes.md | 24 +-- docs/parsing/selection.md | 4 +- 17 files changed, 345 insertions(+), 363 deletions(-) diff --git a/agent-skill/Scrapling-Skill/references/fetching/choosing.md b/agent-skill/Scrapling-Skill/references/fetching/choosing.md index 10ec7e8..1ec0ef8 100644 --- a/agent-skill/Scrapling-Skill/references/fetching/choosing.md +++ b/agent-skill/Scrapling-Skill/references/fetching/choosing.md @@ -27,23 +27,23 @@ The following table compares them and can be quickly used for guidance. ## Parser configuration in all fetchers All fetchers share the same import method, as you will see in the upcoming pages ```python ->>> from scrapling.fetchers import Fetcher, AsyncFetcher, StealthyFetcher, DynamicFetcher +from scrapling.fetchers import Fetcher, AsyncFetcher, StealthyFetcher, DynamicFetcher ``` Then you use it right away without initializing like this, and it will use the default parser settings: ```python ->>> page = StealthyFetcher.fetch('https://example.com') +page = StealthyFetcher.fetch('https://example.com') ``` If you want to configure the parser ([Selector class](parsing/main_classes.md#selector)) that will be used on the response before returning it for you, then do this first: ```python ->>> from scrapling.fetchers import Fetcher ->>> Fetcher.configure(adaptive=True, keep_comments=False, keep_cdata=False) # and the rest +from scrapling.fetchers import Fetcher +Fetcher.configure(adaptive=True, keep_comments=False, keep_cdata=False) # and the rest ``` or ```python ->>> from scrapling.fetchers import Fetcher ->>> Fetcher.adaptive=True ->>> Fetcher.keep_comments=False ->>> Fetcher.keep_cdata=False # and the rest +from scrapling.fetchers import Fetcher +Fetcher.adaptive=True +Fetcher.keep_comments=False +Fetcher.keep_cdata=False # and the rest ``` Then, continue your code as usual. @@ -59,19 +59,19 @@ If your use case requires a different configuration for each request/fetch, you ## Response Object The `Response` object is the same as the [Selector](parsing/main_classes.md#selector) class, but it has additional details about the response, like response headers, status, cookies, etc., as shown below: ```python ->>> from scrapling.fetchers import Fetcher ->>> page = Fetcher.get('https://example.com') +from scrapling.fetchers import Fetcher +page = Fetcher.get('https://example.com') ->>> page.status # HTTP status code ->>> page.reason # Status message ->>> page.cookies # Response cookies as a dictionary ->>> page.headers # Response headers ->>> page.request_headers # Request headers ->>> page.history # Response history of redirections, if any ->>> page.body # Raw response body as bytes ->>> page.encoding # Response encoding ->>> page.meta # Response metadata dictionary (e.g., proxy used). Mainly helpful with the spiders system. ->>> page.captured_xhr # List of captured XHR/fetch responses (when capture_xhr is enabled on a browser session) +page.status # HTTP status code +page.reason # Status message +page.cookies # Response cookies as a dictionary +page.headers # Response headers +page.request_headers # Request headers +page.history # Response history of redirections, if any +page.body # Raw response body as bytes +page.encoding # Response encoding +page.meta # Response metadata dictionary (e.g., proxy used). Mainly helpful with the spiders system. +page.captured_xhr # List of captured XHR/fetch responses (when capture_xhr is enabled on a browser session) ``` All fetchers return the `Response` object. diff --git a/agent-skill/Scrapling-Skill/references/fetching/dynamic.md b/agent-skill/Scrapling-Skill/references/fetching/dynamic.md index a831d7c..5f5b0ce 100644 --- a/agent-skill/Scrapling-Skill/references/fetching/dynamic.md +++ b/agent-skill/Scrapling-Skill/references/fetching/dynamic.md @@ -8,7 +8,7 @@ As we will explain later, to automate the page, you need some knowledge of [Play You have one primary way to import this Fetcher, which is the same for all fetchers. ```python ->>> from scrapling.fetchers import DynamicFetcher +from scrapling.fetchers import DynamicFetcher ``` Check out how to configure the parsing options [here](choosing.md#parser-configuration-in-all-fetchers) diff --git a/agent-skill/Scrapling-Skill/references/fetching/static.md b/agent-skill/Scrapling-Skill/references/fetching/static.md index 4115483..e24f6ce 100644 --- a/agent-skill/Scrapling-Skill/references/fetching/static.md +++ b/agent-skill/Scrapling-Skill/references/fetching/static.md @@ -6,7 +6,7 @@ The `Fetcher` class provides rapid and lightweight HTTP requests using the high- Import the Fetcher (same import pattern for all fetchers): ```python ->>> from scrapling.fetchers import Fetcher +from scrapling.fetchers import Fetcher ``` Check out how to configure the parsing options [here](choosing.md#parser-configuration-in-all-fetchers) @@ -47,41 +47,41 @@ Examples are the best way to explain this: > Hence: `OPTIONS` and `HEAD` methods are not supported. #### GET ```python ->>> from scrapling.fetchers import Fetcher ->>> # Basic GET ->>> page = Fetcher.get('https://example.com') ->>> page = Fetcher.get('https://scrapling.requestcatcher.com/get', stealthy_headers=True) ->>> page = Fetcher.get('https://scrapling.requestcatcher.com/get', proxy='http://username:password@localhost:8030') ->>> # With parameters ->>> page = Fetcher.get('https://example.com/search', params={'q': 'query'}) ->>> ->>> # With headers ->>> page = Fetcher.get('https://example.com', headers={'User-Agent': 'Custom/1.0'}) ->>> # Basic HTTP authentication ->>> page = Fetcher.get("https://example.com", auth=("my_user", "password123")) ->>> # Browser impersonation ->>> page = Fetcher.get('https://example.com', impersonate='chrome') ->>> # HTTP/3 support ->>> page = Fetcher.get('https://example.com', http3=True) +from scrapling.fetchers import Fetcher +# Basic GET +page = Fetcher.get('https://example.com') +page = Fetcher.get('https://scrapling.requestcatcher.com/get', stealthy_headers=True) +page = Fetcher.get('https://scrapling.requestcatcher.com/get', proxy='http://username:password@localhost:8030') +# With parameters +page = Fetcher.get('https://example.com/search', params={'q': 'query'}) + +# With headers +page = Fetcher.get('https://example.com', headers={'User-Agent': 'Custom/1.0'}) +# Basic HTTP authentication +page = Fetcher.get("https://example.com", auth=("my_user", "password123")) +# Browser impersonation +page = Fetcher.get('https://example.com', impersonate='chrome') +# HTTP/3 support +page = Fetcher.get('https://example.com', http3=True) ``` And for asynchronous requests, it's a small adjustment ```python ->>> from scrapling.fetchers import AsyncFetcher ->>> # Basic GET ->>> page = await AsyncFetcher.get('https://example.com') ->>> page = await AsyncFetcher.get('https://scrapling.requestcatcher.com/get', stealthy_headers=True) ->>> page = await AsyncFetcher.get('https://scrapling.requestcatcher.com/get', proxy='http://username:password@localhost:8030') ->>> # With parameters ->>> page = await AsyncFetcher.get('https://example.com/search', params={'q': 'query'}) ->>> ->>> # With headers ->>> page = await AsyncFetcher.get('https://example.com', headers={'User-Agent': 'Custom/1.0'}) ->>> # Basic HTTP authentication ->>> page = await AsyncFetcher.get("https://example.com", auth=("my_user", "password123")) ->>> # Browser impersonation ->>> page = await AsyncFetcher.get('https://example.com', impersonate='chrome110') ->>> # HTTP/3 support ->>> page = await AsyncFetcher.get('https://example.com', http3=True) +from scrapling.fetchers import AsyncFetcher +# Basic GET +page = await AsyncFetcher.get('https://example.com') +page = await AsyncFetcher.get('https://scrapling.requestcatcher.com/get', stealthy_headers=True) +page = await AsyncFetcher.get('https://scrapling.requestcatcher.com/get', proxy='http://username:password@localhost:8030') +# With parameters +page = await AsyncFetcher.get('https://example.com/search', params={'q': 'query'}) + +# With headers +page = await AsyncFetcher.get('https://example.com', headers={'User-Agent': 'Custom/1.0'}) +# Basic HTTP authentication +page = await AsyncFetcher.get("https://example.com", auth=("my_user", "password123")) +# Browser impersonation +page = await AsyncFetcher.get('https://example.com', impersonate='chrome110') +# HTTP/3 support +page = await AsyncFetcher.get('https://example.com', http3=True) ``` The `page` object in all cases is a [Response](choosing.md#response-object) object, which is a [Selector](parsing/main_classes.md#selector), so you can use it directly ```python @@ -102,62 +102,62 @@ The `page` object in all cases is a [Response](choosing.md#response-object) obje ``` #### POST ```python ->>> from scrapling.fetchers import Fetcher ->>> # Basic POST ->>> page = Fetcher.post('https://scrapling.requestcatcher.com/post', data={'key': 'value'}, params={'q': 'query'}) ->>> page = Fetcher.post('https://scrapling.requestcatcher.com/post', data={'key': 'value'}, stealthy_headers=True) ->>> page = Fetcher.post('https://scrapling.requestcatcher.com/post', data={'key': 'value'}, proxy='http://username:password@localhost:8030', impersonate="chrome") ->>> # Another example of form-encoded data ->>> page = Fetcher.post('https://example.com/submit', data={'username': 'user', 'password': 'pass'}, http3=True) ->>> # JSON data ->>> page = Fetcher.post('https://example.com/api', json={'key': 'value'}) +from scrapling.fetchers import Fetcher +# Basic POST +page = Fetcher.post('https://scrapling.requestcatcher.com/post', data={'key': 'value'}, params={'q': 'query'}) +page = Fetcher.post('https://scrapling.requestcatcher.com/post', data={'key': 'value'}, stealthy_headers=True) +page = Fetcher.post('https://scrapling.requestcatcher.com/post', data={'key': 'value'}, proxy='http://username:password@localhost:8030', impersonate="chrome") +# Another example of form-encoded data +page = Fetcher.post('https://example.com/submit', data={'username': 'user', 'password': 'pass'}, http3=True) +# JSON data +page = Fetcher.post('https://example.com/api', json={'key': 'value'}) ``` And for asynchronous requests, it's a small adjustment ```python ->>> from scrapling.fetchers import AsyncFetcher ->>> # Basic POST ->>> page = await AsyncFetcher.post('https://scrapling.requestcatcher.com/post', data={'key': 'value'}) ->>> page = await AsyncFetcher.post('https://scrapling.requestcatcher.com/post', data={'key': 'value'}, stealthy_headers=True) ->>> page = await AsyncFetcher.post('https://scrapling.requestcatcher.com/post', data={'key': 'value'}, proxy='http://username:password@localhost:8030', impersonate="chrome") ->>> # Another example of form-encoded data ->>> page = await AsyncFetcher.post('https://example.com/submit', data={'username': 'user', 'password': 'pass'}, http3=True) ->>> # JSON data ->>> page = await AsyncFetcher.post('https://example.com/api', json={'key': 'value'}) +from scrapling.fetchers import AsyncFetcher +# Basic POST +page = await AsyncFetcher.post('https://scrapling.requestcatcher.com/post', data={'key': 'value'}) +page = await AsyncFetcher.post('https://scrapling.requestcatcher.com/post', data={'key': 'value'}, stealthy_headers=True) +page = await AsyncFetcher.post('https://scrapling.requestcatcher.com/post', data={'key': 'value'}, proxy='http://username:password@localhost:8030', impersonate="chrome") +# Another example of form-encoded data +page = await AsyncFetcher.post('https://example.com/submit', data={'username': 'user', 'password': 'pass'}, http3=True) +# JSON data +page = await AsyncFetcher.post('https://example.com/api', json={'key': 'value'}) ``` #### PUT ```python ->>> from scrapling.fetchers import Fetcher ->>> # Basic PUT ->>> page = Fetcher.put('https://example.com/update', data={'status': 'updated'}) ->>> page = Fetcher.put('https://example.com/update', data={'status': 'updated'}, stealthy_headers=True, impersonate="chrome") ->>> page = Fetcher.put('https://example.com/update', data={'status': 'updated'}, proxy='http://username:password@localhost:8030') ->>> # Another example of form-encoded data ->>> page = Fetcher.put("https://scrapling.requestcatcher.com/put", data={'key': ['value1', 'value2']}) +from scrapling.fetchers import Fetcher +# Basic PUT +page = Fetcher.put('https://example.com/update', data={'status': 'updated'}) +page = Fetcher.put('https://example.com/update', data={'status': 'updated'}, stealthy_headers=True, impersonate="chrome") +page = Fetcher.put('https://example.com/update', data={'status': 'updated'}, proxy='http://username:password@localhost:8030') +# Another example of form-encoded data +page = Fetcher.put("https://scrapling.requestcatcher.com/put", data={'key': ['value1', 'value2']}) ``` And for asynchronous requests, it's a small adjustment ```python ->>> from scrapling.fetchers import AsyncFetcher ->>> # Basic PUT ->>> page = await AsyncFetcher.put('https://example.com/update', data={'status': 'updated'}) ->>> page = await AsyncFetcher.put('https://example.com/update', data={'status': 'updated'}, stealthy_headers=True, impersonate="chrome") ->>> page = await AsyncFetcher.put('https://example.com/update', data={'status': 'updated'}, proxy='http://username:password@localhost:8030') ->>> # Another example of form-encoded data ->>> page = await AsyncFetcher.put("https://scrapling.requestcatcher.com/put", data={'key': ['value1', 'value2']}) +from scrapling.fetchers import AsyncFetcher +# Basic PUT +page = await AsyncFetcher.put('https://example.com/update', data={'status': 'updated'}) +page = await AsyncFetcher.put('https://example.com/update', data={'status': 'updated'}, stealthy_headers=True, impersonate="chrome") +page = await AsyncFetcher.put('https://example.com/update', data={'status': 'updated'}, proxy='http://username:password@localhost:8030') +# Another example of form-encoded data +page = await AsyncFetcher.put("https://scrapling.requestcatcher.com/put", data={'key': ['value1', 'value2']}) ``` #### DELETE ```python ->>> from scrapling.fetchers import Fetcher ->>> page = Fetcher.delete('https://example.com/resource/123') ->>> page = Fetcher.delete('https://example.com/resource/123', stealthy_headers=True, impersonate="chrome") ->>> page = Fetcher.delete('https://example.com/resource/123', proxy='http://username:password@localhost:8030') +from scrapling.fetchers import Fetcher +page = Fetcher.delete('https://example.com/resource/123') +page = Fetcher.delete('https://example.com/resource/123', stealthy_headers=True, impersonate="chrome") +page = Fetcher.delete('https://example.com/resource/123', proxy='http://username:password@localhost:8030') ``` And for asynchronous requests, it's a small adjustment ```python ->>> from scrapling.fetchers import AsyncFetcher ->>> page = await AsyncFetcher.delete('https://example.com/resource/123') ->>> page = await AsyncFetcher.delete('https://example.com/resource/123', stealthy_headers=True, impersonate="chrome") ->>> page = await AsyncFetcher.delete('https://example.com/resource/123', proxy='http://username:password@localhost:8030') +from scrapling.fetchers import AsyncFetcher +page = await AsyncFetcher.delete('https://example.com/resource/123') +page = await AsyncFetcher.delete('https://example.com/resource/123', stealthy_headers=True, impersonate="chrome") +page = await AsyncFetcher.delete('https://example.com/resource/123', proxy='http://username:password@localhost:8030') ``` ## Session Management diff --git a/agent-skill/Scrapling-Skill/references/fetching/stealthy.md b/agent-skill/Scrapling-Skill/references/fetching/stealthy.md index 8939c8d..ff4b2bf 100644 --- a/agent-skill/Scrapling-Skill/references/fetching/stealthy.md +++ b/agent-skill/Scrapling-Skill/references/fetching/stealthy.md @@ -6,7 +6,7 @@ You have one primary way to import this Fetcher, which is the same for all fetchers. ```python ->>> from scrapling.fetchers import StealthyFetcher +from scrapling.fetchers import StealthyFetcher ``` Check out how to configure the parsing options [here](choosing.md#parser-configuration-in-all-fetchers) diff --git a/agent-skill/Scrapling-Skill/references/parsing/adaptive.md b/agent-skill/Scrapling-Skill/references/parsing/adaptive.md index 61df382..25e2b79 100644 --- a/agent-skill/Scrapling-Skill/references/parsing/adaptive.md +++ b/agent-skill/Scrapling-Skill/references/parsing/adaptive.md @@ -68,22 +68,21 @@ To extract the Questions button from the old design, a selector like `#hmenus > Testing the same selector in both versions: ```python ->> from scrapling import Fetcher ->> selector = '#hmenus > div:nth-child(1) > ul > li:nth-child(1) > a' ->> old_url = "https://web.archive.org/web/20100102003420/http://stackoverflow.com/" ->> new_url = "https://stackoverflow.com/" ->> Fetcher.configure(adaptive = True, adaptive_domain='stackoverflow.com') ->> ->> page = Fetcher.get(old_url, timeout=30) ->> element1 = page.css(selector, auto_save=True)[0] ->> ->> # Same selector but used in the updated website ->> page = Fetcher.get(new_url) ->> element2 = page.css(selector, adaptive=True)[0] ->> ->> if element1.text == element2.text: +from scrapling import Fetcher +selector = '#hmenus > div:nth-child(1) > ul > li:nth-child(1) > a' +old_url = "https://web.archive.org/web/20100102003420/http://stackoverflow.com/" +new_url = "https://stackoverflow.com/" +Fetcher.configure(adaptive = True, adaptive_domain='stackoverflow.com') + +page = Fetcher.get(old_url, timeout=30) +element1 = page.css(selector, auto_save=True)[0] + +# Same selector but used in the updated website +page = Fetcher.get(new_url) +element2 = page.css(selector, adaptive=True)[0] + +if element1.text == element2.text: ... print('Scrapling found the same element in the old and new designs!') -'Scrapling found the same element in the old and new designs!' ``` The `adaptive_domain` argument is used here because Scrapling sees `archive.org` and `stackoverflow.com` as two different domains and would isolate their `adaptive` data. Passing `adaptive_domain` tells Scrapling to treat them as the same website for adaptive data storage. @@ -127,11 +126,11 @@ First, enable the `adaptive` feature by passing `adaptive=True` to the [Selector Examples: ```python ->>> from scrapling import Selector, Fetcher ->>> page = Selector(html_doc, adaptive=True) +from scrapling import Selector, Fetcher +page = Selector(html_doc, adaptive=True) # OR ->>> Fetcher.adaptive = True ->>> page = Fetcher.get('https://example.com') +Fetcher.adaptive = True +page = Fetcher.get('https://example.com') ``` When using the [Selector](main_classes.md#selector) class, pass the URL of the website with the `url` argument so Scrapling can separate the properties saved for each element by domain. @@ -159,11 +158,11 @@ Elements can be manually saved, retrieved, and relocated within the `adaptive` f Example of getting an element by text: ```python ->>> element = page.find_by_text('Tipping the Velvet', first_match=True) +element = page.find_by_text('Tipping the Velvet', first_match=True) ``` Save its unique properties using the `save` method. The identifier must be set manually (use a meaningful identifier): ```python ->>> page.save(element, 'my_special_element') +page.save(element, 'my_special_element') ``` Later, retrieve and relocate the element inside the page with `adaptive`: ```python diff --git a/agent-skill/Scrapling-Skill/references/parsing/main_classes.md b/agent-skill/Scrapling-Skill/references/parsing/main_classes.md index 3033e7c..0d54498 100644 --- a/agent-skill/Scrapling-Skill/references/parsing/main_classes.md +++ b/agent-skill/Scrapling-Skill/references/parsing/main_classes.md @@ -131,14 +131,14 @@ Getting the attributes of the element ``` Access a specific attribute with any of the following ```python ->>> article.attrib['class'] ->>> article.attrib.get('class') ->>> article['class'] # new in v0.3 +article.attrib['class'] +article.attrib.get('class') +article['class'] # new in v0.3 ``` Check if the attributes contain a specific attribute with any of the methods below ```python ->>> 'class' in article.attrib ->>> 'class' in article # new in v0.3 +'class' in article.attrib +'class' in article # new in v0.3 ``` Get the HTML content of the element ```python @@ -279,13 +279,13 @@ In the [Selector](#selector) class, all methods/properties that should return a Starting with v0.4, all selection methods consistently return [Selector](#selector)/[Selectors](#selectors) objects, even for text nodes and attribute values. Text nodes (selected via `::text`, `/text()`, `::attr()`, `/@attr`) are wrapped in [Selector](#selector) objects. These text node selectors have `tag` set to `"#text"`, and their `text` property returns the text value. You can still access the text value directly, and all other properties return empty/default values gracefully. ```python ->>> page.css('a::text') # -> Selectors (of text node Selectors) ->>> page.xpath('//a/text()') # -> Selectors ->>> page.css('a::text').get() # -> TextHandler (the first text value) ->>> page.css('a::text').getall() # -> TextHandlers (all text values) ->>> page.css('a::attr(href)') # -> Selectors ->>> page.xpath('//a/@href') # -> Selectors ->>> page.css('.price_color') # -> Selectors +page.css('a::text') # -> Selectors (of text node Selectors) +page.xpath('//a/text()') # -> Selectors +page.css('a::text').get() # -> TextHandler (the first text value) +page.css('a::text').getall() # -> TextHandlers (all text values) +page.css('a::attr(href)') # -> Selectors +page.xpath('//a/@href') # -> Selectors +page.css('.price_color') # -> Selectors ``` ### Data extraction methods diff --git a/agent-skill/Scrapling-Skill/references/parsing/selection.md b/agent-skill/Scrapling-Skill/references/parsing/selection.md index e74cbf1..d82a802 100644 --- a/agent-skill/Scrapling-Skill/references/parsing/selection.md +++ b/agent-skill/Scrapling-Skill/references/parsing/selection.md @@ -346,8 +346,8 @@ It filters all elements in the current page/element in the following order: ### Examples ```python ->>> from scrapling.fetchers import Fetcher ->>> page = Fetcher.get('https://quotes.toscrape.com/') +from scrapling.fetchers import Fetcher +page = Fetcher.get('https://quotes.toscrape.com/') ``` Find all elements with the tag name `div`. ```python diff --git a/docs/cli/interactive-shell.md b/docs/cli/interactive-shell.md index b897ce7..297aa35 100644 --- a/docs/cli/interactive-shell.md +++ b/docs/cli/interactive-shell.md @@ -43,25 +43,24 @@ Once launched, you'll see the Scrapling banner and can immediately start scrapin ```python # No imports needed - everything is ready! ->>> get('https://news.ycombinator.com') +get('https://news.ycombinator.com') ->>> # Explore the page structure ->>> page.css('a')[:5] # Look at first 5 links +# Explore the page structure +page.css('a')[:5] # Look at first 5 links ->>> # Refine your selectors ->>> stories = page.css('.titleline>a') ->>> len(stories) -30 +# Refine your selectors +stories = page.css('.titleline>a') +len(stories) # 30 ->>> # Extract specific data ->>> for story in stories[:3]: +# Extract specific data +for story in stories[:3]: ... title = story.text ... url = story['href'] ... print(f"{title}: {url}") ->>> # Try different approaches ->>> titles = page.css('.titleline>a::text') # Direct text extraction ->>> urls = page.css('.titleline>a::attr(href)') # Direct attribute extraction +# Try different approaches +titles = page.css('.titleline>a::text') # Direct text extraction +urls = page.css('.titleline>a::attr(href)') # Direct attribute extraction ``` ## Built-in Shortcuts @@ -86,12 +85,10 @@ The shell automatically tracks your requests and pages: The `page` and `response` commands are automatically updated with the last fetched page: ```python - >>> get('https://quotes.toscrape.com') - >>> # 'page' and 'response' both refer to the last fetched page - >>> page.url - 'https://quotes.toscrape.com' - >>> response.status # Same as page.status - 200 + get('https://quotes.toscrape.com') + # 'page' and 'response' both refer to the last fetched page + page.url # 'https://quotes.toscrape.com' + response.status # Prints 200; Same as page.status ``` - **Page History** @@ -99,20 +96,17 @@ The shell automatically tracks your requests and pages: The `pages` command keeps track of the last five pages (it's a `Selectors` object): ```python - >>> get('https://site1.com') - >>> get('https://site2.com') - >>> get('https://site3.com') + get('https://site1.com') + get('https://site2.com') + get('https://site3.com') - >>> # Access last 5 pages - >>> len(pages) # `Selectors` object with `page` history - 3 - >>> pages[0].url # First page in history - 'https://site1.com' - >>> pages[-1].url # Most recent page - 'https://site3.com' + # Access last 5 pages + len(pages) # `Selectors` object with `page` history -> 3 + pages[0].url # First page in history -> 'https://site1.com' + pages[-1].url # Most recent page -> 'https://site3.com' - >>> # Work with historical pages - >>> for i, old_page in enumerate(pages): + # Work with historical pages + for i, old_page in enumerate(pages): ... print(f"Page {i}: {old_page.url} - {old_page.status}") ``` @@ -123,8 +117,8 @@ The shell automatically tracks your requests and pages: View scraped pages in your browser: ```python ->>> get('https://quotes.toscrape.com') ->>> view(page) # Opens the page HTML in your default browser + get('https://quotes.toscrape.com') + view(page) # Opens the page HTML in your default browser ``` ### Curl Command Integration @@ -138,29 +132,24 @@ First, you need to copy a request as a curl command like the following: - **Convert Curl command to Request Object** ```python - >>> curl_cmd = '''curl 'https://scrapling.requestcatcher.com/post' \ + curl_cmd = '''curl 'https://scrapling.requestcatcher.com/post' \ ... -X POST \ ... -H 'Content-Type: application/json' \ ... -d '{"name": "test", "value": 123}' ''' - >>> request = uncurl(curl_cmd) - >>> request.method - 'post' - >>> request.url - 'https://scrapling.requestcatcher.com/post' - >>> request.headers - {'Content-Type': 'application/json'} + request = uncurl(curl_cmd) + request.method # -> 'post' + request.url # -> 'https://scrapling.requestcatcher.com/post' + request.headers # -> {'Content-Type': 'application/json'} ``` - **Execute Curl Command Directly** ```python - >>> # Convert and execute in one step - >>> curl2fetcher(curl_cmd) - >>> page.status - 200 - >>> page.json()['json'] - {'name': 'test', 'value': 123} + # Convert and execute in one step + curl2fetcher(curl_cmd) + page.status # -> 200 + page.json()['json'] # -> {'name': 'test', 'value': 123} ``` ### IPython Features @@ -168,17 +157,17 @@ First, you need to copy a request as a curl command like the following: The shell inherits all IPython capabilities: ```python ->>> # Magic commands ->>> %time page = get('https://example.com') # Time execution ->>> %history # Show command history ->>> %save filename.py 1-10 # Save commands 1-10 to file +# Magic commands +%time page = get('https://example.com') # Time execution +%history # Show command history +%save filename.py 1-10 # Save commands 1-10 to file ->>> # Tab completion works everywhere ->>> page.c # Shows: css, cookies, headers, etc. ->>> Fetcher. # Shows all Fetcher methods +# Tab completion works everywhere +page.c # Shows: css, cookies, headers, etc. +Fetcher. # Shows all Fetcher methods ->>> # Object inspection ->>> get? # Show get documentation +# Object inspection +get? # Show get documentation ``` ## Examples @@ -188,23 +177,23 @@ Here are a few examples generated via AI: #### E-commerce Data Collection ```python ->>> # Start with product listing page ->>> catalog = get('https://shop.example.com/products') +# Start with product listing page +catalog = get('https://shop.example.com/products') ->>> # Find product links ->>> product_links = catalog.css('.product-link::attr(href)') ->>> print(f"Found {len(product_links)} products") +# Find product links +product_links = catalog.css('.product-link::attr(href)') +print(f"Found {len(product_links)} products") ->>> # Sample a few products first ->>> for link in product_links[:3]: +# Sample a few products first +for link in product_links[:3]: ... product = get(f"https://shop.example.com{link}") ... name = product.css('.product-name::text').get('') ... price = product.css('.price::text').get('') ... print(f"{name}: {price}") ->>> # Scale up with sessions for efficiency ->>> from scrapling.fetchers import FetcherSession ->>> with FetcherSession() as session: +# Scale up with sessions for efficiency +from scrapling.fetchers import FetcherSession +with FetcherSession() as session: ... products = [] ... for link in product_links: ... product = session.get(f"https://shop.example.com{link}") diff --git a/docs/development/scrapling_custom_types.md b/docs/development/scrapling_custom_types.md index 2f638a9..ef67d27 100644 --- a/docs/development/scrapling_custom_types.md +++ b/docs/development/scrapling_custom_types.md @@ -4,13 +4,12 @@ ### All current types can be imported alone, like below ```python ->>> from scrapling.core.custom_types import TextHandler, AttributesHandler +from scrapling.core.custom_types import TextHandler, AttributesHandler ->>> somestring = TextHandler('{}') ->>> somestring.json() -'{}' ->>> somedict_1 = AttributesHandler({'a': 1}) ->>> somedict_2 = AttributesHandler(a=1) +somestring = TextHandler('{}') +somestring.json() # '{}' +somedict_1 = AttributesHandler({'a': 1}) +somedict_2 = AttributesHandler(a=1) ``` Note that `TextHandler` is a subclass of Python's `str`, so all standard operations/methods that work with Python strings will work. diff --git a/docs/fetching/choosing.md b/docs/fetching/choosing.md index b9f3ee4..b2b4bf2 100644 --- a/docs/fetching/choosing.md +++ b/docs/fetching/choosing.md @@ -31,23 +31,23 @@ In the following pages, we will talk about each one in detail. ## Parser configuration in all fetchers All fetchers share the same import method, as you will see in the upcoming pages ```python ->>> from scrapling.fetchers import Fetcher, AsyncFetcher, StealthyFetcher, DynamicFetcher +from scrapling.fetchers import Fetcher, AsyncFetcher, StealthyFetcher, DynamicFetcher ``` Then you use it right away without initializing like this, and it will use the default parser settings: ```python ->>> page = StealthyFetcher.fetch('https://example.com') +page = StealthyFetcher.fetch('https://example.com') ``` If you want to configure the parser ([Selector class](../parsing/main_classes.md#selector)) that will be used on the response before returning it for you, then do this first: ```python ->>> from scrapling.fetchers import Fetcher ->>> Fetcher.configure(adaptive=True, keep_comments=False, keep_cdata=False) # and the rest +from scrapling.fetchers import Fetcher +Fetcher.configure(adaptive=True, keep_comments=False, keep_cdata=False) # and the rest ``` or ```python ->>> from scrapling.fetchers import Fetcher ->>> Fetcher.adaptive=True ->>> Fetcher.keep_comments=False ->>> Fetcher.keep_cdata=False # and the rest +from scrapling.fetchers import Fetcher +Fetcher.adaptive=True +Fetcher.keep_comments=False +Fetcher.keep_cdata=False # and the rest ``` Then, continue your code as usual. @@ -65,19 +65,19 @@ If your use case requires a different configuration for each request/fetch, you ## Response Object The `Response` object is the same as the [Selector](../parsing/main_classes.md#selector) class, but it has additional details about the response, like response headers, status, cookies, etc., as shown below: ```python ->>> from scrapling.fetchers import Fetcher ->>> page = Fetcher.get('https://example.com') +from scrapling.fetchers import Fetcher +page = Fetcher.get('https://example.com') ->>> page.status # HTTP status code ->>> page.reason # Status message ->>> page.cookies # Response cookies as a dictionary ->>> page.headers # Response headers ->>> page.request_headers # Request headers ->>> page.history # Response history of redirections, if any ->>> page.body # Raw response body as bytes ->>> page.encoding # Response encoding ->>> page.meta # Response metadata dictionary (e.g., proxy used). Mainly helpful with the spiders system. ->>> page.captured_xhr # List of captured XHR/fetch responses (when capture_xhr is enabled on a browser session) +page.status # HTTP status code +page.reason # Status message +page.cookies # Response cookies as a dictionary +page.headers # Response headers +page.request_headers # Request headers +page.history # Response history of redirections, if any +page.body # Raw response body as bytes +page.encoding # Response encoding +page.meta # Response metadata dictionary (e.g., proxy used). Mainly helpful with the spiders system. +page.captured_xhr # List of captured XHR/fetch responses (when capture_xhr is enabled on a browser session) ``` All fetchers return the `Response` object. diff --git a/docs/fetching/dynamic.md b/docs/fetching/dynamic.md index 2e1f537..f987f4a 100644 --- a/docs/fetching/dynamic.md +++ b/docs/fetching/dynamic.md @@ -14,7 +14,7 @@ As we will explain later, to automate the page, you need some knowledge of [Play You have one primary way to import this Fetcher, which is the same for all fetchers. ```python ->>> from scrapling.fetchers import DynamicFetcher +from scrapling.fetchers import DynamicFetcher ``` Check out how to configure the parsing options [here](choosing.md#parser-configuration-in-all-fetchers) diff --git a/docs/fetching/static.md b/docs/fetching/static.md index 7d2a522..b6bcfef 100644 --- a/docs/fetching/static.md +++ b/docs/fetching/static.md @@ -12,7 +12,7 @@ The `Fetcher` class provides rapid and lightweight HTTP requests using the high- You have one primary way to import this Fetcher, which is the same for all fetchers. ```python ->>> from scrapling.fetchers import Fetcher +from scrapling.fetchers import Fetcher ``` Check out how to configure the parsing options [here](choosing.md#parser-configuration-in-all-fetchers) @@ -54,47 +54,47 @@ Examples are the best way to explain this: > Hence: `OPTIONS` and `HEAD` methods are not supported. #### GET ```python ->>> from scrapling.fetchers import Fetcher ->>> # Basic GET ->>> page = Fetcher.get('https://example.com') ->>> page = Fetcher.get('https://scrapling.requestcatcher.com/get', stealthy_headers=True) ->>> page = Fetcher.get('https://scrapling.requestcatcher.com/get', proxy='http://username:password@localhost:8030') ->>> # With parameters ->>> page = Fetcher.get('https://example.com/search', params={'q': 'query'}) ->>> ->>> # With headers ->>> page = Fetcher.get('https://example.com', headers={'User-Agent': 'Custom/1.0'}) ->>> # Basic HTTP authentication ->>> page = Fetcher.get("https://example.com", auth=("my_user", "password123")) ->>> # Browser impersonation ->>> page = Fetcher.get('https://example.com', impersonate='chrome') ->>> # HTTP/3 support ->>> page = Fetcher.get('https://example.com', http3=True) +from scrapling.fetchers import Fetcher +# Basic GET +page = Fetcher.get('https://example.com') +page = Fetcher.get('https://scrapling.requestcatcher.com/get', stealthy_headers=True) +page = Fetcher.get('https://scrapling.requestcatcher.com/get', proxy='http://username:password@localhost:8030') +# With parameters +page = Fetcher.get('https://example.com/search', params={'q': 'query'}) + +# With headers +page = Fetcher.get('https://example.com', headers={'User-Agent': 'Custom/1.0'}) +# Basic HTTP authentication +page = Fetcher.get("https://example.com", auth=("my_user", "password123")) +# Browser impersonation +page = Fetcher.get('https://example.com', impersonate='chrome') +# HTTP/3 support +page = Fetcher.get('https://example.com', http3=True) ``` And for asynchronous requests, it's a small adjustment ```python ->>> from scrapling.fetchers import AsyncFetcher ->>> # Basic GET ->>> page = await AsyncFetcher.get('https://example.com') ->>> page = await AsyncFetcher.get('https://scrapling.requestcatcher.com/get', stealthy_headers=True) ->>> page = await AsyncFetcher.get('https://scrapling.requestcatcher.com/get', proxy='http://username:password@localhost:8030') ->>> # With parameters ->>> page = await AsyncFetcher.get('https://example.com/search', params={'q': 'query'}) +from scrapling.fetchers import AsyncFetcher +# Basic GET +page = await AsyncFetcher.get('https://example.com') +page = await AsyncFetcher.get('https://scrapling.requestcatcher.com/get', stealthy_headers=True) +page = await AsyncFetcher.get('https://scrapling.requestcatcher.com/get', proxy='http://username:password@localhost:8030') +# With parameters + page = await AsyncFetcher.get('https://example.com/search', params={'q': 'query'}) >>> ->>> # With headers ->>> page = await AsyncFetcher.get('https://example.com', headers={'User-Agent': 'Custom/1.0'}) ->>> # Basic HTTP authentication ->>> page = await AsyncFetcher.get("https://example.com", auth=("my_user", "password123")) ->>> # Browser impersonation ->>> page = await AsyncFetcher.get('https://example.com', impersonate='chrome110') ->>> # HTTP/3 support ->>> page = await AsyncFetcher.get('https://example.com', http3=True) +# With headers +page = await AsyncFetcher.get('https://example.com', headers={'User-Agent': 'Custom/1.0'}) +# Basic HTTP authentication +page = await AsyncFetcher.get("https://example.com", auth=("my_user", "password123")) +# Browser impersonation +page = await AsyncFetcher.get('https://example.com', impersonate='chrome110') +# HTTP/3 support +page = await AsyncFetcher.get('https://example.com', http3=True) ``` Needless to say, the `page` object in all cases is [Response](choosing.md#response-object) object, which is a [Selector](../parsing/main_classes.md#selector) as we said, so you can use it directly ```python ->>> page.css('.something.something') +page.css('.something.something') ->>> page = Fetcher.get('https://api.github.com/events') +page = Fetcher.get('https://api.github.com/events') >>> page.json() [{'id': '', 'type': 'PushEvent', @@ -109,62 +109,62 @@ Needless to say, the `page` object in all cases is [Response](choosing.md#respon ``` #### POST ```python ->>> from scrapling.fetchers import Fetcher ->>> # Basic POST ->>> page = Fetcher.post('https://scrapling.requestcatcher.com/post', data={'key': 'value'}, params={'q': 'query'}) ->>> page = Fetcher.post('https://scrapling.requestcatcher.com/post', data={'key': 'value'}, stealthy_headers=True) ->>> page = Fetcher.post('https://scrapling.requestcatcher.com/post', data={'key': 'value'}, proxy='http://username:password@localhost:8030', impersonate="chrome") ->>> # Another example of form-encoded data ->>> page = Fetcher.post('https://example.com/submit', data={'username': 'user', 'password': 'pass'}, http3=True) ->>> # JSON data ->>> page = Fetcher.post('https://example.com/api', json={'key': 'value'}) +from scrapling.fetchers import Fetcher +# Basic POST +page = Fetcher.post('https://scrapling.requestcatcher.com/post', data={'key': 'value'}, params={'q': 'query'}) +page = Fetcher.post('https://scrapling.requestcatcher.com/post', data={'key': 'value'}, stealthy_headers=True) +page = Fetcher.post('https://scrapling.requestcatcher.com/post', data={'key': 'value'}, proxy='http://username:password@localhost:8030', impersonate="chrome") +# Another example of form-encoded data +page = Fetcher.post('https://example.com/submit', data={'username': 'user', 'password': 'pass'}, http3=True) +# JSON data +page = Fetcher.post('https://example.com/api', json={'key': 'value'}) ``` And for asynchronous requests, it's a small adjustment ```python ->>> from scrapling.fetchers import AsyncFetcher ->>> # Basic POST ->>> page = await AsyncFetcher.post('https://scrapling.requestcatcher.com/post', data={'key': 'value'}) ->>> page = await AsyncFetcher.post('https://scrapling.requestcatcher.com/post', data={'key': 'value'}, stealthy_headers=True) ->>> page = await AsyncFetcher.post('https://scrapling.requestcatcher.com/post', data={'key': 'value'}, proxy='http://username:password@localhost:8030', impersonate="chrome") ->>> # Another example of form-encoded data ->>> page = await AsyncFetcher.post('https://example.com/submit', data={'username': 'user', 'password': 'pass'}, http3=True) ->>> # JSON data ->>> page = await AsyncFetcher.post('https://example.com/api', json={'key': 'value'}) +from scrapling.fetchers import AsyncFetcher +# Basic POST +page = await AsyncFetcher.post('https://scrapling.requestcatcher.com/post', data={'key': 'value'}) +page = await AsyncFetcher.post('https://scrapling.requestcatcher.com/post', data={'key': 'value'}, stealthy_headers=True) +page = await AsyncFetcher.post('https://scrapling.requestcatcher.com/post', data={'key': 'value'}, proxy='http://username:password@localhost:8030', impersonate="chrome") +# Another example of form-encoded data +page = await AsyncFetcher.post('https://example.com/submit', data={'username': 'user', 'password': 'pass'}, http3=True) +# JSON data +page = await AsyncFetcher.post('https://example.com/api', json={'key': 'value'}) ``` #### PUT ```python ->>> from scrapling.fetchers import Fetcher ->>> # Basic PUT ->>> page = Fetcher.put('https://example.com/update', data={'status': 'updated'}) ->>> page = Fetcher.put('https://example.com/update', data={'status': 'updated'}, stealthy_headers=True, impersonate="chrome") ->>> page = Fetcher.put('https://example.com/update', data={'status': 'updated'}, proxy='http://username:password@localhost:8030') ->>> # Another example of form-encoded data ->>> page = Fetcher.put("https://scrapling.requestcatcher.com/put", data={'key': ['value1', 'value2']}) +from scrapling.fetchers import Fetcher +# Basic PUT +page = Fetcher.put('https://example.com/update', data={'status': 'updated'}) +page = Fetcher.put('https://example.com/update', data={'status': 'updated'}, stealthy_headers=True, impersonate="chrome") +page = Fetcher.put('https://example.com/update', data={'status': 'updated'}, proxy='http://username:password@localhost:8030') +# Another example of form-encoded data +page = Fetcher.put("https://scrapling.requestcatcher.com/put", data={'key': ['value1', 'value2']}) ``` And for asynchronous requests, it's a small adjustment ```python ->>> from scrapling.fetchers import AsyncFetcher ->>> # Basic PUT ->>> page = await AsyncFetcher.put('https://example.com/update', data={'status': 'updated'}) ->>> page = await AsyncFetcher.put('https://example.com/update', data={'status': 'updated'}, stealthy_headers=True, impersonate="chrome") ->>> page = await AsyncFetcher.put('https://example.com/update', data={'status': 'updated'}, proxy='http://username:password@localhost:8030') ->>> # Another example of form-encoded data ->>> page = await AsyncFetcher.put("https://scrapling.requestcatcher.com/put", data={'key': ['value1', 'value2']}) +from scrapling.fetchers import AsyncFetcher +# Basic PUT +page = await AsyncFetcher.put('https://example.com/update', data={'status': 'updated'}) +page = await AsyncFetcher.put('https://example.com/update', data={'status': 'updated'}, stealthy_headers=True, impersonate="chrome") +page = await AsyncFetcher.put('https://example.com/update', data={'status': 'updated'}, proxy='http://username:password@localhost:8030') +# Another example of form-encoded data +page = await AsyncFetcher.put("https://scrapling.requestcatcher.com/put", data={'key': ['value1', 'value2']}) ``` #### DELETE ```python ->>> from scrapling.fetchers import Fetcher ->>> page = Fetcher.delete('https://example.com/resource/123') ->>> page = Fetcher.delete('https://example.com/resource/123', stealthy_headers=True, impersonate="chrome") ->>> page = Fetcher.delete('https://example.com/resource/123', proxy='http://username:password@localhost:8030') +from scrapling.fetchers import Fetcher +page = Fetcher.delete('https://example.com/resource/123') +page = Fetcher.delete('https://example.com/resource/123', stealthy_headers=True, impersonate="chrome") +page = Fetcher.delete('https://example.com/resource/123', proxy='http://username:password@localhost:8030') ``` And for asynchronous requests, it's a small adjustment ```python ->>> from scrapling.fetchers import AsyncFetcher ->>> page = await AsyncFetcher.delete('https://example.com/resource/123') ->>> page = await AsyncFetcher.delete('https://example.com/resource/123', stealthy_headers=True, impersonate="chrome") ->>> page = await AsyncFetcher.delete('https://example.com/resource/123', proxy='http://username:password@localhost:8030') +from scrapling.fetchers import AsyncFetcher +page = await AsyncFetcher.delete('https://example.com/resource/123') +page = await AsyncFetcher.delete('https://example.com/resource/123', stealthy_headers=True, impersonate="chrome") +page = await AsyncFetcher.delete('https://example.com/resource/123', proxy='http://username:password@localhost:8030') ``` ## Session Management diff --git a/docs/fetching/stealthy.md b/docs/fetching/stealthy.md index 8daf0b0..2cff63d 100644 --- a/docs/fetching/stealthy.md +++ b/docs/fetching/stealthy.md @@ -15,7 +15,7 @@ As with [DynamicFetcher](dynamic.md#introduction), you will need some knowledge You have one primary way to import this Fetcher, which is the same for all fetchers. ```python ->>> from scrapling.fetchers import StealthyFetcher + from scrapling.fetchers import StealthyFetcher ``` Check out how to configure the parsing options [here](choosing.md#parser-configuration-in-all-fetchers) diff --git a/docs/overview.md b/docs/overview.md index c62c7c3..5a72b95 100644 --- a/docs/overview.md +++ b/docs/overview.md @@ -263,19 +263,19 @@ page = Fetcher.get('https://scrapling.requestcatcher.com/get', impersonate="chro ``` With that out of the way, here's how to do all HTTP methods: ```python ->>> from scrapling.fetchers import Fetcher ->>> page = Fetcher.get('https://scrapling.requestcatcher.com/get', stealthy_headers=True) ->>> page = Fetcher.post('https://scrapling.requestcatcher.com/post', data={'key': 'value'}, proxy='http://username:password@localhost:8030') ->>> page = Fetcher.put('https://scrapling.requestcatcher.com/put', data={'key': 'value'}) ->>> page = Fetcher.delete('https://scrapling.requestcatcher.com/delete') +from scrapling.fetchers import Fetcher +page = Fetcher.get('https://scrapling.requestcatcher.com/get', stealthy_headers=True) +page = Fetcher.post('https://scrapling.requestcatcher.com/post', data={'key': 'value'}, proxy='http://username:password@localhost:8030') +page = Fetcher.put('https://scrapling.requestcatcher.com/put', data={'key': 'value'}) +page = Fetcher.delete('https://scrapling.requestcatcher.com/delete') ``` For Async requests, you will replace the import like below: ```python ->>> from scrapling.fetchers import AsyncFetcher ->>> page = await AsyncFetcher.get('https://scrapling.requestcatcher.com/get', stealthy_headers=True) ->>> page = await AsyncFetcher.post('https://scrapling.requestcatcher.com/post', data={'key': 'value'}, proxy='http://username:password@localhost:8030') ->>> page = await AsyncFetcher.put('https://scrapling.requestcatcher.com/put', data={'key': 'value'}) ->>> page = await AsyncFetcher.delete('https://scrapling.requestcatcher.com/delete') +from scrapling.fetchers import AsyncFetcher +page = await AsyncFetcher.get('https://scrapling.requestcatcher.com/get', stealthy_headers=True) +page = await AsyncFetcher.post('https://scrapling.requestcatcher.com/post', data={'key': 'value'}, proxy='http://username:password@localhost:8030') +page = await AsyncFetcher.put('https://scrapling.requestcatcher.com/put', data={'key': 'value'}) +page = await AsyncFetcher.delete('https://scrapling.requestcatcher.com/delete') ``` !!! note "Notes:" @@ -291,14 +291,13 @@ We have you covered if you deal with dynamic websites like most today! The `DynamicFetcher` class (formerly `PlayWrightFetcher`) offers many options for fetching and loading web pages using Chromium-based browsers. ```python ->>> from scrapling.fetchers import DynamicFetcher ->>> page = DynamicFetcher.fetch('https://www.google.com/search?q=%22Scrapling%22', disable_resources=True) # Vanilla Playwright option ->>> page.css("#search a::attr(href)").get() -'https://github.com/D4Vinci/Scrapling' ->>> # The async version of fetch ->>> page = await DynamicFetcher.async_fetch('https://www.google.com/search?q=%22Scrapling%22', disable_resources=True) ->>> page.css("#search a::attr(href)").get() -'https://github.com/D4Vinci/Scrapling' +from scrapling.fetchers import DynamicFetcher +page = DynamicFetcher.fetch('https://www.google.com/search?q=%22Scrapling%22', disable_resources=True) # Vanilla Playwright option +page.css("#search a::attr(href)").get() # -> 'https://github.com/D4Vinci/Scrapling' + +# The async version of fetch +page = await DynamicFetcher.async_fetch('https://www.google.com/search?q=%22Scrapling%22', disable_resources=True) +page.css("#search a::attr(href)").get() # -> 'https://github.com/D4Vinci/Scrapling' ``` It's built on top of [Playwright](https://playwright.dev/python/), and it's currently providing two main run options that can be mixed as you want: @@ -323,18 +322,17 @@ Some of the things it does: 6. and other anti-protection options... ```python ->>> from scrapling.fetchers import StealthyFetcher ->>> page = StealthyFetcher.fetch('https://www.browserscan.net/bot-detection') # Running headless by default ->>> page.status == 200 -True ->>> page = StealthyFetcher.fetch('https://nopecha.com/demo/cloudflare', solve_cloudflare=True) # Solve Cloudflare captcha automatically if presented ->>> page.status == 200 -True ->>> page = StealthyFetcher.fetch('https://www.browserscan.net/bot-detection', humanize=True, os_randomize=True) # and the rest of arguments... ->>> # The async version of fetch ->>> page = await StealthyFetcher.async_fetch('https://www.browserscan.net/bot-detection') ->>> page.status == 200 -True +from scrapling.fetchers import StealthyFetcher +page = StealthyFetcher.fetch('https://www.browserscan.net/bot-detection') # Running headless by default +page.status == 200 # -> True + +page = StealthyFetcher.fetch('https://nopecha.com/demo/cloudflare', solve_cloudflare=True) # Solve Cloudflare captcha automatically if presented +page.status == 200 # -> True + +page = StealthyFetcher.fetch('https://www.browserscan.net/bot-detection', humanize=True, os_randomize=True) # and the rest of arguments... +# The async version of fetch +page = await StealthyFetcher.async_fetch('https://www.browserscan.net/bot-detection') +page.status == 200 # -> True ``` Again, this is just the tip of the iceberg with this fetcher. Check out the rest from [here](fetching/stealthy.md) for all details and the complete list of arguments. diff --git a/docs/parsing/adaptive.md b/docs/parsing/adaptive.md index 23dcaf3..b2ba6cf 100644 --- a/docs/parsing/adaptive.md +++ b/docs/parsing/adaptive.md @@ -76,22 +76,19 @@ If I want to extract the Questions button from the old design, I can use a selec Now, let's test the same selector in both versions ```python ->> from scrapling import Fetcher ->> selector = '#hmenus > div:nth-child(1) > ul > li:nth-child(1) > a' ->> old_url = "https://web.archive.org/web/20100102003420/http://stackoverflow.com/" ->> new_url = "https://stackoverflow.com/" ->> Fetcher.configure(adaptive = True, adaptive_domain='stackoverflow.com') ->> ->> page = Fetcher.get(old_url, timeout=30) ->> element1 = page.css(selector, auto_save=True)[0] ->> ->> # Same selector but used in the updated website ->> page = Fetcher.get(new_url) ->> element2 = page.css(selector, adaptive=True)[0] ->> ->> if element1.text == element2.text: -... print('Scrapling found the same element in the old and new designs!') -'Scrapling found the same element in the old and new designs!' +from scrapling import Fetcher +selector = '#hmenus > div:nth-child(1) > ul > li:nth-child(1) > a' +old_url = "https://web.archive.org/web/20100102003420/http://stackoverflow.com/" +new_url = "https://stackoverflow.com/" +Fetcher.configure(adaptive = True, adaptive_domain='stackoverflow.com') +page = Fetcher.get(old_url, timeout=30) +element1 = page.css(selector, auto_save=True)[0] +# Same selector but used in the updated website +page = Fetcher.get(new_url) +element2 = page.css(selector, adaptive=True)[0] + +if element1.text == element2.text: + print('Scrapling found the same element in the old and new designs!') # Spoiler alert: it does! ``` Note that I introduced a new argument called `adaptive_domain`. This is because, for Scrapling, these are two different domains (`archive.org` and `stackoverflow.com`), so Scrapling will isolate their `adaptive` data. To inform Scrapling that they are the same website, we must pass the custom domain we wish to use while saving `adaptive` data for both, ensuring Scrapling doesn't isolate them. @@ -141,11 +138,11 @@ First, you must enable the `adaptive` feature by passing `adaptive=True` to the Examples: ```python ->>> from scrapling import Selector, Fetcher ->>> page = Selector(html_doc, adaptive=True) +from scrapling import Selector, Fetcher +page = Selector(html_doc, adaptive=True) # OR ->>> Fetcher.adaptive = True ->>> page = Fetcher.get('https://example.com') +Fetcher.adaptive = True +page = Fetcher.get('https://example.com') ``` If you are using the [Selector](main_classes.md#selector) class, you need to pass the url of the website you are using with the argument `url` so Scrapling can separate the properties saved for each element by domain. @@ -175,11 +172,11 @@ You manually save and retrieve an element, then relocate it, which all happens w First, let's say you got an element like this by text: ```python ->>> element = page.find_by_text('Tipping the Velvet', first_match=True) +element = page.find_by_text('Tipping the Velvet', first_match=True) ``` You can save its unique properties using the `save` method, as shown below, but you must set the identifier yourself. For this example, I chose `my_special_element` as an identifier, but it's best to use a meaningful identifier in your code for the same reason you use meaningful variable names :) ```python ->>> page.save(element, 'my_special_element') +page.save(element, 'my_special_element') ``` Now, later, when you want to retrieve it and relocate it inside the page with `adaptive`, it would be like this ```python diff --git a/docs/parsing/main_classes.md b/docs/parsing/main_classes.md index 6d2cac0..d41ff40 100644 --- a/docs/parsing/main_classes.md +++ b/docs/parsing/main_classes.md @@ -140,14 +140,14 @@ Getting the attributes of the element ``` Access a specific attribute with any of the following ```python ->>> article.attrib['class'] ->>> article.attrib.get('class') ->>> article['class'] # new in v0.3 +article.attrib['class'] +article.attrib.get('class') +article['class'] # new in v0.3 ``` Check if the attributes contain a specific attribute with any of the methods below ```python ->>> 'class' in article.attrib ->>> 'class' in article # new in v0.3 +'class' in article.attrib +'class' in article # new in v0.3 ``` Get the HTML content of the element ```python @@ -292,13 +292,13 @@ In the [Selector](#selector) class, all methods/properties that should return a Starting with v0.4, all selection methods consistently return [Selector](#selector)/[Selectors](#selectors) objects, even for text nodes and attribute values. Text nodes (selected via `::text`, `/text()`, `::attr()`, `/@attr`) are wrapped in [Selector](#selector) objects. These text node selectors have `tag` set to `"#text"`, and their `text` property returns the text value. You can still access the text value directly, and all other properties return empty/default values gracefully. ```python ->>> page.css('a::text') # -> Selectors (of text node Selectors) ->>> page.xpath('//a/text()') # -> Selectors ->>> page.css('a::text').get() # -> TextHandler (the first text value) ->>> page.css('a::text').getall() # -> TextHandlers (all text values) ->>> page.css('a::attr(href)') # -> Selectors ->>> page.xpath('//a/@href') # -> Selectors ->>> page.css('.price_color') # -> Selectors +page.css('a::text') # -> Selectors (of text node Selectors) +page.xpath('//a/text()') # -> Selectors +page.css('a::text').get() # -> TextHandler (the first text value) +page.css('a::text').getall() # -> TextHandlers (all text values) +page.css('a::attr(href)') # -> Selectors +page.xpath('//a/@href') # -> Selectors +page.css('.price_color') # -> Selectors ``` ### Data extraction methods diff --git a/docs/parsing/selection.md b/docs/parsing/selection.md index 5311391..aef8f3b 100644 --- a/docs/parsing/selection.md +++ b/docs/parsing/selection.md @@ -362,8 +362,8 @@ Check examples to clear any confusion :) ### Examples ```python ->>> from scrapling.fetchers import Fetcher ->>> page = Fetcher.get('https://quotes.toscrape.com/') +from scrapling.fetchers import Fetcher +page = Fetcher.get('https://quotes.toscrape.com/') ``` Find all elements with the tag name `div`. ```python From b6f0f2a7f2413af7ff3cf72b11dee021ee9df130 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Wed, 22 Apr 2026 17:01:11 +0200 Subject: [PATCH 04/20] build: pump up version and deps --- agent-skill/Scrapling-Skill/SKILL.md | 4 ++-- agent-skill/Scrapling-Skill/examples/README.md | 2 +- pyproject.toml | 4 ++-- scrapling/__init__.py | 2 +- server.json | 4 ++-- setup.cfg | 2 +- 6 files changed, 9 insertions(+), 9 deletions(-) diff --git a/agent-skill/Scrapling-Skill/SKILL.md b/agent-skill/Scrapling-Skill/SKILL.md index 2cd1f84..24af980 100644 --- a/agent-skill/Scrapling-Skill/SKILL.md +++ b/agent-skill/Scrapling-Skill/SKILL.md @@ -1,7 +1,7 @@ --- name: scrapling-official description: Scrape web pages using Scrapling with anti-bot bypass (like Cloudflare Turnstile), stealth headless browsing, spiders framework, adaptive scraping, and JavaScript rendering. Use when asked to scrape, crawl, or extract data from websites; web_fetch fails; the site has anti-bot protections; write Python code to scrape/crawl; or write spiders. -version: "0.4.7" +version: "0.4.8" license: Complete terms in LICENSE.txt metadata: homepage: "https://scrapling.readthedocs.io/en/latest/index.html" @@ -40,7 +40,7 @@ Blazing fast crawls with real-time stats and streaming. Built by Web Scrapers fo Create a virtual Python environment through any way available, like `venv`, then inside the environment do: -`pip install "scrapling[all]>=0.4.7"` +`pip install "scrapling[all]>=0.4.8"` Then do this to download all the browsers' dependencies: diff --git a/agent-skill/Scrapling-Skill/examples/README.md b/agent-skill/Scrapling-Skill/examples/README.md index 388a594..d0f9a2b 100644 --- a/agent-skill/Scrapling-Skill/examples/README.md +++ b/agent-skill/Scrapling-Skill/examples/README.md @@ -9,7 +9,7 @@ All examples collect **all 100 quotes across 10 pages**. Make sure Scrapling is installed: ```bash -pip install "scrapling[all]>=0.4.7" +pip install "scrapling[all]>=0.4.8" scrapling install --force ``` diff --git a/pyproject.toml b/pyproject.toml index ada11a4..741a30d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta" [project] name = "scrapling" # Static version instead of a dynamic version so we can get better layer caching while building docker, check the docker file to understand -version = "0.4.7" +version = "0.4.8" description = "Scrapling is an undetectable, powerful, flexible, high-performance Python library that makes Web Scraping easy and effortless as it should be!" readme = {file = "README.md", content-type = "text/markdown"} license = {file = "LICENSE"} @@ -61,7 +61,7 @@ classifiers = [ "Typing :: Typed", ] dependencies = [ - "lxml>=6.0.3", + "lxml>=6.1.0", "cssselect>=1.4.0", "orjson>=3.11.8", "tld>=0.13.2", diff --git a/scrapling/__init__.py b/scrapling/__init__.py index 97af0c5..3420e25 100644 --- a/scrapling/__init__.py +++ b/scrapling/__init__.py @@ -1,5 +1,5 @@ __author__ = "Karim Shoair (karim.shoair@pm.me)" -__version__ = "0.4.7" +__version__ = "0.4.8" __copyright__ = "Copyright (c) 2024 Karim Shoair" from typing import Any, TYPE_CHECKING diff --git a/server.json b/server.json index 5415056..e4e813e 100644 --- a/server.json +++ b/server.json @@ -14,12 +14,12 @@ "mimeType": "image/png" } ], - "version": "0.4.7", + "version": "0.4.8", "packages": [ { "registryType": "pypi", "identifier": "scrapling", - "version": "0.4.7", + "version": "0.4.8", "runtimeHint": "uvx", "packageArguments": [ { diff --git a/setup.cfg b/setup.cfg index 72d64fc..1c967c8 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = scrapling -version = 0.4.7 +version = 0.4.8 author = Karim Shoair author_email = karim.shoair@pm.me description = Scrapling is an undetectable, powerful, flexible, high-performance Python library that makes Web Scraping easy and effortless as it should be! From fa41ca6fa9a65d714ec19ddf3b5816e59a2b19f2 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Sat, 25 Apr 2026 20:34:33 +0300 Subject: [PATCH 05/20] docs: Updating deps and allow code copy --- docs/requirements.txt | 4 ++-- zensical.toml | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index f218ea6..237ed2f 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,5 +1,5 @@ -zensical>=0.0.30 -mkdocstrings>=1.0.3 +zensical>=0.0.36 +mkdocstrings>=1.0.4 mkdocstrings-python>=2.0.3 griffe-inherited-docstrings>=1.1.3 griffe-runtime-objects>=0.3.1 diff --git a/zensical.toml b/zensical.toml index 4049a8b..9eaea06 100644 --- a/zensical.toml +++ b/zensical.toml @@ -90,6 +90,7 @@ features = [ "search.share", "search.suggest", "search.highlight", + "content.code.copy", ] [[project.theme.palette]] From 835e7ca8c33af9e485cda0c0bdf1b7aeb1d1d966 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Sun, 26 Apr 2026 02:46:13 +0300 Subject: [PATCH 06/20] docs: update old examples --- .../references/fetching/dynamic.md | 2 +- .../references/fetching/static.md | 2 +- docs/fetching/dynamic.md | 161 +++++++++--------- docs/fetching/static.md | 2 +- docs/fetching/stealthy.md | 10 +- docs/overview.md | 10 +- 6 files changed, 94 insertions(+), 93 deletions(-) diff --git a/agent-skill/Scrapling-Skill/references/fetching/dynamic.md b/agent-skill/Scrapling-Skill/references/fetching/dynamic.md index 5f5b0ce..4880ffb 100644 --- a/agent-skill/Scrapling-Skill/references/fetching/dynamic.md +++ b/agent-skill/Scrapling-Skill/references/fetching/dynamic.md @@ -149,7 +149,7 @@ with DynamicSession(proxy_rotator=rotator, headless=True) as session: ### Downloading Files ```python -page = DynamicFetcher.fetch('https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/main_cover.png') +page = DynamicFetcher.fetch('https://raw.githubusercontent.com/D4Vinci/Scrapling/main/docs/assets/main_cover.png') with open(file='main_cover.png', mode='wb') as f: f.write(page.body) diff --git a/agent-skill/Scrapling-Skill/references/fetching/static.md b/agent-skill/Scrapling-Skill/references/fetching/static.md index e24f6ce..9d5b057 100644 --- a/agent-skill/Scrapling-Skill/references/fetching/static.md +++ b/agent-skill/Scrapling-Skill/references/fetching/static.md @@ -301,7 +301,7 @@ def scrape_products(): ```python from scrapling.fetchers import Fetcher -page = Fetcher.get('https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/main_cover.png') +page = Fetcher.get('https://raw.githubusercontent.com/D4Vinci/Scrapling/main/docs/assets/main_cover.png') with open(file='main_cover.png', mode='wb') as f: f.write(page.body) ``` diff --git a/docs/fetching/dynamic.md b/docs/fetching/dynamic.md index f987f4a..49b0877 100644 --- a/docs/fetching/dynamic.md +++ b/docs/fetching/dynamic.md @@ -77,7 +77,7 @@ Scrapling provides many options with this fetcher and its session classes. To ma | wait_selector | Wait for a specific css selector to be in a specific state. | ✔️ | | init_script | An absolute path to a JavaScript file to be executed on page creation for all pages in this session. | ✔️ | | wait_selector_state | Scrapling will wait for the given state to be fulfilled for the selector given with `wait_selector`. _Default state is `attached`._ | ✔️ | -| google_search | Enabled by default, Scrapling will set a Google referer header. | ✔️ | +| google_search | Enabled by default, Scrapling will set a Google referer header. | ✔️ | | extra_headers | A dictionary of extra headers to add to the request. _The referer set by `google_search` takes priority over the referer set here if used together._ | ✔️ | | proxy | The proxy to be used with requests. It can be a string or a dictionary with only the keys 'server', 'username', and 'password'. | ✔️ | | real_chrome | If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch and use an instance of your browser. | ✔️ | @@ -89,12 +89,12 @@ Scrapling provides many options with this fetcher and its session classes. To ma | additional_args | Additional arguments to be passed to Playwright's context as additional settings, and they take higher priority than Scrapling's settings. | ✔️ | | selector_config | A dictionary of custom parsing arguments to be used when creating the final `Selector`/`Response` class. | ✔️ | | blocked_domains | A set of domain names to block requests to. Subdomains are also matched (e.g., `"example.com"` blocks `"sub.example.com"` too). | ✔️ | -| block_ads | Block requests to ~3,500 known ad/tracking domains. Can be combined with `blocked_domains`. | ✔️ | +| block_ads | Block requests to ~3,500 known ad/tracking domains. Can be combined with `blocked_domains`. | ✔️ | | dns_over_https | Route DNS queries through Cloudflare's DNS-over-HTTPS to prevent DNS leaks when using proxies. | ✔️ | | proxy_rotator | A `ProxyRotator` instance for automatic proxy rotation. Cannot be combined with `proxy`. | ✔️ | | retries | Number of retry attempts for failed requests. Defaults to 3. | ✔️ | | retry_delay | Seconds to wait between retry attempts. Defaults to 1. | ✔️ | -| capture_xhr | Pass a regex URL pattern string to capture XHR/fetch requests matching it during page load. Captured responses are available via `response.captured_xhr`. Defaults to `None` (disabled). | ✔️ | +| capture_xhr | Pass a regex URL pattern string to capture XHR/fetch requests matching it during page load. Captured responses are available via `response.captured_xhr`. Defaults to `None` (disabled). | ✔️ | | executable_path | Absolute path to a custom browser executable to use instead of the bundled Chromium. Useful for non-standard installations or custom browser builds. | ✔️ | In session classes, all these arguments can be set globally for the session. Still, you can configure each request individually by passing some of the arguments here that can be configured on the browser tab level like: `google_search`, `timeout`, `wait`, `page_action`, `page_setup`, `extra_headers`, `disable_resources`, `wait_selector`, `wait_selector_state`, `network_idle`, `load_dom`, `blocked_domains`, `proxy`, and `selector_config`. @@ -107,6 +107,65 @@ In session classes, all these arguments can be set globally for the session. Sti 4. If you didn't set a user agent and enabled headless mode, the fetcher will generate a real user agent for the same browser version and use it. If you didn't set a user agent and didn't enable headless mode, the fetcher will use the browser's default user agent, which is the same as in standard browsers in the latest versions. +## Session Management + +To keep the browser open until you make multiple requests with the same configuration, use `DynamicSession`/`AsyncDynamicSession` classes. Those classes can accept all the arguments that the `fetch` function can take, which enables you to specify a config for the entire session. + +```python +from scrapling.fetchers import DynamicSession + +# Create a session with default configuration +with DynamicSession( + headless=True, + disable_resources=True, + real_chrome=True +) as session: + # Make multiple requests with the same browser instance + page1 = session.fetch('https://example1.com') + page2 = session.fetch('https://example2.com') + page3 = session.fetch('https://dynamic-site.com') + + # All requests reuse the same tab on the same browser instance +``` + +### Async Session Usage + +```python +import asyncio +from scrapling.fetchers import AsyncDynamicSession + +async def scrape_multiple_sites(): + async with AsyncDynamicSession( + network_idle=True, + timeout=30000, + max_pages=3 + ) as session: + # Make async requests with shared browser configuration + pages = await asyncio.gather( + session.fetch('https://spa-app1.com'), + session.fetch('https://spa-app2.com'), + session.fetch('https://dynamic-content.com') + ) + return pages +``` + +You may have noticed the `max_pages` argument. This is a new argument that enables the fetcher to create a **rotating pool of Browser tabs**. Instead of using a single tab for all your requests, you set a limit on the maximum number of pages that can be displayed at once. With each request, the library will close all tabs that have finished their task and check if the number of the current tabs is lower than the maximum allowed number of pages/tabs, then: + +1. If you are within the allowed range, the fetcher will create a new tab for you, and then all is as normal. +2. Otherwise, it will keep checking every subsecond if creating a new tab is allowed or not for 60 seconds, then raise `TimeoutError`. This can happen when the website you are fetching becomes unresponsive. + +This logic allows for multiple URLs to be fetched at the same time in the same browser, which saves a lot of resources, but most importantly, is so fast :) + +In versions 0.3 and 0.3.1, the pool was reusing finished tabs to save more resources/time. That logic proved flawed, as it's nearly impossible to protect pages/tabs from contamination by the previous configuration used in the request before this one. + +### Session Benefits + +- **Browser reuse**: Much faster subsequent requests by reusing the same browser instance. +- **Cookie persistence**: Automatic cookie and session state handling as any browser does automatically. +- **Consistent fingerprint**: Same browser fingerprint across all requests. +- **Memory efficiency**: Better resource usage compared to launching new browsers with each fetch. + + ## Examples It's easier to understand with examples, so let's take a look. @@ -137,35 +196,10 @@ page = DynamicFetcher.fetch('https://example.com', timeout=30000) # 30 seconds page = DynamicFetcher.fetch('https://example.com', proxy='http://username:password@host:port') ``` -### Proxy Rotation - -```python -from scrapling.fetchers import DynamicSession, ProxyRotator - -# Set up proxy rotation -rotator = ProxyRotator([ - "http://proxy1:8080", - "http://proxy2:8080", - "http://proxy3:8080", -]) - -# Use with session - rotates proxy automatically with each request -with DynamicSession(proxy_rotator=rotator, headless=True) as session: - page1 = session.fetch('https://example1.com') - page2 = session.fetch('https://example2.com') - - # Override rotator for a specific request - page3 = session.fetch('https://example3.com', proxy='http://specific-proxy:8080') -``` - -!!! warning - - Remember that by default, all browser-based fetchers and sessions use a persistent browser context with a pool of tabs. However, since browsers can't set a proxy per tab, when you use a `ProxyRotator`, the fetcher will automatically open a separate context for each proxy, with one tab per context. Once the tab's job is done, both the tab and its context are closed. - ### Downloading Files ```python -page = DynamicFetcher.fetch('https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/main_cover.png') +page = DynamicFetcher.fetch('https://raw.githubusercontent.com/D4Vinci/Scrapling/main/docs/assets/main_cover.png') with open(file='main_cover.png', mode='wb') as f: f.write(page.body) @@ -229,8 +263,8 @@ page = await DynamicFetcher.async_fetch('https://example.com', page_action=scrol ```python # Wait for the selector page = DynamicFetcher.fetch( - 'https://example.com', - wait_selector='h1', + 'https://quotes.toscrape.com/js-delayed/', + wait_selector='.quote', wait_selector_state='visible' ) ``` @@ -297,63 +331,30 @@ def scrape_dynamic_content(): } ``` -## Session Management - -To keep the browser open until you make multiple requests with the same configuration, use `DynamicSession`/`AsyncDynamicSession` classes. Those classes can accept all the arguments that the `fetch` function can take, which enables you to specify a config for the entire session. +### Proxy Rotation ```python -from scrapling.fetchers import DynamicSession +from scrapling.fetchers import DynamicSession, ProxyRotator -# Create a session with default configuration -with DynamicSession( - headless=True, - disable_resources=True, - real_chrome=True -) as session: - # Make multiple requests with the same browser instance +# Set up proxy rotation +rotator = ProxyRotator([ + "http://proxy1:8080", + "http://proxy2:8080", + "http://proxy3:8080", +]) + +# Use with session - rotates proxy automatically with each request +with DynamicSession(proxy_rotator=rotator, headless=True) as session: page1 = session.fetch('https://example1.com') page2 = session.fetch('https://example2.com') - page3 = session.fetch('https://dynamic-site.com') - - # All requests reuse the same tab on the same browser instance + + # Override rotator for a specific request + page3 = session.fetch('https://example3.com', proxy='http://specific-proxy:8080') ``` -### Async Session Usage +!!! warning -```python -import asyncio -from scrapling.fetchers import AsyncDynamicSession - -async def scrape_multiple_sites(): - async with AsyncDynamicSession( - network_idle=True, - timeout=30000, - max_pages=3 - ) as session: - # Make async requests with shared browser configuration - pages = await asyncio.gather( - session.fetch('https://spa-app1.com'), - session.fetch('https://spa-app2.com'), - session.fetch('https://dynamic-content.com') - ) - return pages -``` - -You may have noticed the `max_pages` argument. This is a new argument that enables the fetcher to create a **rotating pool of Browser tabs**. Instead of using a single tab for all your requests, you set a limit on the maximum number of pages that can be displayed at once. With each request, the library will close all tabs that have finished their task and check if the number of the current tabs is lower than the maximum allowed number of pages/tabs, then: - -1. If you are within the allowed range, the fetcher will create a new tab for you, and then all is as normal. -2. Otherwise, it will keep checking every subsecond if creating a new tab is allowed or not for 60 seconds, then raise `TimeoutError`. This can happen when the website you are fetching becomes unresponsive. - -This logic allows for multiple URLs to be fetched at the same time in the same browser, which saves a lot of resources, but most importantly, is so fast :) - -In versions 0.3 and 0.3.1, the pool was reusing finished tabs to save more resources/time. That logic proved flawed, as it's nearly impossible to protect pages/tabs from contamination by the previous configuration used in the request before this one. - -### Session Benefits - -- **Browser reuse**: Much faster subsequent requests by reusing the same browser instance. -- **Cookie persistence**: Automatic cookie and session state handling as any browser does automatically. -- **Consistent fingerprint**: Same browser fingerprint across all requests. -- **Memory efficiency**: Better resource usage compared to launching new browsers with each fetch. + Remember that by default, all browser-based fetchers and sessions use a persistent browser context with a pool of tabs. However, since browsers can't set a proxy per tab, when you use a `ProxyRotator`, the fetcher will automatically open a separate context for each proxy, with one tab per context. Once the tab's job is done, both the tab and its context are closed. ## When to Use diff --git a/docs/fetching/static.md b/docs/fetching/static.md index b6bcfef..07df635 100644 --- a/docs/fetching/static.md +++ b/docs/fetching/static.md @@ -308,7 +308,7 @@ def scrape_products(): ```python from scrapling.fetchers import Fetcher -page = Fetcher.get('https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/main_cover.png') +page = Fetcher.get('https://raw.githubusercontent.com/D4Vinci/Scrapling/main/docs/assets/main_cover.png') with open(file='main_cover.png', mode='wb') as f: f.write(page.body) ``` diff --git a/docs/fetching/stealthy.md b/docs/fetching/stealthy.md index 2cff63d..ca9c707 100644 --- a/docs/fetching/stealthy.md +++ b/docs/fetching/stealthy.md @@ -54,7 +54,7 @@ Scrapling provides many options with this fetcher and its session classes. Befor | wait_selector | Wait for a specific css selector to be in a specific state. | ✔️ | | init_script | An absolute path to a JavaScript file to be executed on page creation for all pages in this session. | ✔️ | | wait_selector_state | Scrapling will wait for the given state to be fulfilled for the selector given with `wait_selector`. _Default state is `attached`._ | ✔️ | -| google_search | Enabled by default, Scrapling will set a Google referer header. | ✔️ | +| google_search | Enabled by default, Scrapling will set a Google referer header. | ✔️ | | extra_headers | A dictionary of extra headers to add to the request. _The referer set by `google_search` takes priority over the referer set here if used together._ | ✔️ | | proxy | The proxy to be used with requests. It can be a string or a dictionary with only the keys 'server', 'username', and 'password'. | ✔️ | | real_chrome | If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch and use an instance of your browser. | ✔️ | @@ -70,12 +70,12 @@ Scrapling provides many options with this fetcher and its session classes. Befor | additional_args | Additional arguments to be passed to Playwright's context as additional settings, and they take higher priority than Scrapling's settings. | ✔️ | | selector_config | A dictionary of custom parsing arguments to be used when creating the final `Selector`/`Response` class. | ✔️ | | blocked_domains | A set of domain names to block requests to. Subdomains are also matched (e.g., `"example.com"` blocks `"sub.example.com"` too). | ✔️ | -| block_ads | Block requests to ~3,500 known ad/tracking domains. Can be combined with `blocked_domains`. | ✔️ | +| block_ads | Block requests to ~3,500 known ad/tracking domains. Can be combined with `blocked_domains`. | ✔️ | | dns_over_https | Route DNS queries through Cloudflare's DNS-over-HTTPS to prevent DNS leaks when using proxies. | ✔️ | | proxy_rotator | A `ProxyRotator` instance for automatic proxy rotation. Cannot be combined with `proxy`. | ✔️ | | retries | Number of retry attempts for failed requests. Defaults to 3. | ✔️ | | retry_delay | Seconds to wait between retry attempts. Defaults to 1. | ✔️ | -| capture_xhr | Pass a regex URL pattern string to capture XHR/fetch requests matching it during page load. Captured responses are available via `response.captured_xhr`. Defaults to `None` (disabled). | ✔️ | +| capture_xhr | Pass a regex URL pattern string to capture XHR/fetch requests matching it during page load. Captured responses are available via `response.captured_xhr`. Defaults to `None` (disabled). | ✔️ | | executable_path | Absolute path to a custom browser executable to use instead of the bundled Chromium. Useful for non-standard installations or custom browser builds. | ✔️ | In session classes, all these arguments can be set globally for the session. Still, you can configure each request individually by passing some of the arguments here that can be configured on the browser tab level like: `google_search`, `timeout`, `wait`, `page_action`, `page_setup`, `extra_headers`, `disable_resources`, `wait_selector`, `wait_selector_state`, `network_idle`, `load_dom`, `solve_cloudflare`, `blocked_domains`, `proxy`, and `selector_config`. @@ -154,8 +154,8 @@ page = await StealthyFetcher.async_fetch('https://example.com', page_action=scro ```python # Wait for the selector page = StealthyFetcher.fetch( - 'https://example.com', - wait_selector='h1', + 'https://quotes.toscrape.com/js-delayed/', + wait_selector='.quote', wait_selector_state='visible' ) ``` diff --git a/docs/overview.md b/docs/overview.md index 5a72b95..8953255 100644 --- a/docs/overview.md +++ b/docs/overview.md @@ -292,12 +292,12 @@ We have you covered if you deal with dynamic websites like most today! The `DynamicFetcher` class (formerly `PlayWrightFetcher`) offers many options for fetching and loading web pages using Chromium-based browsers. ```python from scrapling.fetchers import DynamicFetcher -page = DynamicFetcher.fetch('https://www.google.com/search?q=%22Scrapling%22', disable_resources=True) # Vanilla Playwright option -page.css("#search a::attr(href)").get() # -> 'https://github.com/D4Vinci/Scrapling' +page = DynamicFetcher.fetch('https://quotes.toscrape.com/js/', disable_resources=True, block_ads=True) +print(len(page.css(".quote"))) # -> 10 # The async version of fetch -page = await DynamicFetcher.async_fetch('https://www.google.com/search?q=%22Scrapling%22', disable_resources=True) -page.css("#search a::attr(href)").get() # -> 'https://github.com/D4Vinci/Scrapling' +page = await DynamicFetcher.async_fetch('https://quotes.toscrape.com/js/', disable_resources=True, block_ads=True) +print(len(page.css(".quote"))) # -> 10 ``` It's built on top of [Playwright](https://playwright.dev/python/), and it's currently providing two main run options that can be mixed as you want: @@ -329,7 +329,7 @@ page.status == 200 # -> True page = StealthyFetcher.fetch('https://nopecha.com/demo/cloudflare', solve_cloudflare=True) # Solve Cloudflare captcha automatically if presented page.status == 200 # -> True -page = StealthyFetcher.fetch('https://www.browserscan.net/bot-detection', humanize=True, os_randomize=True) # and the rest of arguments... +page = StealthyFetcher.fetch('https://www.browserscan.net/bot-detection', block_webrtc=True, hide_canvas=True, dns_over_https=True) # and the rest of arguments... # The async version of fetch page = await StealthyFetcher.async_fetch('https://www.browserscan.net/bot-detection') page.status == 200 # -> True From b17363502d9bc9c0ecea7b36afd3375256f2764b Mon Sep 17 00:00:00 2001 From: yetval Date: Sat, 25 Apr 2026 22:13:38 -0400 Subject: [PATCH 07/20] fix: hash request kwargs and headers correctly --- scrapling/spiders/request.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/scrapling/spiders/request.py b/scrapling/spiders/request.py index ce728ee..faca233 100644 --- a/scrapling/spiders/request.py +++ b/scrapling/spiders/request.py @@ -97,15 +97,19 @@ class Request: } if include_kwargs: - kwargs = (key.lower() for key in self._session_kwargs.keys() if key.lower() not in ("data", "json")) - data["kwargs"] = "".join(set(_convert_to_bytes(key).hex() for key in kwargs)) + filtered_kwargs = { + key.lower(): str(value) + for key, value in self._session_kwargs.items() + if key.lower() not in ("data", "json") + } + data["kwargs"] = tuple(sorted(filtered_kwargs.items())) if include_headers: headers = self._session_kwargs.get("headers") or self._session_kwargs.get("extra_headers") or {} processed_headers = {} # Some header normalization for key, value in headers.items(): - processed_headers[_convert_to_bytes(key.lower()).hex()] = _convert_to_bytes(value.lower()).hex() + processed_headers[_convert_to_bytes(key.lower()).hex()] = _convert_to_bytes(value).hex() data["headers"] = tuple(processed_headers.items()) fp = hashlib.sha1(orjson.dumps(data, option=orjson.OPT_SORT_KEYS), usedforsecurity=False).digest() From a5a56529964520d484dee24b3b48d02158057fef Mon Sep 17 00:00:00 2001 From: yetval Date: Sun, 26 Apr 2026 16:31:58 -0400 Subject: [PATCH 08/20] test: add request fingerprint regressions --- tests/spiders/test_request.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/spiders/test_request.py b/tests/spiders/test_request.py index 997a71b..f54f2cd 100644 --- a/tests/spiders/test_request.py +++ b/tests/spiders/test_request.py @@ -99,6 +99,20 @@ class TestRequestProperties: r2 = Request("https://example.com/page2") assert r1.update_fingerprint() != r2.update_fingerprint() + def test_fingerprint_include_kwargs_uses_kwarg_values(self): + """Test kwargs with different values produce different fingerprints.""" + r1 = Request("https://example.com", timeout=1) + r2 = Request("https://example.com", timeout=2) + + assert r1.update_fingerprint(include_kwargs=True) != r2.update_fingerprint(include_kwargs=True) + + def test_fingerprint_include_headers_preserves_header_value_case(self): + """Test header values are fingerprinted without lowercasing.""" + r1 = Request("https://example.com", headers={"X-Test": "A"}) + r2 = Request("https://example.com", headers={"X-Test": "a"}) + + assert r1.update_fingerprint(include_headers=True) != r2.update_fingerprint(include_headers=True) + class TestRequestCopy: """Test Request copy functionality.""" From 334b0f55383208e3f8842fce878d54547338c35d Mon Sep 17 00:00:00 2001 From: yetval Date: Thu, 30 Apr 2026 09:52:34 -0400 Subject: [PATCH 09/20] fix: str(value) --- scrapling/spiders/request.py | 9 ++++++++- tests/spiders/test_request.py | 18 ++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/scrapling/spiders/request.py b/scrapling/spiders/request.py index faca233..ef766b2 100644 --- a/scrapling/spiders/request.py +++ b/scrapling/spiders/request.py @@ -22,6 +22,13 @@ def _convert_to_bytes(value: str | bytes) -> bytes: return value.encode(encoding="utf-8", errors="ignore") +def _stable_value_repr(value: Any) -> str: + try: + return orjson.dumps(value, option=orjson.OPT_SORT_KEYS, default=repr).decode() + except TypeError: + return repr(value) + + class Request: def __init__( self, @@ -98,7 +105,7 @@ class Request: if include_kwargs: filtered_kwargs = { - key.lower(): str(value) + key.lower(): _stable_value_repr(value) for key, value in self._session_kwargs.items() if key.lower() not in ("data", "json") } diff --git a/tests/spiders/test_request.py b/tests/spiders/test_request.py index f54f2cd..00f6c49 100644 --- a/tests/spiders/test_request.py +++ b/tests/spiders/test_request.py @@ -106,6 +106,24 @@ class TestRequestProperties: assert r1.update_fingerprint(include_kwargs=True) != r2.update_fingerprint(include_kwargs=True) + def test_fingerprint_include_kwargs_handles_non_primitive_values(self): + class _Opaque: + def __repr__(self) -> str: + return "_Opaque(stable)" + + opaque = _Opaque() + r1 = Request("https://example.com", proxies={"http": "p1"}, custom=opaque) + r2 = Request("https://example.com", proxies={"http": "p1"}, custom=opaque) + r3 = Request("https://example.com", proxies={"http": "p2"}, custom=opaque) + + fp1 = r1.update_fingerprint(include_kwargs=True) + r2._fp = None + fp2 = r2.update_fingerprint(include_kwargs=True) + fp3 = r3.update_fingerprint(include_kwargs=True) + + assert fp1 == fp2 + assert fp1 != fp3 + def test_fingerprint_include_headers_preserves_header_value_case(self): """Test header values are fingerprinted without lowercasing.""" r1 = Request("https://example.com", headers={"X-Test": "A"}) From 333b6de0b52c55145d57f3d36fa08d410107b87b Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Sat, 2 May 2026 19:58:53 +0300 Subject: [PATCH 10/20] fix(parser): change the default threshold and add warning --- scrapling/parser.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/scrapling/parser.py b/scrapling/parser.py index eced0a9..2b88af3 100644 --- a/scrapling/parser.py +++ b/scrapling/parser.py @@ -519,7 +519,7 @@ class Selector(SelectorsGeneration): def relocate( self, element: Union[Dict, HtmlElement, "Selector"], - percentage: int = 0, + percentage: int = 40, selector_type: bool = False, ) -> Union[List[HtmlElement], "Selectors"]: """This function will search again for the element in the page tree, used automatically on page structure change @@ -559,6 +559,10 @@ class Selector(SelectorsGeneration): if not selector_type: return score_table[highest_probability] return self.__elements_convertor(score_table[highest_probability]) + log.warning( + f"Adaptive relocation found no element above the {percentage}% threshold " + f"(top score: {highest_probability}%). Lower `percentage` if this is the right element." + ) return [] def css( @@ -567,7 +571,7 @@ class Selector(SelectorsGeneration): identifier: str = "", adaptive: bool = False, auto_save: bool = False, - percentage: int = 0, + percentage: int = 40, ) -> "Selectors": """Search the current tree with CSS3 selectors @@ -627,7 +631,7 @@ class Selector(SelectorsGeneration): identifier: str = "", adaptive: bool = False, auto_save: bool = False, - percentage: int = 0, + percentage: int = 40, **kwargs: Any, ) -> "Selectors": """Search the current tree with XPath selectors @@ -1220,7 +1224,7 @@ class Selectors(List[Selector]): selector: str, identifier: str = "", auto_save: bool = False, - percentage: int = 0, + percentage: int = 40, **kwargs: Any, ) -> "Selectors": """ @@ -1251,7 +1255,7 @@ class Selectors(List[Selector]): selector: str, identifier: str = "", auto_save: bool = False, - percentage: int = 0, + percentage: int = 40, ) -> "Selectors": """ Call the ``.css()`` method for each element in this list and return From 18e912113562ad4465e05fbd0f5d6298d2de33a3 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Sun, 10 May 2026 21:12:36 +0300 Subject: [PATCH 11/20] feat(spiders): Add pure URL discovery primitive --- scrapling/spiders/links.py | 295 +++++++++++++++++++++++++++++++++++++ 1 file changed, 295 insertions(+) create mode 100644 scrapling/spiders/links.py diff --git a/scrapling/spiders/links.py b/scrapling/spiders/links.py new file mode 100644 index 0000000..7b9a4ff --- /dev/null +++ b/scrapling/spiders/links.py @@ -0,0 +1,295 @@ +"""Pure URL discovery primitive""" + +import re +from urllib.parse import urlsplit + +from w3lib.html import strip_html5_whitespace +from w3lib.url import canonicalize_url, safe_url_string + +from scrapling.core._types import ( + TYPE_CHECKING, + Iterable, + Callable, + List, + Optional, + Pattern, + Set, + Tuple, + Union, + Any, +) +from scrapling.core.utils import log + +if TYPE_CHECKING: + from scrapling.engines.toolbelt.custom import Response + + +__all__ = ["LinkExtractor"] +valid_schemas = {"http", "https", "file"} + + +IGNORED_EXTENSIONS = { + # archives + "7z", + "7zip", + "bz2", + "rar", + "tar", + "tar.gz", + "xz", + "zip", + # images + "mng", + "pct", + "bmp", + "gif", + "jpg", + "jpeg", + "png", + "pst", + "psp", + "tif", + "tiff", + "ai", + "drw", + "dxf", + "eps", + "ps", + "svg", + "cdr", + "ico", + "webp", + # audio + "mp3", + "wma", + "ogg", + "wav", + "ra", + "aac", + "mid", + "au", + "aiff", + # video + "3gp", + "asf", + "asx", + "avi", + "mov", + "mp4", + "mpg", + "qt", + "rm", + "swf", + "wmv", + "m4a", + "m4v", + "flv", + "webm", + # office suites + "xls", + "xlsm", + "xlsx", + "xltm", + "xltx", + "potm", + "potx", + "ppt", + "pptm", + "pptx", + "pps", + "doc", + "docb", + "docm", + "docx", + "dotm", + "dotx", + "odt", + "ods", + "odg", + "odp", + # other + "css", + "pdf", + "exe", + "bin", + "rss", + "dmg", + "iso", + "apk", + "jar", + "sh", + "rb", + "js", + "hta", + "bat", + "cpl", + "msi", + "msp", + "py", +} + + +PatternInput = Iterable[Union[str, Pattern[str]]] +StrOrIterable = Union[str, Iterable[str]] + + +def _to_str_tuple(value: StrOrIterable) -> Tuple[str, ...]: + if not value: + return () + if isinstance(value, str): + return (value,) + return tuple(value) + + +def _compile_patterns(patterns: Union[str, Pattern[str], PatternInput, None]) -> Tuple[Pattern[str], ...]: + if not patterns: + return () + if isinstance(patterns, (str, re.Pattern)): + patterns = (patterns,) + return tuple(p if isinstance(p, re.Pattern) else re.compile(p) for p in patterns) + + +def _url_extension(url: str) -> str: + path = urlsplit(url).path + _, _, last = path.rpartition("/") + if "." not in last: + return "" + return last.rsplit(".", 1)[1].lower() + + +def _filler(x): + return x + + +class LinkExtractor: + """Extracts and filters URLs from a `Response` (or a single URL via `matches`). + + All matching is regex-based; allow/deny patterns can be plain strings (compiled + with `re.compile`) or pre-compiled `re.Pattern` objects, individually or as an + iterable. + + :param allow: Regex pattern(s) URLs must match to be kept. String, compiled `re.Pattern`, + or an iterable of either. Empty means match all. + :param deny: Regex pattern(s) URLs must NOT match. Takes precedence over `allow`. + :param allow_domains: Domain(s) to keep. Matches the exact host or any subdomain + (e.g. `"example.com"` matches `"api.example.com"`). String or iterable. + :param deny_domains: Domain(s) to exclude. Same matching rules as `allow_domains`. + :param restrict_css: CSS selectors to scope DOM extraction to. Empty means whole page. + :param restrict_xpath: XPath selectors to scope DOM extraction to. Empty means whole page. + :param tags: Element tags to look for links in. Default ("a", "area"). + :param attrs: Attributes on those tags to read URLs from. Default ("href",). + :param canonicalize: Canonicalize URLs (sort query params, normalize path). Default True. + :param strip: Strip whitespace from extracted URLs. Default True. + :param keep_fragment: Preserve the URL fragment when canonicalizing. Default False. + :param deny_extensions: File extensions to drop. Default `IGNORED_EXTENSIONS`. + :param process: A function to do a process on the values extracted before using them. Return None to drop any value. + """ + + def __init__( + self, + allow: Union[str, Pattern[str], PatternInput] = (), + deny: Union[str, Pattern[str], PatternInput] = (), + allow_domains: StrOrIterable = (), + deny_domains: StrOrIterable = (), + restrict_css: StrOrIterable = (), + restrict_xpath: StrOrIterable = (), + tags: Iterable[str] = ("a", "area"), + attrs: Iterable[str] = ("href",), + canonicalize: bool = True, + strip: bool = True, + keep_fragment: bool = False, + deny_extensions: Optional[Iterable[str]] = None, + process: Callable[[Any], Any] | None = None, + ) -> None: + self.allow: Tuple[Pattern[str], ...] = _compile_patterns(allow) + self.deny: Tuple[Pattern[str], ...] = _compile_patterns(deny) + self.allow_domains: Tuple[str, ...] = tuple(d.lower() for d in _to_str_tuple(allow_domains)) + self.deny_domains: Tuple[str, ...] = tuple(d.lower() for d in _to_str_tuple(deny_domains)) + self.restrict_css: Tuple[str, ...] = _to_str_tuple(restrict_css) + self.restrict_xpath: Tuple[str, ...] = _to_str_tuple(restrict_xpath) + self.tags: Tuple[str, ...] = tuple(tags) + self.attrs: Tuple[str, ...] = tuple(attrs) + self.canonicalize = canonicalize + self.strip = strip + self.keep_fragment = keep_fragment + self.deny_extensions: Set[str] = set( + (ext.lower().lstrip(".") for ext in deny_extensions) if deny_extensions is not None else IGNORED_EXTENSIONS + ) + self.process: Callable[[Any], Any] = process if callable(process) else _filler + + def extract(self, response: "Response") -> List[str]: + """Return absolute, filtered, deduped URLs from `response`.""" + scopes: List[Any] = [] + if self.restrict_xpath: + for xp in self.restrict_xpath: + scopes.extend(response.xpath(xp)) + if self.restrict_css: + for cs in self.restrict_css: + scopes.extend(response.css(cs)) + if not scopes: + scopes = [response] + + out: List[str] = [] + search_selector = "| ".join([f".//{tag}/@{attr}" for tag in self.tags for attr in self.attrs]) + for scope in scopes: + for url in scope._root.xpath(search_selector): + if not url: + continue + url = str(url) + if self.strip: + url = strip_html5_whitespace(url) + if not url: + continue + url = str(response.urljoin(url)) + url = self.process(url) + if not url: + continue + + if self.canonicalize: + url = canonicalize_url(url, keep_fragments=self.keep_fragment) + + try: + url = safe_url_string(url, encoding=response.encoding) + except ValueError: + log.debug(f"Skipping the extraction of bad URL {url!r}") + continue + + if not self._url_passes(url): + continue + + out.append(url) + + # Switching to dict for deduplication instead of Set will keep the insertion order of the links. + return list(dict.fromkeys(out)) + + def matches(self, url: str) -> bool: + """URL-only filter (no response extraction). + + Applies allow/deny/allow_domains/deny_domains/deny_extensions to a single URL. + Used by `SitemapSpider` to dispatch sitemap URLs through `CrawlRule`s without + needing a `Response`. + """ + if self.canonicalize: + url = canonicalize_url(url, keep_fragments=self.keep_fragment) + return self._url_passes(url) + + def _url_passes(self, url: str) -> bool: + if url.split("://", 1)[0] not in valid_schemas: + return False + + ext = _url_extension(url) + if ext and ext in self.deny_extensions: + return False + + if self.allow and not any(p.search(url) for p in self.allow): + return False + if self.deny and any(p.search(url) for p in self.deny): + return False + + if self.allow_domains or self.deny_domains: + host = (urlsplit(url).hostname or "").lower() + if self.allow_domains and not any(host == d or host.endswith("." + d) for d in self.allow_domains): + return False + if self.deny_domains and any(host == d or host.endswith("." + d) for d in self.deny_domains): + return False + return True From f093d0cf12f129e451b4029593e791356e48e84f Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Sun, 10 May 2026 21:13:16 +0300 Subject: [PATCH 12/20] feat(spiders): Add CrawlSpider and CrawlRule --- scrapling/spiders/templates/__init__.py | 6 +++ scrapling/spiders/templates/crawler.py | 72 +++++++++++++++++++++++++ 2 files changed, 78 insertions(+) create mode 100644 scrapling/spiders/templates/__init__.py create mode 100644 scrapling/spiders/templates/crawler.py diff --git a/scrapling/spiders/templates/__init__.py b/scrapling/spiders/templates/__init__.py new file mode 100644 index 0000000..31d3ad1 --- /dev/null +++ b/scrapling/spiders/templates/__init__.py @@ -0,0 +1,6 @@ +from .crawler import CrawlSpider, CrawlRule + +__all__ = [ + "CrawlSpider", + "CrawlRule", +] diff --git a/scrapling/spiders/templates/crawler.py b/scrapling/spiders/templates/crawler.py new file mode 100644 index 0000000..342528c --- /dev/null +++ b/scrapling/spiders/templates/crawler.py @@ -0,0 +1,72 @@ +"""Generic spider templates that build on the `Spider` base.""" + +from dataclasses import dataclass + +from scrapling.spiders.links import LinkExtractor +from scrapling.spiders.request import Request +from scrapling.spiders.spider import Spider +from scrapling.core._types import ( + TYPE_CHECKING, + Any, + AsyncGenerator, + Callable, + Dict, + List, + Optional, + Union, +) + +if TYPE_CHECKING: + from scrapling.engines.toolbelt.custom import Response + + +__all__ = ["CrawlRule", "CrawlSpider"] + + +ParseCallback = Callable[ + ["Response"], + AsyncGenerator[Union[Dict[str, Any], Request, None], None], +] +ProcessRequestFn = Callable[[Request, "Response"], Request] + + +@dataclass +class CrawlRule: + """Rule for `CrawlSpider`: extract links from a response and dispatch them. + + :param link_extractor: `LinkExtractor` that produces URLs from each response. + :param callback: Bound method on the spider to call for each matched URL. + Falls back to the spider's default ``parse()`` by default. + :param priority: Override the priority of the requests that will be dispatched. + :param process_request: Optional bound method to mutate each `Request` before + it is yielded. Signature: ``(request, response) -> request``. Use it to + add headers, change priority, or filter requests. + """ + + link_extractor: LinkExtractor + callback: Optional[ParseCallback] = None + priority: Optional[int] = None + process_request: Optional[ProcessRequestFn] = None + + +class CrawlSpider(Spider): + """A generic spider that can extract and follow links automatically based on crawl rules. + + Override `rules()` to return a list of `CrawlRule`s. + + You can start from it and override it as needed for more custom functionality, or just implement your own spider. + """ + + def rules(self) -> List[CrawlRule]: + """Override to define link-following rules.""" + return [] + + async def parse(self, response: "Response") -> AsyncGenerator[Union[Dict[str, Any], Request, None], None]: + for rule in self.rules(): + for url in rule.link_extractor.extract(response): + req = response.follow(url, callback=rule.callback) + if rule.priority: + req.priority = rule.priority + if rule.process_request is not None: + req = rule.process_request(req, response) + yield req From f7da15771bdb9b164e8b0f2bd21403c77a05e9ae Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Mon, 11 May 2026 02:33:37 +0300 Subject: [PATCH 13/20] feat(spiders): Add SitemapSpider --- scrapling/spiders/__init__.py | 6 + scrapling/spiders/templates/__init__.py | 2 + scrapling/spiders/templates/crawler.py | 2 +- scrapling/spiders/templates/sitemap.py | 193 ++++++++++++++++++++++++ 4 files changed, 202 insertions(+), 1 deletion(-) create mode 100644 scrapling/spiders/templates/sitemap.py diff --git a/scrapling/spiders/__init__.py b/scrapling/spiders/__init__.py index 92eb2e9..b455076 100644 --- a/scrapling/spiders/__init__.py +++ b/scrapling/spiders/__init__.py @@ -4,6 +4,8 @@ from .scheduler import Scheduler from .engine import CrawlerEngine from .session import SessionManager from .spider import Spider, SessionConfigurationError +from .links import LinkExtractor +from .templates import CrawlSpider, SitemapSpider, CrawlRule from scrapling.engines.toolbelt.custom import Response __all__ = [ @@ -15,4 +17,8 @@ __all__ = [ "SessionManager", "Scheduler", "Response", + "LinkExtractor", + "CrawlSpider", + "CrawlRule", + "SitemapSpider", ] diff --git a/scrapling/spiders/templates/__init__.py b/scrapling/spiders/templates/__init__.py index 31d3ad1..fa758c7 100644 --- a/scrapling/spiders/templates/__init__.py +++ b/scrapling/spiders/templates/__init__.py @@ -1,6 +1,8 @@ from .crawler import CrawlSpider, CrawlRule +from .sitemap import SitemapSpider __all__ = [ "CrawlSpider", "CrawlRule", + "SitemapSpider", ] diff --git a/scrapling/spiders/templates/crawler.py b/scrapling/spiders/templates/crawler.py index 342528c..5c6aeaa 100644 --- a/scrapling/spiders/templates/crawler.py +++ b/scrapling/spiders/templates/crawler.py @@ -65,7 +65,7 @@ class CrawlSpider(Spider): for rule in self.rules(): for url in rule.link_extractor.extract(response): req = response.follow(url, callback=rule.callback) - if rule.priority: + if rule.priority is not None: req.priority = rule.priority if rule.process_request is not None: req = rule.process_request(req, response) diff --git a/scrapling/spiders/templates/sitemap.py b/scrapling/spiders/templates/sitemap.py new file mode 100644 index 0000000..bda8d3f --- /dev/null +++ b/scrapling/spiders/templates/sitemap.py @@ -0,0 +1,193 @@ +"""Sitemap template spider.""" + +from dataclasses import dataclass, field +from gzip import GzipFile +from io import BytesIO +from urllib.parse import urlsplit + +from lxml import etree +from protego import Protego + +from scrapling.core._types import ( + TYPE_CHECKING, + Any, + AsyncGenerator, + Dict, + List, + Optional, + Union, +) +from scrapling.spiders.links import LinkExtractor +from scrapling.spiders.request import Request +from scrapling.spiders.spider import Spider +from scrapling.spiders.templates.crawler import CrawlRule + +if TYPE_CHECKING: + from scrapling.engines.toolbelt.custom import Response + + +__all__ = ["SitemapSpider"] + + +_GZIP_MAGIC = b"\x1f\x8b" +_GUNZIP_MAX_SIZE = 64 * 1024 * 1024 # 64 MiB cap, defends against gzip bombs + + +@dataclass +class SitemapResult: + """Parsed sitemap body. + + `urls` holds the entries from a ``; `sitemaps` holds child sitemap + URLs from a `` (each of which is fetched recursively). + """ + + urls: List[str] = field(default_factory=list) + sitemaps: List[str] = field(default_factory=list) + + +class SitemapSpider(Spider): + """A Spider that seeds a crawl from sitemap(s), and follows the rules. + + Override `rules()` to return a list of `CrawlRule`s. + + If there are no rules provided, all non-sitemap urls will be redirected to `parse()`, which must be overridden or it will raise `NotImplementedError`. + + :cvar sitemap_urls: Explicit list of sitemap (or robots.txt) URLs to fetch. + :cvar sitemap_follow: `LinkExtractor` filtering which child sitemaps inside a + `` to descend into. ``None`` means descend into all. + :cvar sitemap_alternate_links: When enabled, alternate-language URLs are also + routed through `rules()`. + """ + + sitemap_urls: List[str] = [] + sitemap_follow: Optional[LinkExtractor] = None + sitemap_alternate_links: bool = False + + def rules(self) -> List[CrawlRule]: + """Override to define dispatch rules for sitemap URLs.""" + return [] + + async def start_requests(self) -> AsyncGenerator[Request, None]: + if self.sitemap_urls: + for url in self.sitemap_urls: + yield Request(url, callback=self._parse_sitemap) + return + + raise RuntimeError("`SitemapSpider` needs `sitemap_urls` to be set.") + + async def parse(self, response: "Response") -> AsyncGenerator[Union[Dict[str, Any], Request, None], None]: + """Default callback for processing responses""" + raise NotImplementedError(f"{self.__class__.__name__} must implement parse() method") + yield # Make this a generator for type checkers + + def _robots_body(self, response: "Response") -> List[str]: + """Extract `Sitemap` directives from a robots.txt body via protego.""" + try: + text = response.body.decode(response.encoding, errors="replace") + parser = Protego.parse(text) + except Exception as e: + self.logger.warning(f"Failed to parse robots.txt: {e}") + return [] + return list(parser.sitemaps) + + @staticmethod + def _decompress(body: bytes, content_type: Optional[str]) -> bytes: + if (content_type and ("gzip" in content_type.lower())) or (body[:2] == _GZIP_MAGIC): + out = bytearray() + with GzipFile(fileobj=BytesIO(body)) as f: + while chunk := f.read1(8192): + out.extend(chunk) + if len(out) > _GUNZIP_MAX_SIZE: + raise OSError(f"gzip output exceeds {_GUNZIP_MAX_SIZE} bytes") + return bytes(out) + return body + + def _extract_urls(self, root: Any) -> List[str]: + urls: List[str] = [] + for url_el in root: + if self._get_type(url_el) != "url": + continue + + for child in url_el: + name = self._get_type(child) + if name == "loc" and child.text: + urls.append(child.text.strip()) + elif self.sitemap_alternate_links and name == "link": + href = child.get("href") + if href: + urls.append(href.strip()) + return urls + + @staticmethod + def _get_type(el: Any) -> str: + return etree.QName(el.tag).localname + + def _sm_body(self, body: bytes, content_type: Optional[str] = None) -> SitemapResult: + """Parse a sitemap body and return its URLs and any child sitemaps.""" + try: + body = self._decompress(body, content_type) + except OSError as e: + self.logger.warning(f"Failed to decompress sitemap: {e}") + return SitemapResult() + + try: + root = etree.fromstring(body) + except etree.XMLSyntaxError as e: + self.logger.warning(f"Failed to parse sitemap XML: {e}") + return SitemapResult() + + root_name = self._get_type(root) + if root_name == "sitemapindex": + locs = [] + for sm_el in root: + if self._get_type(sm_el) == "sitemap": + for child in sm_el: + if self._get_type(child) == "loc" and child.text: + locs.append(child.text.strip()) + break + return SitemapResult(sitemaps=locs) + if root_name == "urlset": + return SitemapResult(urls=self._extract_urls(root)) + + self.logger.warning(f"Unknown sitemap root element: {root_name!r}") + return SitemapResult() + + async def _parse_sitemap(self, response: "Response") -> AsyncGenerator[Union[Dict[str, Any], Request, None], None]: + if urlsplit(response.url).path.endswith("/robots.txt"): + sitemaps = self._robots_body(response) + if not sitemaps: + self.logger.warning(f"No Sitemaps found in {response.url}") + + for sitemap_url in sitemaps: + yield response.follow(sitemap_url, callback=self._parse_sitemap) + return + + content_type = response.headers.get("content-type") if response.headers else None + result = self._sm_body(response.body, content_type=content_type) + + # Descend into child sitemaps (apply sitemap_follow filter if present) + for child_url in result.sitemaps: + if self.sitemap_follow is not None and not self.sitemap_follow.matches(child_url): + continue + yield response.follow(child_url, callback=self._parse_sitemap) + + # Dispatch each URL through rules() (first match wins; unmatched drop unless rules empty) + rules = self.rules() + for url in result.urls: + req = self._dispatch(response, url, rules) + if req is not None: + yield req + + @staticmethod + def _dispatch(response: "Response", url: str, rules: List[CrawlRule]) -> Optional[Request]: + if not rules: + return response.follow(url) + for rule in rules: + if rule.link_extractor.matches(url): + req = response.follow(url, callback=rule.callback) + if rule.priority is not None: + req.priority = rule.priority + if rule.process_request is not None: + req = rule.process_request(req, response) + return req + return None From 8a8b2d14ba9afa1a7fbeb19c9f5bb6a34a7c7c6b Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Mon, 11 May 2026 02:33:43 +0300 Subject: [PATCH 14/20] test: add tests accordingly --- tests/spiders/test_links.py | 243 ++++++++++++++++++++ tests/spiders/test_sitemap.py | 392 ++++++++++++++++++++++++++++++++ tests/spiders/test_templates.py | 230 +++++++++++++++++++ 3 files changed, 865 insertions(+) create mode 100644 tests/spiders/test_links.py create mode 100644 tests/spiders/test_sitemap.py create mode 100644 tests/spiders/test_templates.py diff --git a/tests/spiders/test_links.py b/tests/spiders/test_links.py new file mode 100644 index 0000000..2413992 --- /dev/null +++ b/tests/spiders/test_links.py @@ -0,0 +1,243 @@ +"""Tests for `LinkExtractor`.""" + +import re + +import pytest + +from scrapling.engines.toolbelt.custom import Response +from scrapling.spiders.links import IGNORED_EXTENSIONS, LinkExtractor + + +def _make_response(html: str, url: str = "https://example.com/page") -> Response: + """Build a minimal Response wrapping the given HTML.""" + return Response( + url=url, + content=html, + status=200, + reason="OK", + cookies={}, + headers={}, + request_headers={}, + ) + + +HTML_BASIC = """ + + post 1 + post 2 + external + about + mail + js + pdf + area + + +""" + + +class TestExtractBasic: + def test_default_extracts_a_and_area_with_href(self): + resp = _make_response(HTML_BASIC) + urls = LinkExtractor().extract(resp) + # mailto/javascript filtered (non-http scheme), .pdf filtered (deny_extensions) + # link[rel=stylesheet] not in default tags + assert "https://example.com/posts/1" in urls + assert "https://example.com/posts/2" in urls + assert "https://example.com/about" in urls + assert "https://example.com/area-link" in urls + assert "https://other.com/page" in urls + assert all(not u.startswith("mailto:") for u in urls) + assert all(not u.startswith("javascript:") for u in urls) + assert not any(u.endswith(".pdf") for u in urls) + assert not any(u.endswith(".css") for u in urls) + + def test_relative_urls_become_absolute_via_urljoin(self): + resp = _make_response('x', url="https://example.com/sub/") + assert LinkExtractor().extract(resp) == ["https://example.com/sub/foo/bar"] + + def test_empty_allow_means_match_all(self): + resp = _make_response('xy') + out = LinkExtractor().extract(resp) + assert len(out) == 2 + + +class TestAllowDeny: + def test_allow_regex_filters_in(self): + resp = _make_response(HTML_BASIC) + urls = LinkExtractor(allow=r"/posts/").extract(resp) + assert urls == ["https://example.com/posts/1", "https://example.com/posts/2"] + + def test_deny_regex_filters_out(self): + resp = _make_response(HTML_BASIC) + urls = LinkExtractor(deny=r"/posts/").extract(resp) + assert "https://example.com/posts/1" not in urls + assert "https://example.com/about" in urls + + def test_deny_overrides_allow(self): + resp = _make_response(HTML_BASIC) + urls = LinkExtractor(allow=r"/posts/", deny=r"/posts/2").extract(resp) + assert urls == ["https://example.com/posts/1"] + + def test_compiled_pattern_accepted(self): + resp = _make_response(HTML_BASIC) + pat = re.compile(r"/posts/\d+$") + urls = LinkExtractor(allow=pat).extract(resp) + assert len(urls) == 2 + + def test_iterable_of_patterns(self): + resp = _make_response(HTML_BASIC) + urls = LinkExtractor(allow=[r"/posts/", r"/about"]).extract(resp) + assert "https://example.com/posts/1" in urls + assert "https://example.com/about" in urls + + +class TestDomains: + def test_allow_domains_keeps_only_matching(self): + resp = _make_response(HTML_BASIC) + urls = LinkExtractor(allow_domains="example.com").extract(resp) + assert all("example.com" in u for u in urls) + assert "https://other.com/page" not in urls + + def test_allow_domains_matches_subdomains(self): + html = 'ab' + resp = _make_response(html) + urls = LinkExtractor(allow_domains="example.com").extract(resp) + assert urls == ["https://api.example.com/x"] + + def test_deny_domains_filters_out(self): + resp = _make_response(HTML_BASIC) + urls = LinkExtractor(deny_domains="other.com").extract(resp) + assert "https://other.com/page" not in urls + assert "https://example.com/posts/1" in urls + + +class TestRestrict: + def test_restrict_css_scopes_extraction(self): + html = """ + + +
m
+ + """ + resp = _make_response(html) + urls = LinkExtractor(restrict_css="main").extract(resp) + assert urls == ["https://example.com/main-link"] + + def test_restrict_xpath_scopes_extraction(self): + html = """ + + +
c
+ + """ + resp = _make_response(html) + urls = LinkExtractor(restrict_xpath='//div[@id="content"]').extract(resp) + assert urls == ["https://example.com/c"] + + +class TestTagsAttrs: + def test_custom_tags_and_attrs_for_stylesheets(self): + html = 'p' + resp = _make_response(html) + # Override deny_extensions to allow .css through, and pick up + urls = LinkExtractor(tags=("link",), attrs=("href",), deny_extensions=()).extract(resp) + assert urls == ["https://example.com/style.css"] + + +class TestCanonicalization: + def test_query_params_sorted(self): + resp = _make_response('x') + urls = LinkExtractor().extract(resp) + assert urls == ["https://example.com/x?a=1&b=2"] + + def test_fragment_dropped_by_default(self): + resp = _make_response('x') + urls = LinkExtractor().extract(resp) + assert urls == ["https://example.com/x"] + + def test_keep_fragment_preserves_it(self): + resp = _make_response('x') + urls = LinkExtractor(keep_fragment=True).extract(resp) + assert urls == ["https://example.com/x#section"] + + def test_canonicalize_off_leaves_url_unchanged(self): + resp = _make_response('x') + urls = LinkExtractor(canonicalize=False).extract(resp) + assert urls == ["https://example.com/x?b=2&a=1#f"] + + +class TestDedup: + def test_unique_drops_duplicates(self): + html = 'abc' + resp = _make_response(html) + urls = LinkExtractor().extract(resp) + # canonicalize collapses /x and /x? together + assert urls == ["https://example.com/x"] + + +class TestExtensions: + def test_default_deny_extensions_drops_pdf_zip_images(self): + html = 'pdfzippngok' + resp = _make_response(html) + urls = LinkExtractor().extract(resp) + assert urls == ["https://example.com/d"] + + def test_custom_deny_extensions_overrides_default(self): + html = 'pdfzip' + resp = _make_response(html) + urls = LinkExtractor(deny_extensions={"zip"}).extract(resp) + # .pdf now allowed because we replaced the set + assert urls == ["https://example.com/a.pdf"] + + def test_empty_deny_extensions_allows_everything(self): + html = 'pdf' + resp = _make_response(html) + urls = LinkExtractor(deny_extensions=()).extract(resp) + assert urls == ["https://example.com/a.pdf"] + + +class TestStrip: + def test_strip_removes_whitespace(self): + resp = _make_response('x') + urls = LinkExtractor().extract(resp) + assert urls == ["https://example.com/spaced"] + + +class TestMatches: + def test_matches_honors_allow(self): + ex = LinkExtractor(allow=r"/posts/") + assert ex.matches("https://example.com/posts/1") is True + assert ex.matches("https://example.com/about") is False + + def test_matches_honors_deny(self): + ex = LinkExtractor(deny=r"/admin") + assert ex.matches("https://example.com/admin/x") is False + assert ex.matches("https://example.com/posts/1") is True + + def test_matches_honors_allow_domains(self): + ex = LinkExtractor(allow_domains="example.com") + assert ex.matches("https://api.example.com/x") is True + assert ex.matches("https://other.com/x") is False + + def test_matches_honors_deny_extensions(self): + ex = LinkExtractor() + assert ex.matches("https://example.com/file.pdf") is False + assert ex.matches("https://example.com/page") is True + + def test_matches_rejects_non_http_schemes(self): + ex = LinkExtractor() + assert ex.matches("mailto:x@example.com") is False + assert ex.matches("javascript:void(0)") is False + assert ex.matches("ftp://example.com/x") is False + + def test_matches_canonicalizes_before_checking(self): + ex = LinkExtractor(allow=r"a=1&b=2$") + # the URL has params in the wrong order; canonicalize sorts them + assert ex.matches("https://example.com/x?b=2&a=1") is True + + +class TestIgnoredExtensions: + def test_constant_includes_common_binary_types(self): + for ext in ("pdf", "zip", "png", "mp4", "exe"): + assert ext in IGNORED_EXTENSIONS diff --git a/tests/spiders/test_sitemap.py b/tests/spiders/test_sitemap.py new file mode 100644 index 0000000..cd5739b --- /dev/null +++ b/tests/spiders/test_sitemap.py @@ -0,0 +1,392 @@ +"""Tests for `SitemapParser` and `SitemapSpider`.""" + +import gzip +import pickle + +import pytest + +from scrapling.engines.toolbelt.custom import Response +from scrapling.spiders.links import LinkExtractor +from scrapling.spiders.request import Request +from scrapling.spiders.sitemap import SitemapParser, SitemapResult, SitemapSpider, SitemapUrl +from scrapling.spiders.templates import CrawlRule +from scrapling.core._types import Any, AsyncGenerator, Dict, Union + + +URLSET_XML = b""" + + + https://example.com/posts/1 + 2026-01-15 + daily + 0.8 + + + https://example.com/posts/2 + 2026-02-20 + + + https://example.com/about + + +""" + +URLSET_WITH_ALTERNATES = b""" + + + https://example.com/en/page + + + + +""" + +INDEX_XML = b""" + + https://example.com/posts-sitemap.xml + https://example.com/products-sitemap.xml + https://example.com/skip-sitemap.xml + +""" + +# Sitemap without the standard namespace (some sites do this) +URLSET_NO_NS = b""" + + https://example.com/x + +""" + + +class TestSitemapParserUrlset: + def test_parse_urlset_with_full_metadata(self): + result = SitemapParser().parse(URLSET_XML) + assert len(result.urls) == 3 + assert result.sitemaps == [] + first = result.urls[0] + assert first.loc == "https://example.com/posts/1" + assert first.lastmod == "2026-01-15" + assert first.changefreq == "daily" + assert first.priority == 0.8 + + def test_parse_handles_partial_metadata(self): + result = SitemapParser().parse(URLSET_XML) + second = result.urls[1] + assert second.lastmod == "2026-02-20" + assert second.changefreq is None + assert second.priority is None + + def test_parse_handles_no_namespace(self): + result = SitemapParser().parse(URLSET_NO_NS) + assert len(result.urls) == 1 + assert result.urls[0].loc == "https://example.com/x" + + +class TestSitemapParserAlternates: + def test_alternate_links_off_by_default(self): + result = SitemapParser().parse(URLSET_WITH_ALTERNATES) + assert result.urls[0].alternates == [] + + def test_alternate_links_on_collects_them(self): + result = SitemapParser(alternate_links=True).parse(URLSET_WITH_ALTERNATES) + assert result.urls[0].alternates == [ + "https://example.com/fr/page", + "https://example.com/de/page", + ] + + +class TestSitemapParserIndex: + def test_parse_sitemapindex_returns_child_sitemaps(self): + result = SitemapParser().parse(INDEX_XML) + assert result.urls == [] + assert result.sitemaps == [ + "https://example.com/posts-sitemap.xml", + "https://example.com/products-sitemap.xml", + "https://example.com/skip-sitemap.xml", + ] + + +class TestSitemapParserDecompression: + def test_gz_body_via_magic_bytes(self): + compressed = gzip.compress(URLSET_XML) + result = SitemapParser().parse(compressed) + assert len(result.urls) == 3 + + def test_gz_body_via_content_type_hint(self): + compressed = gzip.compress(URLSET_XML) + result = SitemapParser().parse(compressed, content_type="application/x-gzip") + assert len(result.urls) == 3 + + def test_corrupt_gz_logged_not_raised(self): + # Body starts with gzip magic but is not valid gzip + body = b"\x1f\x8b" + b"junk data" + result = SitemapParser().parse(body) + assert result == SitemapResult() + + +class TestSitemapParserMalformed: + def test_invalid_xml_returns_empty_result(self): + result = SitemapParser().parse(b"") + assert result == SitemapResult() + + +class TestFromRobotsTxt: + def test_extracts_sitemap_directives(self): + body = """ + User-agent: * + Disallow: /admin + Sitemap: https://example.com/sitemap.xml + Sitemap: https://example.com/posts-sitemap.xml + """ + urls = SitemapParser.from_robots_txt(body) + assert urls == [ + "https://example.com/sitemap.xml", + "https://example.com/posts-sitemap.xml", + ] + + def test_ignores_comments_and_blank_lines(self): + body = """ + # This is a comment + Sitemap: https://example.com/sitemap.xml # inline comment + # Sitemap: https://commented.example.com/sitemap.xml + """ + urls = SitemapParser.from_robots_txt(body) + assert urls == ["https://example.com/sitemap.xml"] + + def test_directive_match_is_case_insensitive(self): + body = "SITEMAP: https://example.com/sitemap.xml\nsitemap: https://example.com/other.xml" + urls = SitemapParser.from_robots_txt(body) + assert urls == [ + "https://example.com/sitemap.xml", + "https://example.com/other.xml", + ] + + def test_returns_empty_when_no_directives(self): + body = "User-agent: *\nDisallow: /admin" + urls = SitemapParser.from_robots_txt(body) + assert urls == [] + + +def _make_response(body: bytes, url: str = "https://example.com/sitemap.xml", headers: dict | None = None) -> Response: + resp = Response( + url=url, + content=body, + status=200, + reason="OK", + cookies={}, + headers=headers or {}, + request_headers={}, + ) + resp.request = Request(url, sid="default") + return resp + + +async def _collect(agen: AsyncGenerator) -> list: + return [item async for item in agen] + + +class TestSitemapSpiderFlow: + @pytest.mark.asyncio + async def test_urlset_dispatched_through_rules(self): + class S(SitemapSpider): + name = "s" + sitemap_urls = ["https://example.com/sitemap.xml"] + + def rules(self): + return [CrawlRule(LinkExtractor(allow=r"/posts/"), callback=self.parse_post)] + + async def parse_post(self, response): + yield {"post": response.url} + + spider = S() + out = await _collect(spider._parse_sitemap(_make_response(URLSET_XML))) + post_reqs = [r for r in out if "/posts/" in r.url] + about_reqs = [r for r in out if "/about" in r.url] + # Two posts dispatched to parse_post; about falls through (callback inherited from None → None) + assert len(post_reqs) == 2 + assert all(r.callback == spider.parse_post for r in post_reqs) + assert len(about_reqs) == 1 + assert about_reqs[0].callback is None + + @pytest.mark.asyncio + async def test_no_rules_means_all_urls_fall_through(self): + class S(SitemapSpider): + name = "s" + sitemap_urls = ["https://example.com/sitemap.xml"] + + spider = S() + out = await _collect(spider._parse_sitemap(_make_response(URLSET_XML))) + assert len(out) == 3 + assert all(r.callback is None for r in out) + + @pytest.mark.asyncio + async def test_sitemapindex_descends_into_children(self): + class S(SitemapSpider): + name = "s" + sitemap_urls = ["https://example.com/sitemap.xml"] + + spider = S() + out = await _collect(spider._parse_sitemap(_make_response(INDEX_XML))) + # No urls in this index, just three child sitemap fetches + assert len(out) == 3 + assert all(r.callback == spider._parse_sitemap for r in out) + assert {r.url for r in out} == { + "https://example.com/posts-sitemap.xml", + "https://example.com/products-sitemap.xml", + "https://example.com/skip-sitemap.xml", + } + + @pytest.mark.asyncio + async def test_sitemap_follow_filters_child_sitemaps(self): + class S(SitemapSpider): + name = "s" + sitemap_urls = ["https://example.com/sitemap.xml"] + sitemap_follow = LinkExtractor(allow=r"posts-sitemap") + + spider = S() + out = await _collect(spider._parse_sitemap(_make_response(INDEX_XML))) + assert {r.url for r in out} == {"https://example.com/posts-sitemap.xml"} + + @pytest.mark.asyncio + async def test_alternate_links_dispatched_when_enabled(self): + class S(SitemapSpider): + name = "s" + sitemap_urls = ["https://example.com/sitemap.xml"] + sitemap_alternate_links = True + + spider = S() + out = await _collect(spider._parse_sitemap(_make_response(URLSET_WITH_ALTERNATES))) + urls = {r.url for r in out} + assert urls == { + "https://example.com/en/page", + "https://example.com/fr/page", + "https://example.com/de/page", + } + + @pytest.mark.asyncio + async def test_gzipped_sitemap_handled_via_magic_bytes(self): + class S(SitemapSpider): + name = "s" + sitemap_urls = ["https://example.com/sitemap.xml.gz"] + + spider = S() + body = gzip.compress(URLSET_XML) + out = await _collect(spider._parse_sitemap(_make_response(body, url="https://example.com/sitemap.xml.gz"))) + assert len(out) == 3 + + +class TestSitemapSpiderStartRequests: + @pytest.mark.asyncio + async def test_start_requests_uses_sitemap_urls(self): + class S(SitemapSpider): + name = "s" + sitemap_urls = ["https://a.com/s.xml", "https://b.com/s.xml"] + + spider = S() + out = [req async for req in spider.start_requests()] + assert {r.url for r in out} == {"https://a.com/s.xml", "https://b.com/s.xml"} + assert all(r.callback == spider._parse_sitemap for r in out) + + @pytest.mark.asyncio + async def test_start_requests_falls_back_to_robots_txt(self): + class S(SitemapSpider): + name = "s" + allowed_domains = {"example.com"} + + spider = S() + out = [req async for req in spider.start_requests()] + assert len(out) == 1 + assert out[0].url == "https://example.com/robots.txt" + assert out[0].callback == spider._parse_robots + + @pytest.mark.asyncio + async def test_start_requests_uses_start_urls_if_no_sitemap_urls(self): + class S(SitemapSpider): + name = "s" + start_urls = ["https://example.com/seed"] + + spider = S() + out = [req async for req in spider.start_requests()] + assert len(out) == 1 + assert out[0].url == "https://example.com/seed" + # Should NOT have _parse_sitemap as callback (start_urls path treats them as regular pages) + assert out[0].callback is None + + @pytest.mark.asyncio + async def test_start_requests_raises_when_nothing_configured(self): + class S(SitemapSpider): + name = "s" + + spider = S() + with pytest.raises(RuntimeError, match="needs `sitemap_urls`"): + [req async for req in spider.start_requests()] + + +class TestParseRobots: + @pytest.mark.asyncio + async def test_parse_robots_yields_sitemap_requests(self): + class S(SitemapSpider): + name = "s" + allowed_domains = {"example.com"} + + spider = S() + body = b"User-agent: *\nSitemap: https://example.com/sitemap.xml\n" + resp = _make_response(body, url="https://example.com/robots.txt") + out = await _collect(spider._parse_robots(resp)) + assert len(out) == 1 + assert out[0].url == "https://example.com/sitemap.xml" + assert out[0].callback == spider._parse_sitemap + + @pytest.mark.asyncio + async def test_parse_robots_with_no_directives_warns(self): + # Spider's logger has propagate=False, so we attach our own handler to it. + import logging + + class S(SitemapSpider): + name = "s" + allowed_domains = {"example.com"} + + spider = S() + records: list[logging.LogRecord] = [] + + class _Capture(logging.Handler): + def emit(self, record: logging.LogRecord) -> None: + records.append(record) + + spider.logger.addHandler(_Capture()) + + body = b"User-agent: *\nDisallow: /\n" + resp = _make_response(body, url="https://example.com/robots.txt") + out = await _collect(spider._parse_robots(resp)) + assert out == [] + assert any("No Sitemap:" in r.getMessage() for r in records if r.levelno == logging.WARNING) + + +class TestSitemapSpiderPickle: + @pytest.mark.asyncio + async def test_pickle_request_with_bound_method_callback_via_rules(self): + class S(SitemapSpider): + name = "s" + sitemap_urls = ["https://example.com/sitemap.xml"] + + def rules(self): + return [CrawlRule(LinkExtractor(allow=r"/posts/"), callback=self.parse_post)] + + async def parse_post(self, response): + yield {"post": response.url} + + spider = S() + out = await _collect(spider._parse_sitemap(_make_response(URLSET_XML))) + post_req = next(r for r in out if "/posts/" in r.url) + state = post_req.__getstate__() + assert state["_callback_name"] == "parse_post" + # Round-trip + pickled = pickle.dumps(post_req) + restored = pickle.loads(pickled) + fresh = S() + restored._restore_callback(fresh) + assert restored.callback == fresh.parse_post diff --git a/tests/spiders/test_templates.py b/tests/spiders/test_templates.py new file mode 100644 index 0000000..893e05a --- /dev/null +++ b/tests/spiders/test_templates.py @@ -0,0 +1,230 @@ +"""Tests for `CrawlSpider` and `CrawlRule`.""" + +import pickle + +import pytest + +from scrapling.engines.toolbelt.custom import Response +from scrapling.spiders.links import LinkExtractor +from scrapling.spiders.request import Request +from scrapling.spiders.templates import CrawlRule, CrawlSpider +from scrapling.core._types import Any, AsyncGenerator, Dict, Union + + +HTML = """ + + post 1 + post 2 + next page + about + +""" + + +def _make_response(url: str = "https://example.com/") -> Response: + """Build a Response with a Request attached so `response.follow()` works.""" + resp = Response( + url=url, + content=HTML, + status=200, + reason="OK", + cookies={}, + headers={}, + request_headers={}, + ) + resp.request = Request(url) + return resp + + +class _TestSpider(CrawlSpider): + name = "test" + start_urls = ["https://example.com/"] + + async def parse_post(self, response: Response) -> AsyncGenerator[Union[Dict[str, Any], Request, None], None]: + yield {"post": response.url} + + async def parse_page(self, response: Response) -> AsyncGenerator[Union[Dict[str, Any], Request, None], None]: + yield {"page": response.url} + + +async def _collect(agen: AsyncGenerator) -> list: + return [item async for item in agen] + + +class TestCrawlSpider: + @pytest.mark.asyncio + async def test_empty_rules_yields_nothing(self): + class S(CrawlSpider): + name = "s" + start_urls = ["https://example.com/"] + + spider = S() + out = await _collect(spider.parse(_make_response())) + assert out == [] + + @pytest.mark.asyncio + async def test_single_rule_yields_matching_links(self): + class S(CrawlSpider): + name = "s" + start_urls = ["https://example.com/"] + + def rules(self): + return [CrawlRule(LinkExtractor(allow=r"/posts/"))] + + spider = S() + out = await _collect(spider.parse(_make_response())) + urls = [r.url for r in out] + assert urls == ["https://example.com/posts/1", "https://example.com/posts/2"] + + @pytest.mark.asyncio + async def test_multiple_rules_all_applied(self): + class S(CrawlSpider): + name = "s" + start_urls = ["https://example.com/"] + + def rules(self): + return [ + CrawlRule(LinkExtractor(allow=r"/posts/")), + CrawlRule(LinkExtractor(allow=r"/page/")), + ] + + spider = S() + out = await _collect(spider.parse(_make_response())) + urls = [r.url for r in out] + assert "https://example.com/posts/1" in urls + assert "https://example.com/posts/2" in urls + assert "https://example.com/page/2/" in urls + + @pytest.mark.asyncio + async def test_rule_with_callback_bound_method(self): + spider = _TestSpider() + # rules() defaults to []; override at instance level + spider.rules = lambda: [ # type: ignore[method-assign] + CrawlRule(LinkExtractor(allow=r"/posts/"), callback=spider.parse_post) + ] + out = await _collect(spider.parse(_make_response())) + assert all(r.callback == spider.parse_post for r in out) + + @pytest.mark.asyncio + async def test_rule_with_no_callback_leaves_request_callback_none(self): + # When CrawlRule.callback is None, response.follow() inherits the original + # request's callback. The original request was created with callback=None, + # so the resulting request's callback should also be None (engine then + # falls back to spider.parse). + class S(CrawlSpider): + name = "s" + start_urls = ["https://example.com/"] + + def rules(self): + return [CrawlRule(LinkExtractor(allow=r"/posts/"))] + + spider = S() + out = await _collect(spider.parse(_make_response())) + assert all(r.callback is None for r in out) + + @pytest.mark.asyncio + async def test_process_request_invoked(self): + spider = _TestSpider() + + def add_priority(req: Request, response: Response) -> Request: + req.priority = 99 + return req + + spider.rules = lambda: [ # type: ignore[method-assign] + CrawlRule(LinkExtractor(allow=r"/posts/"), process_request=add_priority) + ] + out = await _collect(spider.parse(_make_response())) + assert all(r.priority == 99 for r in out) + + @pytest.mark.asyncio + async def test_process_request_can_replace_request(self): + spider = _TestSpider() + replacement = Request("https://replaced.example.com/") + + def replace(req: Request, response: Response) -> Request: + return replacement + + spider.rules = lambda: [ # type: ignore[method-assign] + CrawlRule(LinkExtractor(allow=r"/posts/"), process_request=replace) + ] + out = await _collect(spider.parse(_make_response())) + assert all(r is replacement for r in out) + + @pytest.mark.asyncio + async def test_user_can_compose_super_parse(self): + """Override parse() to add custom yields plus call super().parse() for rules.""" + + class S(CrawlSpider): + name = "s" + start_urls = ["https://example.com/"] + + def rules(self): + return [CrawlRule(LinkExtractor(allow=r"/posts/"))] + + async def parse(self, response): + yield {"custom": "item"} + async for req in super().parse(response): + yield req + + spider = S() + out = await _collect(spider.parse(_make_response())) + assert out[0] == {"custom": "item"} + assert all(isinstance(x, Request) for x in out[1:]) + assert len(out) == 3 # 1 dict + 2 requests + + @pytest.mark.asyncio + async def test_referer_set_on_followed_requests(self): + # `response.follow()` sets the referer header; verify it survives the rule path. + class S(CrawlSpider): + name = "s" + start_urls = ["https://example.com/"] + + def rules(self): + return [CrawlRule(LinkExtractor(allow=r"/posts/"))] + + spider = S() + out = await _collect(spider.parse(_make_response())) + for req in out: + assert req._session_kwargs["headers"]["referer"] == "https://example.com/" + + +class TestCrawlSpiderPickle: + """Verify Request produced by CrawlSpider survives pickle round-trip with bound-method callbacks.""" + + @pytest.mark.asyncio + async def test_pickle_request_with_bound_method_callback(self): + spider = _TestSpider() + spider.rules = lambda: [ # type: ignore[method-assign] + CrawlRule(LinkExtractor(allow=r"/posts/"), callback=spider.parse_post) + ] + out = await _collect(spider.parse(_make_response())) + req = out[0] + + # __getstate__ should convert the bound method into a name-string + state = req.__getstate__() + assert state["callback"] is None + assert state["_callback_name"] == "parse_post" + + # Round-trip via pickle (bound methods aren't directly picklable; the + # state machinery handles the conversion) + pickled = pickle.dumps(req) + restored = pickle.loads(pickled) + assert restored._callback_name == "parse_post" + + # Then _restore_callback on a fresh spider instance brings the method back + fresh_spider = _TestSpider() + restored._restore_callback(fresh_spider) + assert restored.callback == fresh_spider.parse_post + + +class TestCrawlRule: + def test_default_callback_is_none(self): + rule = CrawlRule(LinkExtractor()) + assert rule.callback is None + assert rule.follow is None + assert rule.process_request is None + + def test_callback_accepts_callable(self): + spider = _TestSpider() + rule = CrawlRule(LinkExtractor(), callback=spider.parse_post) + assert rule.callback == spider.parse_post From 6a641912cdee03a24d44e0b66444bbff915d45a8 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Mon, 11 May 2026 02:58:00 +0300 Subject: [PATCH 15/20] docs: add docs for the new features --- agent-skill/Scrapling-Skill/SKILL.md | 19 ++ .../references/spiders/generic-templates.md | 167 +++++++++++++++++ docs/spiders/generic-templates.md | 168 ++++++++++++++++++ zensical.toml | 1 + 4 files changed, 355 insertions(+) create mode 100644 agent-skill/Scrapling-Skill/references/spiders/generic-templates.md create mode 100644 docs/spiders/generic-templates.md diff --git a/agent-skill/Scrapling-Skill/SKILL.md b/agent-skill/Scrapling-Skill/SKILL.md index 24af980..27c31c0 100644 --- a/agent-skill/Scrapling-Skill/SKILL.md +++ b/agent-skill/Scrapling-Skill/SKILL.md @@ -306,6 +306,25 @@ Press Ctrl+C to pause gracefully - progress is saved automatically. Later, when While iterating on a spider's `parse()` logic, set `development_mode = True` on the spider class to cache responses to disk on the first run and replay them on subsequent runs - so you can re-run the spider as many times as you want without re-hitting the target servers. The cache lives in `.scrapling_cache/{spider.name}/` by default and can be overridden with `development_cache_dir`. Don't ship a spider with this enabled. +For rules-based crawls (follow links matching a regex), use `CrawlSpider` instead of writing the link-extraction loop yourself: +```python +from scrapling.spiders import CrawlSpider, CrawlRule, LinkExtractor + +class BlogCrawler(CrawlSpider): + name = "blog" + start_urls = ["https://example.com"] + + def rules(self): + return [ + CrawlRule(LinkExtractor(allow=r"/posts/"), callback=self.parse_post), + CrawlRule(LinkExtractor(allow=r"/page/\d+/")), # follow pagination, no callback + ] + + async def parse_post(self, response): + yield {"title": response.css("h1::text").get()} +``` +For sitemap-driven crawls, use `SitemapSpider` with the same `rules()` API. It fetches `sitemap_urls`, descends into sitemap indexes, and dispatches each URL through your rules. Put a `robots.txt` URL directly in `sitemap_urls` and the spider extracts each `Sitemap:` directive from it automatically. See `references/spiders/generic-templates.md` for the full reference, including `LinkExtractor`'s allow/deny/restrict_css/canonicalize options. + ### Advanced Parsing & Navigation ```python from scrapling.fetchers import Fetcher diff --git a/agent-skill/Scrapling-Skill/references/spiders/generic-templates.md b/agent-skill/Scrapling-Skill/references/spiders/generic-templates.md new file mode 100644 index 0000000..d16edf2 --- /dev/null +++ b/agent-skill/Scrapling-Skill/references/spiders/generic-templates.md @@ -0,0 +1,167 @@ +# Generic Spider Templates + +Most crawls fall into one of two patterns: "follow links matching this regex" or "crawl every URL listed in the site's sitemap". Scrapling ships templates for both so you don't have to hand-write the same `parse()` boilerplate every time. + +Both templates build on `LinkExtractor`, which pulls URLs out of a `Response` (or filters a single URL via `matches()`). `SitemapSpider` additionally parses sitemap.xml / sitemap_index.xml bodies internally (gzip-compressed or not). + +You can use `LinkExtractor` directly inside any plain `Spider.parse()`. The templates just save you the wiring. + +## CrawlSpider + +`CrawlSpider` follows links automatically based on declarative rules. + +```python +from scrapling.spiders import CrawlSpider, CrawlRule, LinkExtractor + +class BlogCrawler(CrawlSpider): + name = "blog" + start_urls = ["https://example.com"] + + def rules(self): + return [ + CrawlRule(LinkExtractor(allow=r"/posts/"), callback=self.parse_post), + CrawlRule(LinkExtractor(allow=r"/page/\d+/")), # follow pagination, no callback + ] + + async def parse_post(self, response): + yield { + "title": response.css("h1::text").get(), + "url": response.url, + } + +result = BlogCrawler().start() +``` + +A `CrawlRule` pairs a `LinkExtractor` with an optional `callback` (a bound method on the spider), an optional `priority` override for the dispatched `Request`, and an optional `process_request` (a bound method that mutates each `Request` before it's yielded). The default `parse()` runs every rule against every response and yields a `Request` per matched URL. + +If a rule has no callback, the matched URLs fall through to the spider's default `parse()` (or stay uncallback'd if you didn't override it). This is convenient for pagination: extract the next-page links to keep the crawl going, but don't need a separate handler. + +### Combining rules with custom logic + +Override `parse()` and call `super().parse(response)` to get the rule behavior plus your own yields: + +```python +class MySpider(CrawlSpider): + def rules(self): + return [CrawlRule(LinkExtractor(allow=r"/posts/"), callback=self.parse_post)] + + async def parse(self, response): + yield {"page_url": response.url} + async for req in super().parse(response): + yield req +``` + +### Mutating Requests with `process_request` + +```python +def add_priority(self, request, response): + request.priority = 10 + return request + +def rules(self): + return [CrawlRule( + LinkExtractor(allow=r"/posts/"), + callback=self.parse_post, + process_request=self.add_priority, + )] +``` + +## SitemapSpider + +`SitemapSpider` seeds a crawl from sitemap.xml URLs. It uses the same `rules()` API as `CrawlSpider`, so the mental model is shared. + +```python +from scrapling.spiders import SitemapSpider, CrawlRule, LinkExtractor + +class MySitemap(SitemapSpider): + name = "sm" + sitemap_urls = ["https://example.com/sitemap.xml"] + + def rules(self): + return [ + CrawlRule(LinkExtractor(allow=r"/posts/"), callback=self.parse_post), + CrawlRule(LinkExtractor(allow=r"/products/"), callback=self.parse_product), + ] + + async def parse_post(self, response): + yield {"title": response.css("h1::text").get()} + + async def parse_product(self, response): + yield {"sku": response.css(".sku::text").get()} + +result = MySitemap().start() +``` + +### How URLs are dispatched + +For each URL in the sitemap, `SitemapSpider` checks every rule's `LinkExtractor.matches(url)` in order. The first matching rule wins, and a `Request` is yielded with that rule's callback. If no rule matches and `rules()` is non-empty, the URL is dropped (matches Scrapy's behavior). If `rules()` returns an empty list, every URL is routed to the spider's `parse()` method, which raises `NotImplementedError` by default - override it to handle them. + +### Sitemap indexes + +When `SitemapSpider` encounters a `` (a sitemap of sitemaps), it descends into each child sitemap automatically. To filter which child sitemaps to descend into, set `sitemap_follow` to a `LinkExtractor`: + +```python +class MySitemap(SitemapSpider): + name = "sm" + sitemap_urls = ["https://example.com/sitemap.xml"] + sitemap_follow = LinkExtractor(allow=r"/posts-sitemap-\d+\.xml") # only post sitemaps +``` + +### Robots.txt support + +Put a `robots.txt` URL directly in `sitemap_urls` and `SitemapSpider` will detect it, extract every `Sitemap:` directive (via `protego`), and follow each one: + +```python +class MySitemap(SitemapSpider): + name = "sm" + sitemap_urls = ["https://example.com/robots.txt"] # Sitemap: directives discovered automatically +``` + +### Alternate-language URLs + +Set `sitemap_alternate_links = True` to also dispatch `` URLs through your rules. + +## Using `LinkExtractor` directly + +You don't have to use the templates. `LinkExtractor` works inside any plain `Spider`: + +```python +from scrapling.spiders import Spider, LinkExtractor + +class CustomSpider(Spider): + name = "custom" + start_urls = ["https://example.com"] + + def __init__(self): + super().__init__() + self._links = LinkExtractor(allow=r"/posts/", deny_domains="ads.example.com") + + async def parse(self, response): + for url in self._links.extract(response): + yield response.follow(url, callback=self.parse_post) + + async def parse_post(self, response): + yield {"title": response.css("h1::text").get()} +``` + +## LinkExtractor reference + +| Argument | Default | Description | +|---|---|---| +| `allow` | `()` | URL patterns to keep. Empty means "match all". String, compiled `Pattern`, or iterable of either. | +| `deny` | `()` | URL patterns to drop. Always overrides `allow`. | +| `allow_domains` | `()` | Hostnames to keep. Subdomains match automatically (`example.com` matches `api.example.com`). | +| `deny_domains` | `()` | Hostnames to drop. | +| `restrict_css` | `()` | CSS selectors that scope DOM extraction to a region. | +| `restrict_xpath` | `()` | XPath selectors that scope DOM extraction to a region. | +| `tags` | `("a", "area")` | Element tags to look for links in. | +| `attrs` | `("href",)` | Attributes on those tags to read URLs from. | +| `canonicalize` | `True` | Sort query params and normalize the path. | +| `strip` | `True` | Strip whitespace from extracted URLs. | +| `keep_fragment` | `False` | Preserve the `#fragment` when canonicalizing. | +| `deny_extensions` | `IGNORED_EXTENSIONS` | File extensions to drop (pdf, zip, images, video, etc.). | +| `process` | `None` | Optional callable applied to each extracted URL before filtering. Return a falsy value to drop. | + +`LinkExtractor.extract(response)` returns a `list[str]` of absolute, filtered, deduped URLs. + +`LinkExtractor.matches(url)` returns a `bool` - the URL-only filter (allow/deny/domain/extension), used by `SitemapSpider` to dispatch URLs without a `Response`. diff --git a/docs/spiders/generic-templates.md b/docs/spiders/generic-templates.md new file mode 100644 index 0000000..5014849 --- /dev/null +++ b/docs/spiders/generic-templates.md @@ -0,0 +1,168 @@ +# Generic Spider Templates + +Most crawls fall into one of two patterns: "follow links matching this pattern" or "crawl every URL listed in the site's sitemap". Scrapling ships templates for both so you don't have to hand-write the same `parse()` boilerplate every time. + +All templates build on `LinkExtractor`, which pulls URLs out of a `Response` (or filters a single URL via `matches()`). `SitemapSpider` additionally parses sitemap.xml / sitemap_index.xml bodies internally (gzip-compressed or not). + +You can use `LinkExtractor` directly inside any plain `Spider.parse()`. The templates just save you the wiring. + +## CrawlSpider + +`CrawlSpider` follows links automatically based on declarative rules. + +```python +from scrapling.spiders import CrawlSpider, CrawlRule, LinkExtractor + +class QuotesSpider(CrawlSpider): + name = "blog" + start_urls = ["https://quotes.toscrape.com/"] + + def rules(self): + return [ + CrawlRule(LinkExtractor(allow=r"/author/"), callback=self.parse_author), + CrawlRule(LinkExtractor(allow=r"/page/\d+/")), # follow pagination, no callback + ] + + async def parse_author(self, response): + yield { + '.author-title': response.css('.author-title::text').get(), + "birthday": response.css('.author-born-date::text').get(), + "url": response.url, + } + +result = QuotesSpider().start() +``` + +A `CrawlRule` pairs a `LinkExtractor` with an optional `callback` (a bound method on the spider), an optional `priority` override for the dispatched `Request`, and an optional `process_request` (a bound method that mutates each `Request` before it's yielded). The default `parse()` runs every rule against every response and yields a `Request` per matched URL. + +If a rule has no callback, the matched URLs fall through to the spider's default `parse()`. This is convenient for pagination: extract the next-page links to keep the crawl going, without needing a separate handler. + +### Combining rules with custom logic + +Override `parse()` and call `super().parse(response)` to get the rule behavior plus your own yields: + +```python +class MySpider(CrawlSpider): + def rules(self): + return [CrawlRule(LinkExtractor(allow=r"/posts/"), callback=self.parse_post)] + + async def parse(self, response): + yield {"page_url": response.url} + async for req in super().parse(response): + yield req +``` + +### Mutating Requests with `process_request` + +```python +def add_priority(self, request, response): + request.priority = 10 + return request + +def rules(self): + return [CrawlRule( + LinkExtractor(allow=r"/posts/"), + callback=self.parse_post, + process_request=self.add_priority, + )] +``` + +## SitemapSpider + +`SitemapSpider` seeds a crawl from sitemap.xml URLs. It uses the same `rules()` API as `CrawlSpider`, so the mental model is shared. + +```python +from scrapling.spiders import SitemapSpider, CrawlRule, LinkExtractor + +class MySitemap(SitemapSpider): + name = "sm" + sitemap_urls = ["https://example.com/sitemap.xml"] + + def rules(self): + return [ + CrawlRule(LinkExtractor(allow=r"/posts/"), callback=self.parse_post), + CrawlRule(LinkExtractor(allow=r"/products/"), callback=self.parse_product), + ] + + async def parse_post(self, response): + yield {"title": response.css("h1::text").get()} + + async def parse_product(self, response): + yield {"sku": response.css(".sku::text").get()} + +result = MySitemap().start() +``` + +### How URLs are dispatched + +For each URL in the sitemap, `SitemapSpider` checks every rule's `LinkExtractor.matches(url)` in order. The first matching rule wins, and a `Request` is yielded with that rule's callback. If no rule matches and `rules()` is non-empty, the URL is dropped. If `rules()` returns an empty list, every URL is routed to the spider's `parse()` method, which raises `NotImplementedError` by default if not overridden. + +### Sitemap indexes + +When `SitemapSpider` encounters a `` (a sitemap of sitemaps), it descends into each child sitemap automatically. To filter which child sitemaps to descend into, set `sitemap_follow` to a `LinkExtractor`: + +```python +class MySitemap(SitemapSpider): + name = "sm" + sitemap_urls = ["https://example.com/sitemap.xml"] + sitemap_follow = LinkExtractor(allow=r"/posts-sitemap-\d+\.xml") # only post sitemaps +``` + +### Robots.txt support + +Put a `robots.txt` URL directly in `sitemap_urls` and `SitemapSpider` will detect it, extract every sitemap shown, and follow each one: + +```python +class MySitemap(SitemapSpider): + name = "sm" + sitemap_urls = ["https://example.com/robots.txt"] +``` + +### Alternate-language URLs + +Set `sitemap_alternate_links = True` to also dispatch `` URLs through your rules. + +## Using `LinkExtractor` directly + +You don't have to use the templates. `LinkExtractor` works inside any plain `Spider`: + +```python +from scrapling.spiders import Spider, LinkExtractor + +class CustomSpider(Spider): + name = "custom" + start_urls = ["https://example.com"] + + def __init__(self): + super().__init__() + self._links = LinkExtractor(allow=r"/posts/", deny_domains="ads.example.com") + + async def parse(self, response): + for url in self._links.extract(response): + yield response.follow(url, callback=self.parse_post) + + async def parse_post(self, response): + yield {"title": response.css("h1::text").get()} +``` + +## LinkExtractor reference + +| Argument | Default | Description | +|-------------------|----------------------|---------------------------------------------------------------------------------------------------| +| `allow` | `()` | URL patterns to keep. Empty means "match all". String, compiled `Pattern`, or iterable of either. | +| `deny` | `()` | URL patterns to drop. Always overrides `allow`. | +| `allow_domains` | `()` | Hostnames to keep. Subdomains match automatically (`example.com` matches `api.example.com`). | +| `deny_domains` | `()` | Hostnames to drop. | +| `restrict_css` | `()` | CSS selectors that scope DOM extraction to a region. | +| `restrict_xpath` | `()` | XPath selectors that scope DOM extraction to a region. | +| `tags` | `("a", "area")` | Element tags to look for links in. | +| `attrs` | `("href",)` | Attributes on those tags to read URLs from. | +| `canonicalize` | `True` | Sort query params and normalize the path. | +| `strip` | `True` | Strip whitespace from extracted URLs. | +| `keep_fragment` | `False` | Preserve the `#fragment` when canonicalizing. | +| `deny_extensions` | `IGNORED_EXTENSIONS` | File extensions to drop (pdf, zip, images, video, etc.). | +| `process` | `None` | Optional callable applied to each extracted URL before filtering. Return a falsy value to drop. | + +`LinkExtractor.extract(response)` returns a `list[str]` of absolute, filtered, deduped URLs. + +`LinkExtractor.matches(url)` returns a `bool` - the URL-only filter (allow/deny/domain/extension), used by `SitemapSpider` to dispatch URLs without a `Response`. diff --git a/zensical.toml b/zensical.toml index 9eaea06..70362a5 100644 --- a/zensical.toml +++ b/zensical.toml @@ -35,6 +35,7 @@ nav = [ {"Requests & Responses" = "spiders/requests-responses.md"}, {"Sessions" = "spiders/sessions.md"}, {"Proxy management & Blocking" = "spiders/proxy-blocking.md"}, + {"Generic crawlers" = "spiders/generic-templates.md"}, {"Advanced features" = "spiders/advanced.md"} ]}, {"Command Line Interface" = [ From c0ed8949f62106e7f464c1e89ac52f59ab0303eb Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Mon, 11 May 2026 03:00:11 +0300 Subject: [PATCH 16/20] docs: update zensical version --- docs/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index 237ed2f..fe3cea2 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,4 +1,4 @@ -zensical>=0.0.36 +zensical>=0.0.41 mkdocstrings>=1.0.4 mkdocstrings-python>=2.0.3 griffe-inherited-docstrings>=1.1.3 From ebd7e0971c7e7d6be1b27d68f01c7f910e62f89b Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Mon, 11 May 2026 03:18:26 +0300 Subject: [PATCH 17/20] build: update deps and browser useragents --- pyproject.toml | 6 +++--- scrapling/engines/toolbelt/fingerprints.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 741a30d..8cb4768 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -73,10 +73,10 @@ dependencies = [ fetchers = [ "click>=8.3.0", "curl_cffi>=0.15.0", - "playwright==1.58.0", - "patchright==1.58.2", + "playwright==1.59.0", + "patchright==1.59.1", "browserforge>=1.2.4", - "apify-fingerprint-datapoints>=0.12.0", + "apify-fingerprint-datapoints>=0.13.0", "msgspec>=0.21.1", "anyio>=4.13.0", "protego>=0.6.0", diff --git a/scrapling/engines/toolbelt/fingerprints.py b/scrapling/engines/toolbelt/fingerprints.py index 3d256fa..f4fdfc1 100644 --- a/scrapling/engines/toolbelt/fingerprints.py +++ b/scrapling/engines/toolbelt/fingerprints.py @@ -13,8 +13,8 @@ from scrapling.core._types import Dict, Literal, Tuple __OS_NAME__ = platform_system() OSName = Literal["linux", "macos", "windows"] # Current versions hardcoded for now (Playwright doesn't allow to know the version of a browser without launching it) -chromium_version = 145 -chrome_version = 145 +chromium_version = 147 +chrome_version = 147 @lru_cache(1, typed=True) From 34651abc6b474718173e2f72a9c3962a78d1a1d3 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Mon, 11 May 2026 03:30:59 +0300 Subject: [PATCH 18/20] tests: remove old code and update the rest --- tests/spiders/test_sitemap.py | 173 +++----------------------------- tests/spiders/test_templates.py | 2 +- 2 files changed, 14 insertions(+), 161 deletions(-) diff --git a/tests/spiders/test_sitemap.py b/tests/spiders/test_sitemap.py index cd5739b..17c760d 100644 --- a/tests/spiders/test_sitemap.py +++ b/tests/spiders/test_sitemap.py @@ -1,4 +1,4 @@ -"""Tests for `SitemapParser` and `SitemapSpider`.""" +"""Tests for `SitemapSpider`.""" import gzip import pickle @@ -8,9 +8,9 @@ import pytest from scrapling.engines.toolbelt.custom import Response from scrapling.spiders.links import LinkExtractor from scrapling.spiders.request import Request -from scrapling.spiders.sitemap import SitemapParser, SitemapResult, SitemapSpider, SitemapUrl +from scrapling.spiders.templates.sitemap import SitemapSpider from scrapling.spiders.templates import CrawlRule -from scrapling.core._types import Any, AsyncGenerator, Dict, Union +from scrapling.core._types import AsyncGenerator URLSET_XML = b""" @@ -50,127 +50,6 @@ INDEX_XML = b""" """ -# Sitemap without the standard namespace (some sites do this) -URLSET_NO_NS = b""" - - https://example.com/x - -""" - - -class TestSitemapParserUrlset: - def test_parse_urlset_with_full_metadata(self): - result = SitemapParser().parse(URLSET_XML) - assert len(result.urls) == 3 - assert result.sitemaps == [] - first = result.urls[0] - assert first.loc == "https://example.com/posts/1" - assert first.lastmod == "2026-01-15" - assert first.changefreq == "daily" - assert first.priority == 0.8 - - def test_parse_handles_partial_metadata(self): - result = SitemapParser().parse(URLSET_XML) - second = result.urls[1] - assert second.lastmod == "2026-02-20" - assert second.changefreq is None - assert second.priority is None - - def test_parse_handles_no_namespace(self): - result = SitemapParser().parse(URLSET_NO_NS) - assert len(result.urls) == 1 - assert result.urls[0].loc == "https://example.com/x" - - -class TestSitemapParserAlternates: - def test_alternate_links_off_by_default(self): - result = SitemapParser().parse(URLSET_WITH_ALTERNATES) - assert result.urls[0].alternates == [] - - def test_alternate_links_on_collects_them(self): - result = SitemapParser(alternate_links=True).parse(URLSET_WITH_ALTERNATES) - assert result.urls[0].alternates == [ - "https://example.com/fr/page", - "https://example.com/de/page", - ] - - -class TestSitemapParserIndex: - def test_parse_sitemapindex_returns_child_sitemaps(self): - result = SitemapParser().parse(INDEX_XML) - assert result.urls == [] - assert result.sitemaps == [ - "https://example.com/posts-sitemap.xml", - "https://example.com/products-sitemap.xml", - "https://example.com/skip-sitemap.xml", - ] - - -class TestSitemapParserDecompression: - def test_gz_body_via_magic_bytes(self): - compressed = gzip.compress(URLSET_XML) - result = SitemapParser().parse(compressed) - assert len(result.urls) == 3 - - def test_gz_body_via_content_type_hint(self): - compressed = gzip.compress(URLSET_XML) - result = SitemapParser().parse(compressed, content_type="application/x-gzip") - assert len(result.urls) == 3 - - def test_corrupt_gz_logged_not_raised(self): - # Body starts with gzip magic but is not valid gzip - body = b"\x1f\x8b" + b"junk data" - result = SitemapParser().parse(body) - assert result == SitemapResult() - - -class TestSitemapParserMalformed: - def test_invalid_xml_returns_empty_result(self): - result = SitemapParser().parse(b"") - assert result == SitemapResult() - - -class TestFromRobotsTxt: - def test_extracts_sitemap_directives(self): - body = """ - User-agent: * - Disallow: /admin - Sitemap: https://example.com/sitemap.xml - Sitemap: https://example.com/posts-sitemap.xml - """ - urls = SitemapParser.from_robots_txt(body) - assert urls == [ - "https://example.com/sitemap.xml", - "https://example.com/posts-sitemap.xml", - ] - - def test_ignores_comments_and_blank_lines(self): - body = """ - # This is a comment - Sitemap: https://example.com/sitemap.xml # inline comment - # Sitemap: https://commented.example.com/sitemap.xml - """ - urls = SitemapParser.from_robots_txt(body) - assert urls == ["https://example.com/sitemap.xml"] - - def test_directive_match_is_case_insensitive(self): - body = "SITEMAP: https://example.com/sitemap.xml\nsitemap: https://example.com/other.xml" - urls = SitemapParser.from_robots_txt(body) - assert urls == [ - "https://example.com/sitemap.xml", - "https://example.com/other.xml", - ] - - def test_returns_empty_when_no_directives(self): - body = "User-agent: *\nDisallow: /admin" - urls = SitemapParser.from_robots_txt(body) - assert urls == [] - - def _make_response(body: bytes, url: str = "https://example.com/sitemap.xml", headers: dict | None = None) -> Response: resp = Response( url=url, @@ -206,11 +85,10 @@ class TestSitemapSpiderFlow: out = await _collect(spider._parse_sitemap(_make_response(URLSET_XML))) post_reqs = [r for r in out if "/posts/" in r.url] about_reqs = [r for r in out if "/about" in r.url] - # Two posts dispatched to parse_post; about falls through (callback inherited from None → None) + # Two posts dispatched to parse_post; /about is dropped (matches no rule, non-empty rules) assert len(post_reqs) == 2 assert all(r.callback == spider.parse_post for r in post_reqs) - assert len(about_reqs) == 1 - assert about_reqs[0].callback is None + assert about_reqs == [] @pytest.mark.asyncio async def test_no_rules_means_all_urls_fall_through(self): @@ -291,31 +169,6 @@ class TestSitemapSpiderStartRequests: assert {r.url for r in out} == {"https://a.com/s.xml", "https://b.com/s.xml"} assert all(r.callback == spider._parse_sitemap for r in out) - @pytest.mark.asyncio - async def test_start_requests_falls_back_to_robots_txt(self): - class S(SitemapSpider): - name = "s" - allowed_domains = {"example.com"} - - spider = S() - out = [req async for req in spider.start_requests()] - assert len(out) == 1 - assert out[0].url == "https://example.com/robots.txt" - assert out[0].callback == spider._parse_robots - - @pytest.mark.asyncio - async def test_start_requests_uses_start_urls_if_no_sitemap_urls(self): - class S(SitemapSpider): - name = "s" - start_urls = ["https://example.com/seed"] - - spider = S() - out = [req async for req in spider.start_requests()] - assert len(out) == 1 - assert out[0].url == "https://example.com/seed" - # Should NOT have _parse_sitemap as callback (start_urls path treats them as regular pages) - assert out[0].callback is None - @pytest.mark.asyncio async def test_start_requests_raises_when_nothing_configured(self): class S(SitemapSpider): @@ -326,29 +179,29 @@ class TestSitemapSpiderStartRequests: [req async for req in spider.start_requests()] -class TestParseRobots: +class TestRobotsTxt: @pytest.mark.asyncio - async def test_parse_robots_yields_sitemap_requests(self): + async def test_parse_sitemap_yields_requests_from_robots_directives(self): class S(SitemapSpider): name = "s" - allowed_domains = {"example.com"} + sitemap_urls = ["https://example.com/robots.txt"] spider = S() body = b"User-agent: *\nSitemap: https://example.com/sitemap.xml\n" resp = _make_response(body, url="https://example.com/robots.txt") - out = await _collect(spider._parse_robots(resp)) + out = await _collect(spider._parse_sitemap(resp)) assert len(out) == 1 assert out[0].url == "https://example.com/sitemap.xml" assert out[0].callback == spider._parse_sitemap @pytest.mark.asyncio - async def test_parse_robots_with_no_directives_warns(self): + async def test_parse_sitemap_robots_with_no_directives_warns(self): # Spider's logger has propagate=False, so we attach our own handler to it. import logging class S(SitemapSpider): name = "s" - allowed_domains = {"example.com"} + sitemap_urls = ["https://example.com/robots.txt"] spider = S() records: list[logging.LogRecord] = [] @@ -361,9 +214,9 @@ class TestParseRobots: body = b"User-agent: *\nDisallow: /\n" resp = _make_response(body, url="https://example.com/robots.txt") - out = await _collect(spider._parse_robots(resp)) + out = await _collect(spider._parse_sitemap(resp)) assert out == [] - assert any("No Sitemap:" in r.getMessage() for r in records if r.levelno == logging.WARNING) + assert any("No Sitemaps" in r.getMessage() for r in records if r.levelno == logging.WARNING) class TestSitemapSpiderPickle: diff --git a/tests/spiders/test_templates.py b/tests/spiders/test_templates.py index 893e05a..95f3e21 100644 --- a/tests/spiders/test_templates.py +++ b/tests/spiders/test_templates.py @@ -221,7 +221,7 @@ class TestCrawlRule: def test_default_callback_is_none(self): rule = CrawlRule(LinkExtractor()) assert rule.callback is None - assert rule.follow is None + assert rule.priority is None assert rule.process_request is None def test_callback_accepts_callable(self): From 55931897d6d5f5db0d1a028eadf60e6bb00d4e67 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Mon, 11 May 2026 03:34:17 +0300 Subject: [PATCH 19/20] docs(agent): update skill zip file --- agent-skill/Scrapling-Skill.zip | Bin 83861 -> 90018 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/agent-skill/Scrapling-Skill.zip b/agent-skill/Scrapling-Skill.zip index daa1f080a27fc242b4729575c05ba52e57cd9a24..22fbb3231c28ed74331a39fbddbe65c71e15ae82 100644 GIT binary patch delta 53068 zcma&N19)Z4wk{mo?$~C>wr$(Cv0~fq*y`A})3I%%lXQ5y_u2cNv%mBG`~LHJYSmg# z%^Gveftoee`_7a&@PZz2It6J^Ff^b)2OCZ($1Vt+1k>^ zCMkgxopS{p*y>;Hzh*-LK>@inC;Z#&FaLXXf-OG4AZO2V`gi-&U#2t34KSjGeqsii z%#o244gnK829-4miw-5i3ZSIJ2{uYZbApQR1{#b2L{l0;m1$nOM}HCj`gKSqYy;^p zvz=v42x@>tw`&9$YAYi2Ev(NsR-J>#sL8>$i+gzv*^0gU2$5&Uv*TmD34Jdr<0TaU zczQ4vyPc|VQ(bSL8-L?l=OXAn*Z&bL*D`Fe~$+_v=M z*7F5u`Ur~wB=Nc72BqgcmHvcY3lk*LiP8^9K^hW@mT44^8W;!&3=8P@?>}z)JHwOy zHoU&Rte~*GvIfIn{Ko)7ub*It2lp>8%#r|GB@wY>Crtx0C23)zq2c^*uQ8-Wa|7ch z#bQ!XlKm&q^!iSwW~NT2cE+YjyIAOOlD|Ftr~UI9Hk5zDO3=UJ{}m++3HW$i;{iZc z|7!nztACA9tiRJXffX1R@FzYaaQ-V!r+7{iaauAyf1yD&H3?IjD2Fwq6foCBi;+l2 z#71z$CMl_quD~bWu;Ao~Ih&oVyu!R%1FnrPx)-lVF5H1mg0g8+Rn6Ll&+K%zeRn`Z z2Mj5atXVPG=uL9ZY&cn>LSma_xFZ77xEPwohK&NZQ(9F3&IuF6^+v||Og+PPdrE0E zL=8%a$0ca@w)EBLT`J#b9J4cJ>m^L{2Ukl*ZOWAw381v8GlWW05+y70es(gZaZvk= zyB5IEV1UzJ{XjbgB|?RkZd2$~vLb5`c@xwdg{D(RqYje0Aw${L&^VJa60L!GvPuAkRe)7)f{a_i=ZWwFsFK0Nj zB2RWQxL>1oJe5JJb+KH!EY)P-ilAEBL7Fc+I{6vf@N#3nlb!KltuMTjUARaIlphKo_!jmZKQoyGcMyC&Z zSi?FU6X}CCWg9hvRH3*V%wWVt!zz8dUnSZ}yMLXvJj@&?ry#asB_J6-+))$QbBCa@L^oKa2MsjiVtWto)MdWF2K&`E`Q+gL=Xs$N2e%qr%yr^)v)E z3=9JX6vM1R2mD9x*J|X9{7@+F-mJXtNZ1shF2fu*7nhfDFJL+1^nFRvX#>+zk;NEr zF|uqtetvXo6@L!riwCT_HX>~hB;!J|aN{FG;foh=0A)Ig#8P%grW%iWUL^_`1O^`0 z{6%X?Y43u+oF*+htq3VxmuB|CyllcWgi$D7Nf~cx1pN(o0~xW~QmLr^s7hgHfP+aM zfH_equ3?NmM4*D2QOzugxPwUb*gserD}iHwF>BuadM0)&m)C~_W4lHHe>Kubzjq!R z9cfem!07lv#)u!emt{|G&M_kAjaZcqAEtyT?Ew)@!JMpVUZ{Z%T)bpJrEj|Zef1^9 z1K5MS1D2-Nj3z-I_)@G)W?GSITx3#EGH1Uca~!rhyHmj09U&DK$QP(WBPEZ?E012K zXPy(H5A>a4D1?3-es&u=bbKdd%;GSBsG+(QkSq`l%h<{B(;u1x)UGW(0mw`MtrAsC z*f)AGnmgx9I+%?@fz>I5dD;VO5=I>AhHg1#@rLa!bRkVW=A7FY>elW-<{K-5&x5 z0EzG7U91W8%xR-H7iN7Xxf%tfS=GS!=@{$NABPVo7l&2yES`k9pube+I0I|*hO${; zy2>=zna0f^Eua|!DU2(Y)4r1IAFErxgCg`6!gfJ$;&MdzW8@_$KxyQd^wM)xHZa^^ zNAbRhhEKGA25VLKe}#vvQX~=VcRn(R1oSydbOWJdo{!TI38fp^-yc%b&nmB;vY3s8 zL_le9`@O?#NCHKAl!RJd3rzFS8bQ-$tEV;?Q9U1Z$uj1X-B}InqUD7N5CU^c=rzva z<541nN0otOVkkSW49pU2<>YpGJ-OW};`VsoADvt}-|n8gFDJz*f_25=bXy&TYbaGEen);y}5g-JB=L`4+t;#~^f#Zy&ncM3vJAQ~^98*%u zwV-OU_(E5n(#-|VCKC>roI!QH-?GlD6Tx6rn;u}`^<}mL!D!HXXizX~0oDia1&s}n zj=<56mmObiVj^Z8OGiuD;?Njb%S14rJl7TaEex%m5RyS-Z%{FIAO`&%A!u)6$K-4f z429%`5oXngu>>YfeXcBetv3#kAGj*ku@{cYZsI6V9Tt>kHA@K5ehsbY40#3Q^5w01 zDPSR(@AipDRO7u|F*h$G?%{DW zy8n6yqG!1MU{k-j=>&}1Y?i;yyuIe%4|sSy>37{e>+$lBAo?Q@q=7nwGS!CVA57Kl ze``qU$-V_uTz3akpIa@XXPnb`(=5SRMDJ>sm$ZaR21GyFe$FdH07b-bLT-N|Ol9@r zcK9{Za>zjK7)<}X-39Nzf0hoAmw!H$d_onj1BKfgYkaFd_Xl|Xe7ra~PkMT|*`nyt z0r*1*xe6mo6kwp7Fk;9P_*%i}bZ~)$fZ-jGp@(!yQ)sbK+fFSIlr;o{l$i(*g zBN8sRM99LX^8;+rz=zYrX|<(xbZj-H_`V?+uxY2Wu%-z!%Au8NG6kk(toq9Lb`F8z z0E3F<2-&*qmzS4istL_EIpw#Xg!_u?SFp{_9Dft@%x9Qw?BRMZB{DJS{kafT4cW(g zY3oPFSTk<|^6|Cx=cq|?SB(J|4cpnVDmWb6vtKVMc@p4f-L;khHJi5@xJDBZvE!r- zy3z+9(s{YXmjDV#z6)$1VTRZoD0#1X#{X_V?!nx^J4AWV>O=vvd^%+UjFe{Lf)UeKC&X00`PRF&VL{aPV25Qp6 zJl^@@4?RFciX6%f@_<^zguk$Mhh~Ghy5NUU$>TS$_b|ER9A`!Ct5djnf@xETZ6`4r zJMNNiW4AV*@!!UTNrfoNPAw6e$cz{m?}Z2?Vunu175QF|-+#49HWyHIMS&qtqMr!= z_}JooQTKwmQv9}h&Yk^}JAUo43XtCE?LDJ;J%|P9g`}YWsVFVLhcr_J4tuWbo1@+D z3@1Sn4Z|r=I^r6bl{VvBA79YzVJzLacwxQCcIGMwm|A~r5$BK;Zg$%M-HNgZer%cY zv2_FQv+7{heX5quYJy>h0F}hDdQ7v=YV1Cj=`z04i6npH>ACOiu={qWFugw_pX(=i zdNlyJo}?}u!X<;I=usD1e_XPz4XZV6-cX3uFdeWb-;77I8QSv;12uBgIFWkr z!7yk{0zGNCyZ03Yxk3ee))!u%xz}>MrQrcW)cKo-^M{Aq(01EGfZvo?_nxw7pHcIK zg(=oH-rr(HgOn^>8;9_OZhER$(Y_mcAl5$hY8cC0{+vz1#}D(4y3EeVPel_2Rh<098~J^iOB*8 zbZIFhIu1oRKjvD$No3VDR*$Y@N3_o}wJuKrHR_MD8d<#RJ50f0VMDnpw~ZNXcpfU& zr?XjF#@B_Rx|tU0vTV^L{TMIIR{fR4S|xfdUC?`dKOY&;E+W=0yicxRu+dm~nFn0< zPy(N?lid0m=?#2vs)F?CGrL98a5n*%u59!tmfFmvX@@V9uH`sFN(>B{fkh1h5lK6^ z|F}w-eKjq;w>pO#b1=%yQr@G>8`b;-KkUyEfKQr4m;(3cIH>Zf-ty`=SM)zq-utyK zxq?u|HiAGaBFNBYCaIfeVA!ED=By4q1`JalZbzEM7vgA~S2k!#nFJ*NvtSPJ+d9lP z2f|;**WO>}`|R;~QI5Zexc001=iPCX65~u-`JDaS23o*lX8J~l7zFO+UQXK~X8hr= z9kv$wLZ(f1V72Zn|Mdt}j3`E67Zn0ThM@{vPE{LE25OXUqL>3?g1@>sb56V!QmJIo3_{Eu zL2bkvHG%7V-(Tb$b78-_f)I{g2-SZy(xh+jZgXfShEG~ZyTaCVjj0j=bPJ4@-@D1b zU>4-9l_Oy#c22cFYMscetA?itOr65&+C3E+x4xXom6s~gJgFeAas3>+e2&vdNII6n z;P(Vhj^3NppXa?0v#Tc}7A8wh?7zW(cnN7%$vqM#h`my$p2Ebbk(OWZX&40g{voet zNSClUDtr-fsIOXd%;KG}P+i#M)|QYQI}f?QXEy8Ym`wnZTUd*xF1J&# z5I20MNz92O13L~Ik zg`uUenO%u>#3E4gOp2LP*SGLaK3pKK9vgyg(qU?O*MA~H2uq}tiVd4yK8c97i<=%> z0bfm{i>b!Iyf!g8?y`Q`N!d7#!j;=FY%DW`#f;LevG#)qm3GxtbxzGFAbE1STxf!f z?&zd}7%&e_a-!V?NP|ul30%Sf8gD)Ek%e(6-gB;TJK9%!;;>D*VIwE5oiC?LC3khb zBHRusNcm9!p`kws`CLb<<;--DiFpy;FZw=GhB@2TGkbbVLWOx%*gbU5 z=z;|%#u9bbI~V|8T+!Nx?R<;X3{z`;I~ndS*gGL3)VKo+=oPb%m2YD~A5Px8#)0R! zw;tf^d$@@p(_l`6hPYt*yc>pBcU!ie)LHz-zhc7985%kI?88zmJm<%%#SvY+yfRCX z3GqRIieEOLZWU<1Y*$LZ$3to@U+9R46_?C@Qnn>1axD_f zjkM47!F#gLA~2yx(6qqK^%dME|3<)4z;2{^a(5Sj=G^^?mzTP{&@F~VtEofTajy=! z>{Op+Y}IT1{wO-0eP=*rUTT0@ZvElCP|5Rs6dG?BfQEBJsjTF4QGHQJQ{CRb4Ic^# z)yZO(QBe-gPojrcPa_)i!_E?~gWK&`t`S_u{(CM~CoWhoo4}rxqpk10f`ttb#zlWg zprEMnE}`w*E+5hp_Y){w+YvZE{3_jk5BE0DvC(cX+f~~b>r|R;iKE{v%d?50MzU^3e!CRSkEF*E(A>xOd(WTqFSL4{L>@3U`q;4be)frlG^>KZ9EK&pnq2YMv@$@gci=Q9lix0pT5l7DOPC{Ky*{RU@D0_YpRel^`d& z!*B#QC2OucmJ!&m!cXJ}uKY@IFw>TGePMgIBzz2ZM;DWplsR(8F)eL2>y?i_lKjLGJH_cJ`-G;h*mMTEsHQ^ zfTy@&?l-38I8R1)fh;1g_dYv*zOD7MF$VjySL1yu9+O$av(^r91V15othKv@_NwsR zBU$jHPCid?t(rP1paR)U98D3n&G+|mHs4&T`UV_x<4 zNkQ+&mNAkzuT14WQ?0OwjtB=SO;wG*uW?;eH!$=(RVJ!CG|1**JT_0FUWZwL(Bg(g zG~F>0g?;_xH)0n|so7OqmLXR|fVBk}HpASIA^V=S(us**ixOI6GKlfXe!T?1cajr` z>x*zx3Hm4#86skyG^9A~M-u6!$$9>>x&3Qx zqHJSj;HE;)BACP5A=pMOTvoK>cd6_cx6oNi!~+Bxp?oyDskv{RyQCcMCchfL)Uq#P zR>hbr&SojUu)ILdVC0v)0vuU{Q}4LHKm|&}8zk%B)S!NguiTG;D6qyOzTMc#cXhbA z^Y^|ItRb@JC#t@I0lUKub&gmsiyiQ;F(DWly`$%|RoqP>JcD6DBddu0&eU*M<>$9e zuyd5fU4NfHF`6$Z6*E@81~znarB=qa=ufa^KMU>$KhUwsrR-sS2Y}-^Pv0EVt2UlU zHF!VLm4((wG!TW-E%LKION_!F3=7qQIlo{{T`kOs7%1TGr?%ogdwjAgrSh}&i;2y;;xa-goe5v%3)lb&1ptkN3!`eEs3ePuTTIdv zK%LWHK5M;0-KR)s4quajr;rfah8w`QYIFf+X9MRoxTYAL;i?01VPIvS3**mJqwa>Ht9O4Pgv`&s-s!d+v2UBzpC{VIxUe zId`^&JfK`a1zc1&6Ftg5h(f)WOFr!jqMx^T2}0${%w2N%+d+cf#(|!BGGajTT*fW@ z%mAsG6d9gczA z3mNw2$CbP-C|o1@^evojQ65idC4w{%NegSN;KPjP2IN-Wf;JWDc}s={th(kO+xpg? zVtr*Fp}9$A?7^Ofi63A(Jo%Nm=)w^nGSf7S79r<|h+=A@-wW#_e+n#uKfQ+U&p!_h z8FnR9-02Osw5|JDyv&sXZhL>nUz%@G%!!RW&;-mg)z{YASz~)&c(3+(_C^U{jqO0K znVnFL04xiJ&{YpE)GqGQwL4J2wN0~pz|#lftc{fGu?W8`be*^Y!n}p_fsH?65#W3d zL5@g{e`JFAWaFxQZ?15gC2)7~8^VsODu7)Y#DTehn)f5=ZWe37{#vx=6N$CjWoxdy zs&c)qAz1wE0!VaOvXOeiiC57}5pJ)f;mFdF0Km&paSEY4O9pu?{&^%0v>8;Dv{@G|?&??Auh4 zGNApXcA3KV=d?aA`#}`L&OEt}Jc`|Vf-@4x3ANXK%w{Y`h!P1Do6MZMySf&2GnWX_ zQa~Sx!f>yAS6E~Bhi+={16L}ioo8fErw_l;9AsE*rxHyf{3aHXRk1tN>Y7lvXVvLq z1rTDZo5HF6hs)A-ILh3>6@5-ljX`?NDgdc0Vg8ok2XcupSAE4X8EGp-Vgu^K!woj4 z+pv$FSUM-ZGlGbzj9@901#d4ANeRLy&dgaz5hvWJbfOW^fJHyu|6c-JqKNjTF)Nc>pKc z@BN={6|6Cq%YmXo6WwZ-ae@wHF{cAH`SH6e&;3$1p?M2R7+<*^3A`#&xc74K_!q6N z2sd_@?IuLJpVl)oJSpAmA`w@#$VU*lm% zqR+rt{#A6~YmWEwY@qeYsN(?)2tXol5?i78)<2iOO*>_5Cop#rvbT+AhCy(R%{eja zSpO}xNE?noT6KF^`=F;EeBjDCHnRt;pDak1G7~Hhc|!> zRd^A(oaq=Od+Wyzm3BzlFU4Op+t@I&rU+UQvD*_aF{P<*@S(pTc&Ix-Q~>PsQx&z5 z^mm2~!4q;fTMV6c0?#ChSQS5Fnds54GasM{^tC{M(HthV8rBG@-n~eoit;)ly~L?n zk@C>O_VDny4y}vcfBz*p4UDaG_t-i+!11=jEA8Fv5pQH})-S}f-oq{Oh4~v#Ul7ih z&50AcCY#_ijhvPfMb(iRh0dWftT-t1nfZjF^Ce1z5}5Cwkbi*MWIvJ$30xqc4DJ63)czH{B>8(Z z75pDads-|jFag}ZV9bAFs18Y0@koD<(Qb47#%OQiAsg&bML}`N|6cTeVfeim)L{Px z$|5EFJ1F}{vA+VxX&gMjq@wUZkPhE~td@VbKmC6rvwsyq_n#D)!lHuvmzaOu8R;{w z-@hTlTz{iPL+BSMXf(>d8A&tY1*ZCqQ9}ZoH4G5nBI7duCk1+0n4!%6U(od5Ss@9) zN+I+wT>4M{-xl~Qvi+YdkQU7cOpf^5nEw*>Z|?Z3q(6ZEU)_9C z@6fr4hyA}&N&o7BzcWD6H8eTkkLf1=NmU3@k+xfBMB2PZ3-Xx9PQ7f@L<839eJ_#? z$g{iDMOtZOsb85T9>`CGx+$o{&l`nyPK@PiKGd*mLOH$$ z8!0B4VAD^7!2tY%bPR|5*PK6zl#o?D@VHjpFoa`^%G#2p+{!UxR+u zIl=DdPqw|*^J60ghf*Psru=G;OZ9ZbI>I@O>CxgTWw|676WzJD z#l?@3-Rfz&No(EJ>y+BZHWO)^ezW9JMs#=#Mw9RP4RTBbQ5#YP@AVW7R`M0(I3OAF zQm>ts>!|>`cDVE|?yh2ccua0HY4tviYjc5d_Ex>L_K$>Os&Bz3Zq^D23QO~rE$b`! z<4voZf|aBQg=^$cvjO%P%D<6#)uPDbV)9F*JXZ`lAo&%3Fsr(C!WPg%Isn{=wSHvb zNW>C{{LiR`Pf7C4E-*AHUFdV?;dG&7dz1~TZhwG$oeDdHD`KM-`49nChcDdsaWTg7 zX~Cg*VzI0zDcZRKpvM*|ARay*9=^B0d#rA`-d))eT(O(ySHxuHo#gizBv(L3X9(M_ zYfeKx%L*y~T$2Ukg1^Oub%0WNedka8KTk8;_yW>d5Fnr_i2wOC`;#O8p!EEq_53;h z(?XCY%lY36-k*f|*Rr291&@UGXNCG}0ypf!vw{5WNE#d$FvCAmTN8;G1efzarM3qb z@YnyD+Wu%?|L$7F>;bT79Dg&C1}^~2*G&hW4r_D;ucG znhSaF1E?ehu*tonX`oil-<43~!nzxt~f? zeLK=}41EeZ!9q!SkvhHP7S^$-;bF6r5mlws$r_HK71^ZHb2R`t=0R_DZAzq_Mb)&q z;K$DxRGibOrW*DB+9p)B>IGfuXml!A)HAi!oae(cb6R)3l=!F=DUQoU=aC+^dP!!- zn4hwufg4)va?f(?--8=NZmoBtS2EO+OZ={ew6>^$(U&;wi&h|^ zB@+?54~a?RqE<(F7xDwYdWM9JuKg}ZHO4hh+2?#+&uOXO0QY=Y14$ed}v zT&=JSzw825lWq=SP{{XsA4vOvry~fc8}?YoV?OPcJ?0Vo(mHa^3-IAJ9TXsytB%f(SJ(bW~&%68Rsk zdLE7P?Yg>kDc)((`-V{{x1TsfoaO}6n3@3|7-U%IaxjU8Bwpqnl8&?J(mEp_>-=TRP z!z=(gMBtlUsn}q?j8RSOpeC9-wN`&$1N!FH#wb82?J5>8h9h-}4H1?&dq7a_#8BU!pW_o3m463fsTKlhf+{#eZ&#an?eI1EjWqRl5UeRi`?eJ!1031&sMfW3$^YV z4`xVSm26a0#l4`KJAYanRl+g0RE*7RxAbKxA zG4D&FDyHOJjk1S5#ji{_v^Hy;>BE}}hWcz4;o>@@i}Mg6=LHk3MXtLrbTjRd@TpJ< zY=N68`n)YrsAtm1cXt1CLlI9D86}wMH#EfV2iPDHn6K1f=_Beo9Nhltg!tSbS_&sy z3YLloMl~ZlIcP_-EhbQzubniXTljhaFCz_~T-OO;!|iv@FRfhk5{Zz;PS|l3I_{*>~Lge2PAz zp3=hWIr6Yji6h|{C>?u`hm$J``rWN9Re!;&>K#=9Hb8H z{`M6*Hfsa}o?j_m(0ZBI9cpw05KMx<5`q=dUW7kmwD+CXLcbb{As*~3$w-0{W1CD3l>EmGtSg9K)3RR)Te?T(7U&{kp4*) zn(^c_L(0IH@5$`vQbvWVMXg@e?qVo&>>~)5P_`Kx)CV)i=ExUQf=_E^he}p2KuGy4 zvPGnV1($A~J$n##I{1kJ5w^ZJ>-1rfmNmSn9L?SHJ)Q}kr=%nyi=K~5{Nnt6oHp7$ zqvu`(y9|eP%o~Z9qxZ2E;3U`Bht&1-B?EPI-Ep=B+oVPX-y3O6a)wj*$gR?R(?O2p zCcG9INL8;|h7aNJ3*GTbmexB{z{+~AA2Fue+Et2cuo*Zyl;f@;##PR1!Cn+>ZAnU% zI-R3Ue?Ms(eo4-S+7u`-KPG`qr4Y87#Vwvyfz#EJpYoK&7l(Wrto0(yw>a>vKFjlfQT^TbpVw$-htyVS zQVT{fa@5MH054>X_&oH+?_ZBXD`^&*u1E>}n^6VBv*bKpVR@FKQtgVLBPWqG#1uIS zJAMHliE@U3@fXdW%tQ`>kM>|Lg8>K&QyIn%$r&fsR^V$*p~&MzI1h@B6o7b%9zjj9 zHOm-n#99Wq%UUx~%)1+%L`xdtmn}GG4|(5eFKjs&0DxxwMlwJ4+1y_5cqR^U;7?hZ z%n5?+?v8d3)}H5|14HZQ6Ylo4r?=M02`jZ1N|Mo;C<1qy#z3<;EYdgk+GyYwnRlW~ zre@hwnjhLoATKvH9YRGW$5zSk*wQCTJia zRh%SBIb1Z>zu6_Gc?$ruHTcQ(LjC>z_DBGjp}|o-2^Edy?`BI00<$*oT8qN|?d*n7 zHzt_>Rlxq;4m)YW|LZ>r*ol7@u+>_&&g*O_A2s>`YXqd8CgW~vZRI$xw^pA(D=4o8$Gha0|eQSdytu;@L zSg~+x={(DoV1}VgKccu64-R(Lf8su0m+nUd$G+2E6glL%Gfj&PL*e- zNm0d#&E+YQgETZIxoO%YZl=oIR23M_XeH&>k~eB}(almt)|yqx$Wb}7A*!A=>J@bg z{YzipB{Ii|QJcmmRJ2~ooSOPzlHed_Ci+j{j-?dG(5DyNVm@)S*{P_Ao!%9pJC=^T z7{ARd1Kg4jbIo4#irQF?j!5ZXXGGNk!Oza^XB+6Hu;Artsr}M>+he*DE zwVdvJI>AVr&66BMibc%yV#&&U*_0^{N$RVQp>4|fDFITl=FG299KLI#wi8uvul3Ff7Ov8+2VU($-$tj7@ zJ%Eu|B0)UV5)%&>^j#bh`K`mghhPD4_b7h1MdxT*r-kQ!VTLaUS#-ePSDyt3W|LO+bem)4Cj3F@~W4etb; zC9Qbq1X<)OzPfsq8BduoF0}I?dUf|Q3gFaj*X~m69NQF`RZ9zzbJfx;lMjQXTR@QZ zEN&*INudh$c$37ruwQgqD*x`*;P2ST3dTGtR_TbhW|Les&w3LI4q>3tU-!TnZkcaU zt!^!9P<)1_uMi>l&g@m_tZCg*=!HQ8CG6_aSTzwdz`F!rr@W-L;VCaSC>_#X0XXX` ze+~2u#~ck{jcJ}du)6nY8817X&kdfaPgM!w%E zTO6l>^ zcc)KxghxoZ*lUiGAT9c4V(>krii!3TL5w-%%9&6U zihJQs#iWeJC@`KD8C2Mm7Pwp7~`o^0mEHVCl+!CPE^z;mT zwFrFjmf#GLkS0&Osl??@5~I8hq_rey^^+6hkTAr_H?psap{#Mj0Ef~B=7SBHw2 z!>@NOW15hv@MRwea`v|x0h-;m>_05ExtL0da_sGzz{-_@-S%F2`q%?C5Zd_p7EI{8 zwOq5qqB?B%*ybjKY7dB3ds#Fudb&vn0rd$Qjwn|0az@41xeYo8CQLABsZd-KA3#t{ zGu^jl9MSYQtz|8SK$kB&QXcd<{PlCa^~$?)CZtL|hP$`*ocUOzOl40Q?m!Cp{RG5( z+UW>VFH4!hs>Kw{K`yoOtcGm1WD*)LzrMzpO=yq_*$Jy`w5Rxw_sa5ThOhpK}UcZq% zg=)`@nnVaQ-$bv*pSot+^-O7#tdBNK9x=LKdrf?6&b8}jM75~s5 zNNPg+$#AFwr@#PR=p_Sjv{$%d28Mj;l(>RvYK;zi1pBGIEsD{ z&Np4=(}*ZKmqdw)4iRQr!~@Deer98M5P-8iNDNy0RJ3G*T_lXN$Y2v{+=V)PsW&c> zr9Q8sAw_eX#Qqp#z0?>(aE~aOcH(2**uN*9_t5 z%CdxrVGa4T*xdHQ)mW^@v5cNRc7X+0skd_6PO|ZZC{^IY<p)HLzf6fBgm7PC)Hll3G7@x#CJ(6#(+&EdiSNlTCTZ3ag95sAqoNzAjnP%d}mW${_Eis_3Dgpj3l3t z+Zgy;(>N!VE^Y(ka}f8obW5KhWQg@7toHagNj7~c37GL@W*!&zSAN&#@Ju0XI_G!( zs?TX&C+~zygmWDSul#mAM--j;Sw%k;WExTxhw=XX85pMP8h(n|0MWBeKU`7Z1^k5; zMH3Ogndz4$Q-oHw=qs1IK#O4T4-9yk(6!n%AQ$Q`Izg=d7&nAc4qao7s+4NhJ#x3J zISqc5ltG`IaDtQ3hz3*s=yuLZ`Jsp`$cJ&-IfLnVy#U`vtz;UWfTQW^5Sr~i=77kx zNcbiWax#IUVc{doByJ7Bl|Db%YZwFoncd03P!#>LLvj>sZg{W~VO!8bKCNRC zD1wqguh0J51)^{X-ZUC9dUn5FLN%>TDI_&@Q!}GeK&Cv|VX{0Q_PA&WPberSw~RX0wDq?~Bi{f9b0 zk|GvGWXZQpKYkRHxa=sQo0%14(=LU&z4Zncn%C2+heV|}Xz6LeAQS~s8B`$$s(!S4 zASkUI|CD$tiaIx405G zBJ$dX^!Q*T_Rj&=Ya5djzW@4|%Gqn(YAs|saSn9+i}q6Mzrr55lD7Q*tfGQ1|1cT1tW|CbZmnaPgqrZuWd3^FjgY4R%oja!V! z(&sm=0}MGK+S*e_EWOqtrYJRZ*o#&NM@`H#5;8WDy#GG4tx_QWkr1&$r-Yhvk~dDx zU?lcR*sg9!7);_OT9zA^XstY^FVc29bK~kl5%r5})c2-v_7F;h$OOxH*qefVMv=OF zkqQG3D-kO96iXbKnuN`r7l{&p!;o=G2F$<}*e{#2^vyPiHRY1#*F$;c)*q9FXeAJ( z^!Iu(C1D{Ghu6fBr=9y5r4wutUCWx{5TM*obnVLEk*G?@%En%U(SwBxw^4fTWaR^C z9T@r!0s^U5(Tp4s>!O$_BkQoI6z^RX1f}o$kPEiN!zwM_IeUXH?T@7ZJKYLbGYk9P zshVTvzUJ<8&!P(714CKu01ftU)BK9IIqt&N>!pbcLIGc|khQ`lF3EaVM|3 z4?Gi0Q!rS#$IguSYbH!;@BR=kbxvCd-t|Ru(%rI<``H z{f+B4Cy3yyrL&EhyQ;+glRR{VeW-k~#CbD-~62KAUst>b!U3m?B3`Yhj9+eA{~ znYizk?+1@eIF|F5fH3}oZ=iYS)-VOT>6CCfoPld2AZNntckroqWHeAV-ykj)`%9c|vxvbbVhE6@yer(wyRHXkwo_>dUT`8lgmiq`JD^g$0IRNxsv3U0$3V7y46m_Gv#lQ63BC+ ziGtMlc=@2p>A!@&GDvxb;Y7C1y+Nxu7$2r)-<}s4wccI=| zESoI=cRc>q4KVjB#uklv=#yrSmQD;tH*T7AFWhyg^O+ zCO{gmPyd|+=phT(U+`2@p|Gp-8S@kS8vRy}*~3*K(x{&6l;e0Vz~LzYr#4 z(?8x)|0|^WpJ1D$*ErUHpwfRT(|;jF|Es3`J19A+H=g<5YT84db*Fu{)aw_RDz6qw zPtx%;8L~?|?~zlN#N1Pzl1m%&aYxN~D9LE5L?X&cR$bjDf_nRVihHV8GH-eyD5XR? z?zGiSN!1M$Xk6UvS5A&u)Z60_YBfhLbZSNCl+<=tzTdBb?-bq^Q0^$!qj)78YXTIl z5+;Q9D;XHNz2Nmrq+oyAWz)L&R2t||ObtHWm=&lLb?`DN8Mn8_Zq((=Gz@HB3L;3i&bI}&9{5_e5ZX7PRC}H5Ob&(5d zDC+*6#1Ua;*i`Lg$&j-9Qw`+gY!0AE$rtANQ8FZ9g&si*9xho9!;59kSP_3@ zCZjR^bpv+I&U8s*xAI^Qwtd|<18(m*aXwOP?v{~c?)F%U4&Nrad$Zf3#nQla2sma zGjaG;2Y*hEVG9o72@wIpUa$Kku8HrAJI~jr`}5)EG!JKK%sd5!)Dq!DKGKFhlraMm z&KpF7APso5nOd;O@i-(gA->QCc#k=1gxT?w2=Rgm3D`UE$X}JmDNYCw42}clbLpQ8Cen3R%1+sxA@Vr&B4K{F5TeMJAK==ENH!aSauxyvh(@K=GrS9ypQ~mylv! z(x*`1^=VMl>_m7CGRdESHvq$egNJX}e85lxhhhTdxe3UyHD#=yAjY1nJiGGB0qQ_V zchH_PYEsz%CR3VkbuRq%T?G=x2{U;zi*kWN6hIPbunP{1wtiLg0RSpGA@lTRCf zRfH;ode{dAbB%5R)&GGp*gfY;0q(g_2IURH*}p!d(ZpP;9&Q&9Bq&NSz!_$}!iWTa z4@@4IyJ1dhF-y+e7q5^ffuTUt^Hd#N9LbzY60#7LUS$C$rvqYq^+f87@;kYU3l zG%%=F)(zA>4cJBTOxPSAWll6n_qIK%=P7J3(^N~aFKa262(rpeL^{s2su#mA$vmKN z6}CYFS=yrR-BIz$(NxYO?X8EBj}|JRYpm?=j;}@ouMfi34#Jxhk5ZHd6Xx(Ckkz_(jpM?B{kFs^jGfd=qeW)WS|U?KQx{~yB6 zAxM+3U9e@_w$WwVwr$(4uk5NW8(p?-+qP|^i_`zyn23oR6L&V5xyeN?GBe&d&pEG; z&)D!`H(vpMAquqHN@9V1 zsW@$}mXgUdhCm3J4i!jC57&Wx<{lW?B<2WLd_KpG0|3@}a2@y!f{R?^+(>=ItialH z8exqKQqP_7nHZsdz27ikkXw%fNy%FV2G^$}Do_(*sLMiWLrv1@y#>6R`1>ZjQCKc4 zceghj#hH}2FKTB=F_Gdz@NAjj%Lsvu>kTxn0(l@QExcj$uTa()TrW5rx@baKSu`t1 zeTo(S7yy5uRo;%)8^qHN^SsF_!QF%M#sNv@Xs$STq+q^Lu$Fy%!hncX!~juBNJF%mZ z<7AJigoDc@Dl~Y@b!SVX_8iVOQN*22tR9#ce6|$BZOYe<7|y{@Z9-4A7Ne;Pz{}^5 z_t49vP_qd}3Aj>L(|y{)0324k#A`!&$}}GUVFA4b)cRv}i+7vh54>s#;(Y zPjgsJJ4+a!n8)GSbO2Fn`C_l|Z85XM(<+tfvht?l7X!-x$vosC5btN69~h}kj=Q}^ z4zM}s+u5o21Sh@|t5+16@Apc`l&mY#8|e!fr}zSiHlDHzg~HXBMR0`VE+QWx0%Wh^ zB*DYk+f*{@;u-oG2_vVQzmZCNvm)UfsDi?&`GWTFMM?Ebq)MMU$2@bp5-{q-Vxp=3 zV}y?RMEZoAqRKFEzNjoPJD>*WuoS$IHU1q9Lw786@LHsizoLK(5FtEtJG|Ugl1+4G z)C%IOBO_;Mfa^Dm4F3#PqEJiA060P4PA*pg<>tgsoYR=(uyooVKyD79vc%M*8C1ur z$c_(&htX>=;=iJqQzMBRq?P=L{#u{KF+j}VBt$d0hcJ?cKnaYc$j!q(D#K~g1of{j zSm67x68*-@zYe<$(0V11Qc_F57FF;xQk}LB00oj~(m9;Gw z#hZGSE%CIa+`pL?ajGNmETWl7HTP5#Jypky#iq2+Cd9e=lHh`PvLjz+*IO7Y!5Q{0 z%N%inP@r%}+P%=K0wI122_d~2NrtdOnRr2cLO?;{gIOxHO~&LvECe@2C(b@M{U&*z z?fEO694efYP{?oP9%t@K1(-uOL>jR45{${5U3!B!hHW2FGYD~bG}A7C2^C^5Pv0ll ztFi+A_nP2+mW@5&*^MFp2}CFQ18n-(R8>1C8Z>@T3(!-OhNBxkpMC_!q%iswd`^KS zz$O7fGNOg7Umthm>?rt^KBOWk?*t(^fCWM9r{oYt$Mm#V444puWO27>GvYCv ztXFqEA;|xW&Aw*+w8k4Dqo@@T@I)s=qH4WF=b-gs;_@dCfQItcd4P?(mDw1L%}rPwY@#Gf}z;& z2(ovoRpv2mqDZp-0-#h)Kaw`fpMoM9g-}{!@43OwKG{x5nBb@jkNjg&=5=%yF4)x@E160aqmDZ2D0{Sfe-!VYA182e1rTuH%V|mL*zd8uM0+ zqyKefPGqo?J1^GY%dE;N%8#WwN{qj$hc~yYma?H|E3Rm2!NHbCg9t}L9&+K9{hQsZ zvI1t*`Bz%V&k(kvkgjvyC<7U7Orzm%fbpxN`V1P{C1tvKlgtA}FUH6MZ;@pka;n;yQ` z|D6?Xuy2hkot1}s+l)k>yLh{M?4&ULV!a-EJ3NGnj8;JSo%skw1HHRnT`74w0Z1$> zvZyYSmU{oteS2(PyPvPq|KhP+d(3d#YpY^=MK9={2oPf%3EiEK4yU74)CF{i$!p4R z$b@0HZ4?J4Q9F54Shj|$AMgLb1?E(-+3`7jbY9VNc`dZVOewI@MrRZ{4 zB4}a*?uG&s5#Y-k#3_W^mH+~E%8a0;Rv#6xG6mFQtp{Oo;KrMivNCe0?@eEy(D#Pb z)1xFn4=`q&a!@_W^~3mASt3f&yyw<1F)XVw+?NtzmvD4okyQNMU;A;(KdtU<=jV%% zF&g^@<6B*1VL(l<2Ao0P*`iSY;>KSRvM^)QE4=w5m+1)g*NitUQder*{N-+*oPyMH zU_SMg_@apmGc53zPd*-KY_^$jc3hGeSSEsCA7Jy(*AFrur`L!rG!w{)#n3tuqB$z# z26-N}@l~9|Z-CSHizuI6ky9_kBFEhsfi|H2{B79-Z@Oc|H}tuvUTXig0hslV3K-yL z^B8d+aew)icQgT30~1Vh^^k5n>g9a*=j9{#?(laW;)9ApZ{eHzQf0&{Yo#BJFXt75 zJD@*^)+A6X7M@sXTVOeVXpH(#qoXwXh)sP{(@X7M3QvU7dIIrYj~2obfYvwN(z*=?@GCyf(E z;QLHN-&lZm?Z++c?dmqzw3q4o*?bum$p(98yjbYuVun%T z1(o%A@07|g@bCSbcN(LGopI9WrHX4y`Y6utRO00Kp^%Wys%Cbl@MiOdv5)mL2|!&m zj=5jVN(|pME~;~HsluTLZmbg zqPbVA+uhSmq3(o|B5@T#+sxYgLz-)%39_Xr$r}tPl!6t(uN*9=Ez)?%kB<+3m_VaZ zKIdkuEsBi0Fm4Rc`7$ZsaglL(B7owYtO2A)S316owG}-R+@J{s?~*k$613@MBtf7i z!nMG211jN)nuO);w#R2}Y;)#IJ-GD@>+g`NJfoM3q%})+^Uz%26HsMC)OJospP5Gv zqH|{4-72BPR+MFH`9Gq57p+9*xAB`fGJUcG{MA#As%Urx3TX4@zydOE!j+C|6Gc$**Cb`j(X8 zdODI7Ll`R@KV~-zR}8T;Du6*iHmJrqzC8?;qAQ*NDi@)ca+yiK@aotKG~6D44JPAj zCp{sXO`z{k6PKVnyc`=D?!~~IEcwa8zRD`jXWobRfw`yjgUC}S;Q(R5 zYdB%1cY~0+VuuTi8mh4Tpqvr{E`j0R(U0hRPk{nC-DotbZfWDInD#MYF5%H$u4I^^*6LaXinLw|A( zjD$waNHXuFJ@bwfPOfzN$A|LO>wSOaJiJp@V1xUsY~8LQ=eCOdNh*nOrR@G zfb$S7&qL?Vk@yU<8pEC8a9Hz%tjCnDUklRHmrPWSC~Ci$pBY^V*@xHuYu-6!{X_u zgFxlMr0k)+-Po7lRs3b$xP7Ms!6sa;HB zzXI%rH?%9(rtSZ9d9vPxgdv=qUufiY=Co}^mY>t!`hKZ^V+|#Vf!wPSSfZ8!jpbju z+^Ur$5^TjLa{-d=AvjpabJW(cBsel{Wnl;KQ2t;Ullch5QsA^KC3@H|Fl#)%t-$nn zEZ%LPd5r&_ati9s7KQAvIG=FkTtii;dQ~q4Eadowo3oxJahaCf^L5045?B6$#Dwkh zbgZfSF!>@vMh$7ojPs*{Xlkh|Os9&1hm&JHNsISCE*aS1Gp)7h4Zzhc6Lv1a4y*aGh)M^t<>u%Io=(%lpE zgJE2trU2g1o7t*e%m3;1sImqTaup_%pj14N$;bXGFY%53DaE0b1i28%jc`N#`MaZVpLmZ1viSx2*@Po8ry_ z!tnFe`J%7QK$Q^k7|#+1&+`;reK_)p-F#D3eSlj@YP=Uu;|%{_eqF5+%8pErq42tl zZz>+(!89lx+!5^gxkl+$bgsfDf8ygaaQWAG63p4W3UcLkxLa5}>4}zc?VIY(-aHi# zCy8nCFwK=n7|Nz8jqMI@xD-?>!ME9}xS)eFB1&a=H(4-CDw8%J&-fE6YUr0Ye)-S) zM*{LN#&X}=+jc4H|91SgX}}S`zsAQ0(G-U^l-x^k>0W4ywy&eR;Tr`jQhy#2t>QO5 zTrv-BMo^wg2Gf3(SnquIer|Rjon@%wHr}t}?k{SO?I&-`!(a|v3pwk7J@)4PcqjWq ztwSYsyKz>3>YaTY#c$*bSjgS3?=BMf{SL@ub(g5SAv&&YPAxocZ>KaZ0zjg4BSI0J z(8(4f{(zZo6y*KJXa(4l@Jl+_g?qW_4TK2}&ONzs+}O6WX8?UuWGk68nHMZeNPEpr z3T%S4$t`;)1yoyj^^!}pN?EzVN5aSe{k_@Vq<{E4+*nTxokK4*-rdB@jhnN3Oain> zZ|YK#eM4w8(R(}ft)~81$9Q{0vGVvP)`O*+*>h-xZctbU8}$jRSu&q3OrBI@NOmfXE|}|2kp6s>~MuxL8YFX z@4VjzRc4#XZ;AJZUeOx!Ab{xN$T0W4GXm!&xsTm}d4ZRoB@P_J)pLHru^(K#XSJ3U zzn7!PH{s%_%CDW-FuONuCLOuF2TvC+jN0nVB zyDa-LonZmF4T|5f)cM6@Z0RdDo#M2Al8#j#08eQ%T8&bK@b0H|m%Tx~V>8dl3+yr6 z*7M>7f^-^Ig;8n8us2RT$QAhe*1h8`>0WLLACOT(V!-eDpbj6NlVsZPh8}M}$NMQ? zoPE^?uk^e=b36XPH2^-G`wQo4|Js+Wxo73rOT*&7ymSXc`N4OwH{t zpE~kE&WA4XRre}0E|1C#ZiZ!Pli1eF2!aqG@lrN*bq%>gC3h`QAu2Jh*bhjv!fpW_nQHockJFC8;p)A>fdpF~mW;;sR{TW_GiV z20gmj`HyAk8P#^};Qq!k!*D<5r&uHdaFQtc$*R$(#KtzWftHV4%j%c7)(Q9}%p#f( zV(U9Nnbwyl;m*m?W3!AyLM6gC6cu4GE&fa)C8yB{LnnoVAom@j`hJeE&YFD-(TabY zS$J4)>UpcpHh}BnRIXaCqY2(030#Gi2XN5RoqE|jXh9enF2T;JZ!pLj5zKNFHn!oA zI#-}3U`e@kxl+rj523TEV0Ay8wt1%34V^;B_916u*{PCd^SbCn*-QYnh#dt9MoCGM zjt}muzJ7+MciS|04XJk(&&iTvP@3^r(Aa*Q1 zL4A_xcBS4=>zF2*GzT>}m3`+>wWLBmw0{9_9Q=w+P~hD$LK3-8&D@EpVq8s$*~NUV z=n7?l5_^BTW#o}&<@jgsmQ=-yTt&zf`jSj9lZNiI;cWE`2X+~o*rdPVfnf)H@P>s|DNUg(n=yJ` zd&0!qvm)!3k2KC^kGi&znnCzR4EBHd=9@Z;QNBIWR5QD|>Ct7-<|+IIZ^_&+pFI_8 zV7g9v%1tSr%;zdqXi(J<`G$iht3rX=wq`|}A{#%xi?oT3&o<~j!8m}H^}O-ZUMEHY zuga})JRYR&+|&TL)uyaU)tc%nUXm$ zU>;PSs#qSYv$M$@VnWuwR#|jg{MNo*|9;NpbMHPCnDn0YS@HQUd(I^k+^yxe>aJZu zxAuLqSq+-z@HdA7uR&}bhpus_s+gA6Lpc8TKv z#D%FG7RP~PB6h*CK+U=61K?rEu|Huey`#C7xZs@2bkxQxb3a9b9|1`%`J>qPGJHn< zb2a;8v+RJKYz++!U!`=vao6hZ@eV;v zXSSV3phlP8=dV+@w8!gTr<{`-OT(u{efdGQtxX||WA-V?)I{**aMcsWeyvn1%WIS& z*T!%Ko2&}rfnHW{1-;tvvYjIMaDn)&buf+s*i!&7o#Z!DQx*(=NKYvTsmN+Uo*>kR znY~c2df?*_JA;*_C1*7|-!Dq);N*kj8(8pu>C*I z+SYe|BQNG)XCn9EcIYg4 z*@tK6dI;ARDP5%3FQH{CV_AKRLoyEtbnC?n@Gc=AnxCM-6H{wBnQC<3adB{^R>%J) zpEd(O=v$v}l=_6}(<1~ma1CF_v6k*ip(p`sepbvTm$k8Q$`9!QA-_o=l`P8Unq+`w z15zY_$uOG6%wbP@Pbg@-B zw0`k0-UYF4NPS&OY`BaBbLPfEHKc?yaSVEW>n^SP$m>$% zJn-erC8I=~W8UlR+Ev~UR_f8KHhJp@Y>6QRZb|s86 z2w^`4r{K|P!L0doHZ%)IeuS)G1+k+&B0T%`+K(yzSpvhr!nJ$!f)RWz0}+v4>SK|t zueDVdy?sq0Dal2M9>1s_u=E_&L6BC3CksyfDBa4jj(~Fdzg2;}s2uf?cz2NnHWmiq zoP!9UhusqHhLd^Ng|knk?ka#)MmUi|PW0x~ND?AQuFxIJN|?|6Wkir|j;^~6$nC~^ z^#*OY9exS*Hn{g~7`b57Kp_PjIM|%AhxPqLM!)(+bSw?`8ih0USurZpr+-W;VI}uc zrDHpNJ7aMk9Pj?)(xz!m+K|!^l_RL94UHFg;)L9*<q(jDo*_E4M1J|~JG3IV{YzS(frhuSO^Co$7} z%lQ3?jlerb*qezfF%y>4HiHTtmQ6Oi(jyM5IF>%Rxv7qX1>qRx#&xOH6})GxF*tM>dybqBT-{#HUR?CRKnr89T`DMtz)Oq0>y$GdSm zUj)V<#LC*+f1v*GafUcBJE^sW0HAM*rsx;1Z;A^5sr~l3RwiWO@^-J5AUN>&QT>#WI7k2Y|!bw9i`#ixcU_wyqI^DChrU@<(x(S4`47xJ2tz} zi`P@72HiqM{fee<3pKz0m~dAIA}TX};xa306T!gDBM5O_V}VJ$Sj~JsDfEk;--c7$6g#c(pb- zDS#f?s3V}}p2-Uviz=CK1u!qtT!^rU%a~&W0(=#lm6mfEPYXpYw+ERdEpNQofnye>B78 zF*;2w=~SsSs)tdj1W;=s&K`G3!kurNkL7!+aA4g_T7H%)i(+CX8!;l!C%e$x-5-F2 z@134I+;ib172f72qm3x|7}g|}2F^BD0J|Udyk1(dR1%$|XYC)5kKUPDlA zcSZCuOEO~w*IRz#MjmM$O-;8t@ft`4Em<8y5I0Q)jk zxTWIIANB+S37J>D2-ZQR4Vox#X9Xke(n-5r2JV7)w{GkjyAZl{SHh3yW(CwI8bkQ}cS{oLP39^tXV{vqN zVMZ$Dtcd*WNGD6&Eqn(hF_|KG1OZE*h99VcW?-_6^G8Wqh4c0H{iN9~G||N)G->Dn z-nnLshang*j4p{N_h3F2b90#7RV0L&W_Q4Kb9;0a0YteSH`FTdn!fZwP*VPNk`3AA z&YAm2W*cF{rr|V?2BWUjk|YXRS89;TsxgPt79ne^qqx63$bNc#+3`KbSeFMnbLNL@ zfoRrQ#(j?<=auOkkQo$;^3rm~16n(XodF#=MIc3D0MI!fD;0|IjWp!^yrnDp`r4*R z31(6!`uw znPCoY#6X?Z{WD?3Q)WG2^zWB)2_3E?#9E+D2)K7+!Kf}%x+`YvISItXBRC2}mmmRW zaY~aH3Cl@HqMXB@L51Uhm7dz{*}el=daY28&oRF`uLf$K&r0wC@@&BT8*2TE8 z4B+d7)z{84370qY38))5*>=m9A;fbJ5d-w$>Rt1w&9KV-ZHd-#A4`i*H9a0-@3P7$ zEnf9FX=8)yT!%s6#E#)Z%o0r5cg7l=t!mS$EC<_!9T}qK2Repkl);(x_QO9%x&56X zB9da-uuj~{Yx_F`_UMHe|8-Wan6^t$3~+s`=8g#Y(`fNilM!U7NH_pk=s-xX^wHD7_Y1#AFb6-MxNsDte9Y@Cbm*;nWVrgQ9h|w@5jy|H zK~u@fH4C1nbVc?2b1isk9-Jz?l-4=AyCDj!$>v?iWhj&uU=->eO0Ppi+{QVIajlw>WE zzEBIvN}Xzt%3S=BqF>$E5xQjqTqC6lU7MBB8*~aPd?bHf8HvQZAd>Jq58PO--3e`$ z?^j7Q_O%squ4j<$Pt{bNktzT!i=cuC(@k#$F@!qB*7k|c!f3^+^U;m+046ND>f5@| zhLk%InR8v1GQGPfuR?OdHFBqHCQ9j=uYre>5Z{khJ@HRO85D9C2AR3I!o$s}5U0Z7 z(40qE*dq?+`u|3zHukCBO^fWV+DAJ4LdKUA-4^LUXs|-i|GKDD5&i{=!1AeE)QMt# zq~CC#TzH)^`f;!%KzjZ%0x18nJ9Qqo`j?3~b@}yaQi|q@UY_uyx>qcc+##GoUU%TX z7)}*AxXceWS;z@0ojeK_Q2Sm=`p2r{_Sj_lz+Y`26bT z&A`$@uI?G!m!-5TXmVNol5-V|$rGgc%iAp5?!;rz#71uxd>itfC!sO+7ue&MEG}T9 z{1r>v5TYGZ`H!hxcdiT7)Z1sa!LS&SjB{3cWlS}0u-#~+48UH_tm`VXTUefOSj_t@ zqf#z`l8?xcD-Wmg8c;Kal^iIk)`r7*cta`}ybNXCN8y`~qH(Z=$JtO#64dHXEj+p0 z1O)9#>N0NgWZ!)%?<~hA`NXH}Y zsM>_-QA5Y7S9!uH@q}+t^&pDT_;X1kDxCI9^2nyFlxm-oc&Jn5+7-&(;-HS+We6hq z^@bIOYLlFG-UegA$nx(7!Zvt~hVtoqOW9Rut?(}Y{pG`^(Xn`eK3;R_lJjI3rB*4nh5F#u8bq!DLBmYDRBuf$Z3IeqNH zchcYR?8$upbJ&foW!u-PvA=RiV82#@DX6~N0_xDf4(^e`25FJO0_~B* z1nUq{c?wVBdrF)+xP4KPBH%sN?7hp5@$nL&u)Usa3GJ>IaK!7=NNvpMQO_KOEE=f| zVxeB_3V29(tX-uuZBl zLV8|Xb=}@ICF$bxk7Vbb9>u&Nz12}QpdX735_ZgDM%y&rL|UI-I-o!-XnZ*^pr<(K zRQ)8HeBHL;tekHD6EG4z%9yPMzX|XS^K7k0e|`Eb7%1_ku(B z4ya6c9}%fIKD|WikNdQ_aSLfLPOBSZ(T28oVmGn)w!6zQ9?y$QJ?63;JlsdO2XvTS zv-cQ?nv-_Wz*>S4PT(EYzKE~{d6<`9meq>5|x5+ z4+zjt9tK^#6iR|xy{K)Xevi`?D>L`ZLg(5ze`uxk<`+*TMn$~#xIl8w&B5Oys5yZ@ldW3xe1Mp{2{SL5aTbk zZF~O0tMqBy?2hxwdQz~S0GnPG-YM0b2b?ODFw*J^=EHQdlBtRH>AaO75q^6OKTpBd zl4W>3mR*><+?KA~D%6Ot%!XA{5N0X*s7+jW#CqGfpSO6f=Xi0^uGaynXxqw*U8awnDjHfvHdvj zX;wfyl&${r(m zwwizeR$8zI?Pm0PSe=RURhuPP$_~b@7n0&z$DA*E?}*T>X=-pz3Eg56cDq4~Fu-TN zT-pG>&X1`G%dZY^wUgyica~a5j{QyY`V`UB5ZbJZ|7#U5gSU3jnp8(e5Kwzc$)0C_ zHM{0i3wz7QW__Rr^|yLRc+dsAx0LrLETl}2iTFG+_hnG4qQlH!6Wk7ag^seFhI>Wr z!ri24`yBoluPHtp9S9qNepguI*P{y;^#r$c!^ zHfbB`DU4}){RRi@Ubllz2zT@pWwIq#_ZJp!?je^(16VA+#0GEt9UGgsZ2v{wHQH+I zJT8ZR_GsF}3&l%~bXpe5Z%#v^tf!OD40-__iIe^~<(utD$d~s7SisjqVhc?>Vif=m z70uLH_=e{e0;#jU-o8=KJW-hnvCfp)df$W9=)5#q)+|{_YWa|7Vqo5RswDR&F5MR0 zwjfg*zxb^S$61%AM&0Y)6oMYfpygRGzR}cR;8ZU-tP%Tt(7aKoE|gbgdNF4+uqHt> z?Rv3+wRzP3KFTe;W;pYXeMQySa6C}O^%?7aM7Vr;LlEt|u~`W8q>Fgfv3?Dt@AXGj zHQg(xQBQqjQ=xKVbOa+vcs@a^b7i-h&bLz>YW%j$F2Gi(88A#la5k~5`|Oo-7goaf zJllCr$d1AFpgFm<*94f#DObbYjCrfyckKhID+0yl?m$94A4;doo|r>9xt9G3=2+W#?F1WsZE9$8{(+6<*}r{Mew;s^s?q@+hckU z)qv4!2ppi0dHWa*#EKREn9}sxD@c661F|NG(+Ff!tJw{15p6AhHv|FA@(b!v*_4~U zv1O<-G?}_qJ$YrWK5=v_)t#6J*0tJdU&|CPTb~IC6o9pQYN#_xy@~(O^sfA;cZ_x+kY>%{Yz4!rSXaa zC*s5o97xRAN<51om}>$5M~0Pk|jRofST=Jvg*_ zAc^)cjXN-T#`WB#f1m$>|9Gw}klVf(26?*#B73BaIms;$$`*z|US3|5?w3B5iCm={ zzmoBr__}4mLw*E`RZRNvdTd+@GN3*CTyu;SBeExRO-Jm9oj79P>`7WPAmfjRN6CXz zs)88~u)XThI1$e8XCxr;_NmhY_07n5lJTmskpWF|R8~=E3JLCcb=pOT=9bps-wS7w zx>#Pt+76|SbgeOiDm2{U@+osQCKT0bUNixkV`RuC9AmD(wX63XIkPsUQUNU^UhL=; z1DPm_TZq9RNQfB}P5SbxuJjE9QBv--wiE?)_?ML=8nQ#OCtYW@=Kogn_t=k1W|Wzx z*v+q{_w4XWf9=yLt@diYW)7Vc2zxw5&=W1X-U2|bXIBDN3fd~iy4=#NbJMuG5+vc_kq8>HXHwGo)ARNQ z`|4#i1&K;-f+PvL8y6~lOx0f}Up3MoY1>NKgsxOm6dD)rK896v;6ozr$pziOfsl;Ll{b_%X6$tL8s0(PTR8c0*xSDqM3l5EJ zEfsg2nc?ptcBxATg?9fqNQN`@%*C{pw8TeQ>3~5z_Z(@9nP!{4f`)Q_gQ#+RBqB7n zwnG7nP#E808oVe6i2x8)G^NM&{c6Ah&x!IHZ$&XC0#`zre7+FECk0O^-H=u78Y>z6 z0y44~HJU)TR5iy=x&=q*H{)D{8ep)XkcV#o$bf92H5Wr zVKA#f{N{eU#E9-;WD~X*1p7czMno(JEAfjyE-Gq}G)Id(L9fMNttpw*oaK zb|4~IXy!kvga>$`Lb>Il|JxPl$hO4v1`i((#%H2~%Opev_&~kDUUh+T6o}|9v<`Bt z!Z`L6lLuNI{UzE_1&Y!NZcM8}7pv^@y5S_5mRf^IpBXT)@Xo_ScNf{*D1!0~UZ?1| z!DE^=T|j3fittgM1ySBEzRMq^$%XL4YDRR+LcL9;nirJrc9ao}Ot9wjo& zd4QQ1?oOr#PGj_afkcDpuQZBcd;$DC_Hr;2hc77BJCluy3PUdR)QxA?nP8Ux4ulqe z(lsqW?lPBYK;Rd`ZACyz8?2;y(O^$AQSL_=b#Xp?+OOKQ5a<)a>#{KEQ$^qoU?H<7 zyuJq3mj}H4mIStMhJ)S=V=u;mZFGiSJa)V_o1kbXhR^w}ZiYLCx#2P)c~yGNno&}pLxosz&lnbB|T zDQ!onw}|w>n(Bfx!xXgb<^L$EEQ#WhS&!2nFAm_QcMFGX2nf$L|J{LM4R9y4V&aZX zB+;)H$6gYU@P6E8I;QEn%Hm3Zlf$`Q74JZ6mpRVl|? zS^%J~TJMCe%A=h}-oGQ{4k9V*gx~aGr5NIC@jITUqdG1kCC~|#n3-#osPboXGcXF# z9W+szc0u7eCJwS70@^q}%ckINbpNX2NUhGy_!J?YK+RnAWneTM;MMIn(9WuJYoqwn z8!IKQLi6{wB9UyJ0?TsDx_04F|GOF}1K6^jP0{(+!drt8R9V}s*>yW61Cn+Kcx50? z>K1}UxWkDJn}?Fq+b|CFerv428IDDok3siY)xHS9@FGBflh@%b>{Ui-O}*=bC%)Sc zKELRMQ2=_x9rcXGiqeSz^pse=7L~ju4NuU;_7BVb<)8Elb*<|;}@_}1F({h zgIUrdW!D2=b>q~Iwys=Ne~9&jrRF&X1mpUehB6mW8n^{ynF{6NSkQ4UUog@3@QiM3 zvZ<3=s7*$34!qJw*5!Av)z?CMT{=-Wcu(Xd>Otph<)va{m^s|`d`Q$;j-QBJP!RA- z`lz7u`+$zovP;Vay7_C1sp9`40fbW+8+M7J z-$aVR3s6{mn3myf#%&_X!qr9D-A7iXn#ICf+7nC!@&&W9Wg^M1BpudX?`}#Izz;ll zeZ;XZAv78626Jt~vH;x&vENYEnalh!Xqdmoe|m-XkI$6Jh##1%p*IfJ4YjcWuc;W6adBa*M{8=OTP zfxYz43{9y7<(Z3RI5^+zmR$E_D3?$l?)Hhj93gTfqk{PUZkF63S&`uDbL?R|A7_7p zgUX%FW;dKSp|?S$<}FO97W&A!vTGEpVeJ0*0rjbJ5Fz-5MMAiT0w5z}BIVvJxQeX$ zl?XAfkc91zU}g0f&-_~5X>Ys=3I%CtZ!Kxhz{AhAlSSf;z}YSKg1|`Jee>|wL{i2R z(>8QdgzPbM|L1NHF$n5LV#IaBk9$xQ4?^mglCcu6d4_L1GZ1=MX$cl%9Kr_>b0>nnb zQCfb9v-&Oq0kV60Jfk{$G+$PRah$ibWQI{pis|84hH~bs9)=ZwTEvA`sRtX4{YCA+ zscLnWoYz0D%jjB!|-@RjP1=scNKW2DXDvD*($>%kWMfazh|0CN-MpL6vteJ%F1L6rdB zEIEINbH)3G3})YoANi6$!R@==Dryh1(L)zn1r>|ejs_%cRpm--OSB9xA{nXS1U2br zFWklQmkz>0n~RZZIjtBJgk!73 zlvNr$?Y2vtUygr*{WUCsC3DM<_3~+3saN~h>=pC7D%8s&YUPSIzyT?$Fbc8rCV@+2 zj|AJsNcKHM%oSY&ZlS_M7`2YB67}V@PjH2UfVMPUe=+wPjXs(q+NMmLrGpxZ2!tHs zDqixF#Zc{@i=J9`{+owl>Cy0yo6g!7E`D3LNbKAm(wogNDcwp3oB*X2=V|!JVsi{# ziSs%?~!Xq*^}%|h_K@JlFN5BFm)*MVizT$Y--+L26YBjV#F^pZ#| zKq~pn*JrB$;IN`Ih|McMwO@^lZe!0c6rORCmqjh!OH{9TbP?4}Tbtg%1z3D}lP50& zwf~Ws3DU6vu=#qie&$u*a?1Fm8`N4^^+)N=>FUS8#iv0y!(bKmHnpPOHzDrbn2%>4 z21KH_dg!@r2z1r*x;sfyBDaR?izLNF^tu#spmk*Zg2oJd*>BR#SQ=*1Fs{qcf`U!k@w1FP4mwfTS~z@Spi}&JT*zVY(N6QTsYf9#^}fGW{M3 zYTww1^!AG~8j~u98w>6)8NCF4h~V4G|q6Q^|}2 zZ+>zSwv_W`e9$=P3`#gNc~bNz|G+#X;2U7%OINy&y{VTu`WRHW2d{(Yv^`WIMYvUIq?^k+@@PJxiTzd)RT@ z;6CfS#k`#jVe;P7i~toF0BgEG_N%==O`b(ECp92Wn&b2zLTV9+)psE{LZP1p9IiO)*Fbz8lqIzSVt*cNepcTuqdQ+niZlavGYu5!`zxQfWn$$a zN6hEl;QK^zG_vM|Zfh>DQyXt^=$gfHA^6oUo@hGdl4PAT4shl450Kwo^A?UC+^4@j zD*+!2u~3a37)XxQ!}{4`=D#QCtL}#}ll5OMbgBA*NrPhmJgP{3Tl!dYsEc&skJn@5 zZ6+)6PCx9cV}zRL;#$15SxH?BHQ!>NS9%|)^e`6!dy+V^)?~2-6X1?#`YP`VnQouY zHF`J)MD!);Zhi1@tqnvD*q?|m9Cud9!Ri?G^#021ss=1g4K|@|AhCY0C