From f614651a979146b170331e9eeebb11e33316f2a2 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Wed, 1 Apr 2026 17:36:35 +0200 Subject: [PATCH 01/25] fix(Proxy Rotation): Fix an MRO issue The stub shadows the real implementation, and proxy rotation always hits NotImplementedError. Possible fix for #215 Co-Authored-By: Yuval Dinodia <102706514+yetval@users.noreply.github.com> --- scrapling/engines/_browsers/_base.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/scrapling/engines/_browsers/_base.py b/scrapling/engines/_browsers/_base.py index 16a995d..eefb9f3 100644 --- a/scrapling/engines/_browsers/_base.py +++ b/scrapling/engines/_browsers/_base.py @@ -48,9 +48,8 @@ from scrapling.engines.constants import STEALTH_ARGS, HARMFUL_ARGS, DEFAULT_ARGS class SyncSession: _config: "PlaywrightConfig | StealthConfig" _context_options: Dict[str, Any] - - def _build_context_with_proxy(self, proxy: Optional[ProxyType] = None) -> Dict[str, Any]: - raise NotImplementedError # pragma: no cover + if TYPE_CHECKING: + _build_context_with_proxy: Callable[..., Dict[str, Any]] def __init__(self, max_pages: int = 1): self.max_pages = max_pages @@ -216,9 +215,8 @@ class SyncSession: class AsyncSession: _config: "PlaywrightConfig | StealthConfig" _context_options: Dict[str, Any] - - def _build_context_with_proxy(self, proxy: Optional[ProxyType] = None) -> Dict[str, Any]: - raise NotImplementedError # pragma: no cover + if TYPE_CHECKING: + _build_context_with_proxy: Callable[..., Dict[str, Any]] def __init__(self, max_pages: int = 1): self.max_pages = max_pages From 966e17a1dc9088c178f41dddb7fc02f752675b38 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Wed, 1 Apr 2026 17:38:54 +0200 Subject: [PATCH 02/25] build: pump version up --- agent-skill/Scrapling-Skill/SKILL.md | 4 ++-- agent-skill/Scrapling-Skill/examples/README.md | 2 +- pyproject.toml | 2 +- scrapling/__init__.py | 2 +- server.json | 4 ++-- setup.cfg | 2 +- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/agent-skill/Scrapling-Skill/SKILL.md b/agent-skill/Scrapling-Skill/SKILL.md index 1fddd58..60168a1 100644 --- a/agent-skill/Scrapling-Skill/SKILL.md +++ b/agent-skill/Scrapling-Skill/SKILL.md @@ -1,7 +1,7 @@ --- name: scrapling-official description: Scrape web pages using Scrapling with anti-bot bypass (like Cloudflare Turnstile), stealth headless browsing, spiders framework, adaptive scraping, and JavaScript rendering. Use when asked to scrape, crawl, or extract data from websites; web_fetch fails; the site has anti-bot protections; write Python code to scrape/crawl; or write spiders. -version: "0.4.3" +version: "0.4.4" license: Complete terms in LICENSE.txt metadata: homepage: "https://scrapling.readthedocs.io/en/latest/index.html" @@ -40,7 +40,7 @@ Blazing fast crawls with real-time stats and streaming. Built by Web Scrapers fo Create a virtual Python environment through any way available, like `venv`, then inside the environment do: -`pip install "scrapling[all]>=0.4.3"` +`pip install "scrapling[all]>=0.4.4"` Then do this to download all the browsers' dependencies: diff --git a/agent-skill/Scrapling-Skill/examples/README.md b/agent-skill/Scrapling-Skill/examples/README.md index 3f753c5..0c2f031 100644 --- a/agent-skill/Scrapling-Skill/examples/README.md +++ b/agent-skill/Scrapling-Skill/examples/README.md @@ -9,7 +9,7 @@ All examples collect **all 100 quotes across 10 pages**. Make sure Scrapling is installed: ```bash -pip install "scrapling[all]>=0.4.3" +pip install "scrapling[all]>=0.4.4" scrapling install --force ``` diff --git a/pyproject.toml b/pyproject.toml index 8e468fa..bb69194 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta" [project] name = "scrapling" # Static version instead of a dynamic version so we can get better layer caching while building docker, check the docker file to understand -version = "0.4.3" +version = "0.4.4" description = "Scrapling is an undetectable, powerful, flexible, high-performance Python library that makes Web Scraping easy and effortless as it should be!" readme = {file = "README.md", content-type = "text/markdown"} license = {file = "LICENSE"} diff --git a/scrapling/__init__.py b/scrapling/__init__.py index e95acff..5d20532 100644 --- a/scrapling/__init__.py +++ b/scrapling/__init__.py @@ -1,5 +1,5 @@ __author__ = "Karim Shoair (karim.shoair@pm.me)" -__version__ = "0.4.3" +__version__ = "0.4.4" __copyright__ = "Copyright (c) 2024 Karim Shoair" from typing import Any, TYPE_CHECKING diff --git a/server.json b/server.json index 796f94b..6ef6d9b 100644 --- a/server.json +++ b/server.json @@ -14,12 +14,12 @@ "mimeType": "image/png" } ], - "version": "0.4.3", + "version": "0.4.4", "packages": [ { "registryType": "pypi", "identifier": "scrapling", - "version": "0.4.3", + "version": "0.4.4", "runtimeHint": "uvx", "packageArguments": [ { diff --git a/setup.cfg b/setup.cfg index eb133a5..7dcdf68 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = scrapling -version = 0.4.3 +version = 0.4.4 author = Karim Shoair author_email = karim.shoair@pm.me description = Scrapling is an undetectable, powerful, flexible, high-performance Python library that makes Web Scraping easy and effortless as it should be! From 6c6aabeb73a872202db5bc295890226859b5eccb Mon Sep 17 00:00:00 2001 From: yetval Date: Thu, 2 Apr 2026 11:57:17 -0400 Subject: [PATCH 03/25] fix: proxy rotation page pool leak --- scrapling/engines/_browsers/_base.py | 6 ++++++ tests/fetchers/test_pages.py | 9 +++++++++ 2 files changed, 15 insertions(+) diff --git a/scrapling/engines/_browsers/_base.py b/scrapling/engines/_browsers/_base.py index eefb9f3..2d42b08 100644 --- a/scrapling/engines/_browsers/_base.py +++ b/scrapling/engines/_browsers/_base.py @@ -196,11 +196,14 @@ class SyncSession: context_options = self._build_context_with_proxy(proxy) context: BrowserContext = self.browser.new_context(**context_options) + page_info = None try: context = self._initialize_context(self._config, context) page_info = self._get_page(timeout, extra_headers, disable_resources, blocked_domains, context=context) yield page_info finally: + if page_info is not None and page_info in self.page_pool.pages: + self.page_pool.pages.remove(page_info) context.close() else: # Standard mode: use PagePool with persistent context @@ -380,6 +383,7 @@ class AsyncSession: context_options = self._build_context_with_proxy(proxy) context: AsyncBrowserContext = await self.browser.new_context(**context_options) + page_info = None try: context = await self._initialize_context(self._config, context) page_info = await self._get_page( @@ -387,6 +391,8 @@ class AsyncSession: ) yield page_info finally: + if page_info is not None and page_info in self.page_pool.pages: + self.page_pool.pages.remove(page_info) await context.close() else: # Standard mode: use PagePool with persistent context diff --git a/tests/fetchers/test_pages.py b/tests/fetchers/test_pages.py index 69dc4ba..b726c74 100644 --- a/tests/fetchers/test_pages.py +++ b/tests/fetchers/test_pages.py @@ -84,6 +84,15 @@ class TestPagePool: with pytest.raises(RuntimeError): pool.add_page(Mock()) + def test_proxy_rotation_pool_leak(self): + pool = PagePool(max_pages=1) + page_info = pool.add_page(Mock()) + assert pool.pages_count == 1 + pool.pages.remove(page_info) + assert pool.pages_count == 0 + pool.add_page(Mock()) + assert pool.pages_count == 1 + def test_cleanup_error_pages(self): From 1d15349e07d91c34cc0cd7f26076bdf150882fcd Mon Sep 17 00:00:00 2001 From: Abdullah <52079299+AbdullahY36@users.noreply.github.com> Date: Fri, 3 Apr 2026 15:08:33 +0200 Subject: [PATCH 04/25] feat(deps): add protego for robots.txt parsing and fix pyright type error in static.py --- pyproject.toml | 1 + scrapling/engines/static.py | 1 + 2 files changed, 2 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index bb69194..3aee4f1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -67,6 +67,7 @@ dependencies = [ "tld>=0.13.2", "w3lib>=2.4.1", "typing_extensions", + "protego>=0.4.0", ] [project.optional-dependencies] diff --git a/scrapling/engines/static.py b/scrapling/engines/static.py index 1f4b09b..a962ccf 100644 --- a/scrapling/engines/static.py +++ b/scrapling/engines/static.py @@ -250,6 +250,7 @@ class _SyncSessionLogic(_ConfigurationLogic): request_args = self._merge_request_args(stealth=stealth, proxy=proxy, **kwargs) try: response = session.request(method, **request_args) + assert response is not None result = ResponseFactory.from_http_request(response, selector_config, meta={"proxy": proxy}) return result except CurlError as e: # pragma: no cover From 0bbe62fc7f7011e28dc3d535d36d32944bd30d9a Mon Sep 17 00:00:00 2001 From: Abdullah <52079299+AbdullahY36@users.noreply.github.com> Date: Fri, 3 Apr 2026 15:08:33 +0200 Subject: [PATCH 05/25] feat(spiders): implement RobotsTxtManager with concurrent fetch deduplication --- scrapling/spiders/robotstxt.py | 169 +++++++++++++++++++++++++++++++++ 1 file changed, 169 insertions(+) create mode 100644 scrapling/spiders/robotstxt.py diff --git a/scrapling/spiders/robotstxt.py b/scrapling/spiders/robotstxt.py new file mode 100644 index 0000000..4e8612f --- /dev/null +++ b/scrapling/spiders/robotstxt.py @@ -0,0 +1,169 @@ +from asyncio import Event +from urllib.parse import urlparse + +from protego import Protego + +from scrapling.core._types import Dict, Optional, Callable, Awaitable +from scrapling.core.utils import log + + +class RobotsTxtManager: + """Manages fetching, parsing, and caching of robots.txt files. + + Accepts a fetch callable ``(url: str, sid: str) -> Awaitable[Response]`` + so it stays decoupled from any specific session or transport layer. + + All public methods accept only ``(url, sid)`` — domain and scheme are + derived internally from the URL so callers don't pass redundant data. + + Handles all standard robots.txt directives including: + - User-agent specific rules + - Allow/Disallow directives (including wildcards and $ anchors) + - Crawl-delay directives + + Deduplicates concurrent robots.txt fetches for the same domain — if multiple + requests for the same domain arrive before the first fetch completes, they + all wait for that single fetch instead of triggering redundant requests. + """ + + def __init__(self, fetch_fn: Callable[[str, str], Awaitable]): + self._fetch_fn = fetch_fn + self._cache: Dict[tuple[str, str], Protego] = {} + self._inflight: Dict[tuple[str, str], Event] = {} + + async def _get_parser(self, url: str, sid: str) -> Protego: + parsed = urlparse(url) + domain = parsed.netloc + scheme = parsed.scheme or "https" + cache_key = (domain, sid) + + # Return cached parser if available + if cache_key in self._cache: + return self._cache[cache_key] + + # If a fetch is already in-flight for this domain, wait for it to complete + if cache_key in self._inflight: + await self._inflight[cache_key].wait() + return self._cache[cache_key] + + # Mark fetch as in-flight to deduplicate concurrent requests + event = Event() + self._inflight[cache_key] = event + + try: + robots_url = f"{scheme}://{domain}/robots.txt" + content = "" + try: + response = await self._fetch_fn(robots_url, sid) + if response.status == 200: + content = response.body.decode(response.encoding, errors="replace") + except Exception as e: + log.warning(f"Failed to fetch robots.txt for {domain}: {e}") + + try: + parser = Protego.parse(content) + except Exception as e: + log.warning(f"Failed to parse robots.txt for {domain}: {e}") + parser = Protego.parse("") + + self._cache[cache_key] = parser + finally: + event.set() + del self._inflight[cache_key] + + return parser + + async def can_fetch(self, url: str, sid: str) -> bool: + """Check if a URL can be fetched according to the domain's robots.txt. + + Handles: + - User-agent specific rules (e.g., User-agent: SpinarakBot) + - Wildcard user-agent rules (User-agent: *) + - Allow/Disallow directives with wildcards (e.g., /*.pdf$) + - Allow directives that override Disallow (e.g., Allow: /admin/public-docs/) + + Uses the wildcard user-agent (*) which matches standard robots.txt directives + that apply to all bots. This is the conservative approach — if a URL is + disallowed for all bots, we respect that. + + Args: + url: The full URL to check + sid: Session ID for fetching robots.txt + + Returns: + True if the URL can be fetched, False otherwise + """ + parser = await self._get_parser(url, sid) + return parser.can_fetch(url, "*") + + async def get_crawl_delay(self, url: str, sid: str) -> Optional[float]: + """Get the crawl delay for this crawler. + + Uses the wildcard user-agent (*) to get the general crawl delay + that applies to all bots. + + Args: + url: Any URL on the domain to check + sid: Session ID for fetching robots.txt + + Returns: + The crawl delay in seconds, or None if not specified + """ + parser = await self._get_parser(url, sid) + delay = parser.crawl_delay("*") + return float(delay) if delay is not None else None + + async def get_request_rate(self, url: str, sid: str) -> Optional[tuple[int, int]]: + """Get the request rate for this crawler. + + Uses the wildcard user-agent (*) to get the general request rate + that applies to all bots. + + Args: + url: Any URL on the domain to check + sid: Session ID for fetching robots.txt + + Returns: + A tuple of (requests, seconds) if specified, or None if not specified + """ + parser = await self._get_parser(url, sid) + rate = parser.request_rate("*") + if rate is not None: + return (rate.requests, rate.seconds) + return None + + async def _get_delay_directives(self, url: str, sid: str) -> tuple[Optional[float], Optional[tuple[int, int]]]: + """Return both crawl-delay and request-rate in a single parser lookup. + + Args: + url: Any URL on the domain to check + sid: Session ID for fetching robots.txt + + Returns: + A tuple of (crawl_delay, request_rate) where crawl_delay is in seconds + or None, and request_rate is (requests, seconds) or None. + """ + parser = await self._get_parser(url, sid) + c_delay = parser.crawl_delay("*") + rate = parser.request_rate("*") + return ( + float(c_delay) if c_delay is not None else None, + (rate.requests, rate.seconds) if rate is not None else None, + ) + + def clear_cache(self, domain: Optional[str] = None, sid: Optional[str] = None) -> None: + """Clear the robots.txt cache. + + Args: + domain: If specified, only clear cache for this domain + sid: If specified, only clear cache for this session ID + If both are None, clears the entire cache + """ + if domain is None and sid is None: + self._cache.clear() + else: + keys_to_remove = [ + key for key in self._cache if (domain is None or key[0] == domain) and (sid is None or key[1] == sid) + ] + for key in keys_to_remove: + del self._cache[key] From 5c40c6a8539282fcf6573f82f9152099c41120fa Mon Sep 17 00:00:00 2001 From: Abdullah <52079299+AbdullahY36@users.noreply.github.com> Date: Fri, 3 Apr 2026 15:08:33 +0200 Subject: [PATCH 06/25] feat(spiders): integrate robots.txt compliance into the crawl engine --- scrapling/spiders/engine.py | 84 ++++++++++++++++++++++++++++++++++--- scrapling/spiders/result.py | 2 + scrapling/spiders/spider.py | 3 ++ 3 files changed, 83 insertions(+), 6 deletions(-) diff --git a/scrapling/spiders/engine.py b/scrapling/spiders/engine.py index d77f838..cf1715e 100644 --- a/scrapling/spiders/engine.py +++ b/scrapling/spiders/engine.py @@ -10,6 +10,7 @@ from scrapling.core.utils import log from scrapling.spiders.request import Request from scrapling.spiders.scheduler import Scheduler from scrapling.spiders.session import SessionManager +from scrapling.spiders.robotstxt import RobotsTxtManager from scrapling.spiders.result import CrawlStats, ItemList from scrapling.spiders.checkpoint import CheckpointManager, CheckpointData from scrapling.core._types import Dict, Union, Optional, TYPE_CHECKING, Any, AsyncGenerator @@ -41,8 +42,18 @@ class CrawlerEngine: ) self.stats = CrawlStats() + if self.spider.robots_txt_obey: + + async def _fetch_robots(url: str, sid: str): + return await self.session_manager.fetch(Request(url, sid=sid)) + + self._robots_manager: Optional[RobotsTxtManager] = RobotsTxtManager(_fetch_robots) + else: + self._robots_manager = None + self._global_limiter = CapacityLimiter(spider.concurrent_requests) self._domain_limiters: dict[str, CapacityLimiter] = {} + self._domain_delays: dict[str, float] = {} self._allowed_domains: set[str] = spider.allowed_domains or set() self._active_tasks: int = 0 @@ -68,13 +79,58 @@ class CrawlerEngine: return True return False + async def _get_domain_delay(self, request: Request) -> float: + """Resolve the effective download delay for a domain. + + Takes the max of the spider's configured delay and any robots.txt + directives (Crawl-delay / Request-rate). Result is cached per domain. + Also pre-creates a per-domain concurrency limiter of 1 when robots.txt + enforces any delay, before the caller acquires it via _rate_limiter(). + """ + robots_manager = self._robots_manager + if robots_manager is None: + return self.spider.download_delay + + domain = request.domain + + # Return cached delay if available + if domain in self._domain_delays: + return self._domain_delays[domain] + + # Fetch both robots.txt directives in a single parser lookup + c_delay, r_rate = await robots_manager._get_delay_directives(request.url, request.sid) + + delay = self.spider.download_delay + robots_enforced_delay = False + + if r_rate: + req_count, period = r_rate + if req_count > 0: + delay = max(delay, period / req_count) + robots_enforced_delay = True + + if c_delay is not None: + delay = max(delay, c_delay) + robots_enforced_delay = True + + self._domain_delays[domain] = delay + + # Enforce 1 concurrent request for this domain when robots.txt adds a delay + if robots_enforced_delay and delay > 0 and domain not in self._domain_limiters: + if self.spider.concurrent_requests_per_domain: + log.warning( + f"robots.txt for {domain} enforces a delay, overriding" + f" concurrent_requests_per_domain={self.spider.concurrent_requests_per_domain} with 1" + ) + self._domain_limiters[domain] = CapacityLimiter(1) + + return delay + def _rate_limiter(self, domain: str) -> CapacityLimiter: """Get or create a per-domain concurrency limiter if enabled, otherwise use the global limiter.""" if self.spider.concurrent_requests_per_domain: - if domain not in self._domain_limiters: - self._domain_limiters[domain] = CapacityLimiter(self.spider.concurrent_requests_per_domain) - return self._domain_limiters[domain] - return self._global_limiter + self._domain_limiters.setdefault(domain, CapacityLimiter(self.spider.concurrent_requests_per_domain)) + return self._domain_limiters.get(domain, self._global_limiter) def _normalize_request(self, request: Request) -> None: """Normalize request fields before enqueueing. @@ -87,9 +143,21 @@ class CrawlerEngine: async def _process_request(self, request: Request) -> None: """Download and process a single request.""" + if self._robots_manager: + can_fetch = await self._robots_manager.can_fetch(request.url, request.sid) + if not can_fetch: + self.stats.robots_disallowed_count += 1 + log.debug(f"Request disallowed by robots.txt: {request.url}") + return + # Must be called before _rate_limiter: may create CapacityLimiter(1) in _domain_limiters + # when robots.txt enforces a delay, which _rate_limiter then picks up. + delay = await self._get_domain_delay(request) + else: + delay = self.spider.download_delay + async with self._rate_limiter(request.domain): - if self.spider.download_delay: - await anyio.sleep(self.spider.download_delay) + if delay: + await anyio.sleep(delay) if request._session_kwargs.get("proxy"): self.stats.proxies.append(request._session_kwargs["proxy"]) @@ -227,15 +295,19 @@ class CrawlerEngine: self._pause_requested = False self._force_stop = False self.stats = CrawlStats(start_time=anyio.current_time()) + self._domain_limiters.clear() + self._domain_delays.clear() # Check for existing checkpoint resuming = (await self._restore_from_checkpoint()) if self._checkpoint_system_enabled else False self._last_checkpoint_time = anyio.current_time() async with self.session_manager: + # Set stats from spider configuration self.stats.concurrent_requests = self.spider.concurrent_requests self.stats.concurrent_requests_per_domain = self.spider.concurrent_requests_per_domain self.stats.download_delay = self.spider.download_delay + await self.spider.on_start(resuming=resuming) try: diff --git a/scrapling/spiders/result.py b/scrapling/spiders/result.py index 08a7658..b374152 100644 --- a/scrapling/spiders/result.py +++ b/scrapling/spiders/result.py @@ -47,6 +47,7 @@ class CrawlStats: concurrent_requests_per_domain: int = 0 failed_requests_count: int = 0 offsite_requests_count: int = 0 + robots_disallowed_count: int = 0 response_bytes: int = 0 items_scraped: int = 0 items_dropped: int = 0 @@ -95,6 +96,7 @@ class CrawlStats: "sessions_requests_count": self.sessions_requests_count, "failed_requests_count": self.failed_requests_count, "offsite_requests_count": self.offsite_requests_count, + "robots_disallowed_count": self.robots_disallowed_count, "blocked_requests_count": self.blocked_requests_count, "response_status_count": self.response_status_count, "response_bytes": self.response_bytes, diff --git a/scrapling/spiders/spider.py b/scrapling/spiders/spider.py index 4f38912..6aaa24f 100644 --- a/scrapling/spiders/spider.py +++ b/scrapling/spiders/spider.py @@ -72,6 +72,9 @@ class Spider(ABC): start_urls: list[str] = [] allowed_domains: Set[str] = set() + # Robots.txt compliance + robots_txt_obey: bool = False + # Concurrency settings concurrent_requests: int = 4 concurrent_requests_per_domain: int = 0 From 132f33c84611c10d8c2fb2082fb63266bf1cc809 Mon Sep 17 00:00:00 2001 From: Abdullah <52079299+AbdullahY36@users.noreply.github.com> Date: Fri, 3 Apr 2026 15:08:34 +0200 Subject: [PATCH 07/25] test(spiders): add comprehensive test suite for robots.txt compliance --- tests/spiders/test_engine.py | 2 + tests/spiders/test_robotstxt.py | 615 ++++++++++++++++++++++++++++++++ 2 files changed, 617 insertions(+) create mode 100644 tests/spiders/test_robotstxt.py diff --git a/tests/spiders/test_engine.py b/tests/spiders/test_engine.py index b7bfd0f..e362036 100644 --- a/tests/spiders/test_engine.py +++ b/tests/spiders/test_engine.py @@ -83,6 +83,7 @@ class MockSpider: is_blocked_fn=None, on_scraped_item_fn=None, retry_blocked_request_fn=None, + robots_txt_obey: bool = False, ): self.concurrent_requests = concurrent_requests self.concurrent_requests_per_domain = concurrent_requests_per_domain @@ -93,6 +94,7 @@ class MockSpider: self.fp_include_headers = fp_include_headers self.fp_keep_fragments = fp_keep_fragments self.name = "test_spider" + self.robots_txt_obey = robots_txt_obey # Tracking lists self.on_start_calls: list[dict] = [] diff --git a/tests/spiders/test_robotstxt.py b/tests/spiders/test_robotstxt.py new file mode 100644 index 0000000..5a447c0 --- /dev/null +++ b/tests/spiders/test_robotstxt.py @@ -0,0 +1,615 @@ +"""Tests for RobotsTxtManager.""" + +import asyncio + +import pytest + +from scrapling.spiders.robotstxt import RobotsTxtManager +from scrapling.core._types import List, Optional + + +# --------------------------------------------------------------------------- +# Fixtures and helpers +# --------------------------------------------------------------------------- + + +class MockResponse: + """Minimal response stub matching the shape _get_parser expects.""" + + def __init__(self, status: int = 200, body: bytes = b"", encoding: str = "utf-8"): + self.status = status + self.body = body + self.encoding = encoding + + +def make_fetch_fn(status: int = 200, content: str = "", encoding: str = "utf-8"): + """Return an async fetch callable that returns a fixed response. + + Attaches a `.calls` list so tests can assert how many times it was invoked + and with which arguments. + """ + calls: List[tuple] = [] + + async def _fetch(url: str, sid: str) -> MockResponse: + calls.append((url, sid)) + return MockResponse(status=status, body=content.encode(encoding), encoding=encoding) + + _fetch.calls = calls # type: ignore[attr-defined] + return _fetch + + +# --------------------------------------------------------------------------- +# Shared robots.txt fixtures +# --------------------------------------------------------------------------- + +ROBOTS_BASIC = """\ +User-agent: * +Disallow: /admin/ +Crawl-delay: 2 +""" + +ROBOTS_WITH_RATE = """\ +User-agent: * +Request-rate: 1/10 +Disallow: /private/ +""" + +ROBOTS_WITH_SITEMAP = """\ +User-agent: * +Disallow: + +Sitemap: https://example.com/sitemap.xml +Sitemap: https://example.com/sitemap2.xml +""" + +ROBOTS_ALLOW_OVERRIDE = """\ +User-agent: * +Disallow: /secret/ +Allow: /secret/public.html +""" + +ROBOTS_DISALLOW_ALL = """\ +User-agent: * +Disallow: / +""" + + +# --------------------------------------------------------------------------- +# Tests: can_fetch +# --------------------------------------------------------------------------- + + +class TestCanFetch: + @pytest.mark.asyncio + async def test_allowed_url_returns_true(self): + mgr = RobotsTxtManager(make_fetch_fn(content=ROBOTS_BASIC)) + + assert await mgr.can_fetch("https://example.com/products", "s1") is True + + @pytest.mark.asyncio + async def test_disallowed_url_returns_false(self): + mgr = RobotsTxtManager(make_fetch_fn(content=ROBOTS_BASIC)) + + assert await mgr.can_fetch("https://example.com/admin/", "s1") is False + + @pytest.mark.asyncio + async def test_disallowed_subpath_returns_false(self): + mgr = RobotsTxtManager(make_fetch_fn(content=ROBOTS_BASIC)) + + assert await mgr.can_fetch("https://example.com/admin/users", "s1") is False + + @pytest.mark.asyncio + async def test_root_url_is_allowed(self): + mgr = RobotsTxtManager(make_fetch_fn(content=ROBOTS_BASIC)) + + assert await mgr.can_fetch("https://example.com/", "s1") is True + + @pytest.mark.asyncio + async def test_allow_directive_overrides_disallow(self): + mgr = RobotsTxtManager(make_fetch_fn(content=ROBOTS_ALLOW_OVERRIDE)) + + assert await mgr.can_fetch("https://example.com/secret/public.html", "s1") is True + assert await mgr.can_fetch("https://example.com/secret/private.html", "s1") is False + + @pytest.mark.asyncio + async def test_disallow_all_blocks_every_path(self): + mgr = RobotsTxtManager(make_fetch_fn(content=ROBOTS_DISALLOW_ALL)) + + assert await mgr.can_fetch("https://example.com/", "s1") is False + assert await mgr.can_fetch("https://example.com/page", "s1") is False + assert await mgr.can_fetch("https://example.com/a/b/c", "s1") is False + + @pytest.mark.asyncio + async def test_empty_robots_allows_everything(self): + mgr = RobotsTxtManager(make_fetch_fn(content="")) + + assert await mgr.can_fetch("https://example.com/anything", "s1") is True + assert await mgr.can_fetch("https://example.com/admin/secret", "s1") is True + + @pytest.mark.asyncio + async def test_non_200_response_allows_everything(self): + for status in [403, 404, 500, 503]: + mgr = RobotsTxtManager(make_fetch_fn(status=status)) + result = await mgr.can_fetch("https://example.com/page", "s1") + assert result is True, f"Expected True for HTTP {status}" + + @pytest.mark.asyncio + async def test_fetch_error_allows_everything(self): + async def failing_fetch(url: str, sid: str) -> MockResponse: + raise ConnectionError("network failure") + + mgr = RobotsTxtManager(failing_fetch) + + assert await mgr.can_fetch("https://example.com/page", "s1") is True + + @pytest.mark.asyncio + async def test_wildcard_path_pattern(self): + content = "User-agent: *\nDisallow: /*.pdf$" + mgr = RobotsTxtManager(make_fetch_fn(content=content)) + + assert await mgr.can_fetch("https://example.com/report.pdf", "s1") is False + assert await mgr.can_fetch("https://example.com/report.html", "s1") is True + + @pytest.mark.asyncio + async def test_returns_bool(self): + mgr = RobotsTxtManager(make_fetch_fn(content=ROBOTS_BASIC)) + result = await mgr.can_fetch("https://example.com/", "s1") + assert isinstance(result, bool) + + +# --------------------------------------------------------------------------- +# Tests: get_crawl_delay +# --------------------------------------------------------------------------- + + +class TestGetCrawlDelay: + @pytest.mark.asyncio + async def test_returns_float_when_set(self): + mgr = RobotsTxtManager(make_fetch_fn(content=ROBOTS_BASIC)) + + delay = await mgr.get_crawl_delay("https://example.com/", "s1") + + assert delay == 2.0 + assert isinstance(delay, float) + + @pytest.mark.asyncio + async def test_returns_none_when_not_set(self): + content = "User-agent: *\nDisallow: /admin/" + mgr = RobotsTxtManager(make_fetch_fn(content=content)) + + assert await mgr.get_crawl_delay("https://example.com/", "s1") is None + + @pytest.mark.asyncio + async def test_returns_none_for_empty_robots(self): + mgr = RobotsTxtManager(make_fetch_fn(content="")) + + assert await mgr.get_crawl_delay("https://example.com/", "s1") is None + + @pytest.mark.asyncio + async def test_returns_none_on_fetch_error(self): + async def failing_fetch(url: str, sid: str) -> MockResponse: + raise ConnectionError("network failure") + + mgr = RobotsTxtManager(failing_fetch) + + assert await mgr.get_crawl_delay("https://example.com/", "s1") is None + + @pytest.mark.asyncio + async def test_returns_none_for_non_200_response(self): + mgr = RobotsTxtManager(make_fetch_fn(status=404)) + + assert await mgr.get_crawl_delay("https://example.com/", "s1") is None + + @pytest.mark.asyncio + async def test_fractional_delay(self): + content = "User-agent: *\nCrawl-delay: 0.5" + mgr = RobotsTxtManager(make_fetch_fn(content=content)) + + delay = await mgr.get_crawl_delay("https://example.com/", "s1") + + assert delay == 0.5 + + @pytest.mark.asyncio + async def test_url_path_does_not_affect_result(self): + """Any URL on the same domain should return the same delay.""" + mgr = RobotsTxtManager(make_fetch_fn(content=ROBOTS_BASIC)) + + d1 = await mgr.get_crawl_delay("https://example.com/", "s1") + d2 = await mgr.get_crawl_delay("https://example.com/deep/path/page.html", "s1") + + assert d1 == d2 + + +# --------------------------------------------------------------------------- +# Tests: get_request_rate +# --------------------------------------------------------------------------- + + +class TestGetRequestRate: + @pytest.mark.asyncio + async def test_returns_tuple_when_set(self): + mgr = RobotsTxtManager(make_fetch_fn(content=ROBOTS_WITH_RATE)) + + rate = await mgr.get_request_rate("https://example.com/", "s1") + + assert rate is not None + assert isinstance(rate, tuple) + assert len(rate) == 2 + + @pytest.mark.asyncio + async def test_tuple_contains_integers(self): + mgr = RobotsTxtManager(make_fetch_fn(content=ROBOTS_WITH_RATE)) + + rate = await mgr.get_request_rate("https://example.com/", "s1") + + assert rate is not None + requests, seconds = rate + assert isinstance(requests, int) + assert isinstance(seconds, int) + + @pytest.mark.asyncio + async def test_returns_none_when_not_set(self): + mgr = RobotsTxtManager(make_fetch_fn(content=ROBOTS_BASIC)) + + assert await mgr.get_request_rate("https://example.com/", "s1") is None + + @pytest.mark.asyncio + async def test_returns_none_for_empty_robots(self): + mgr = RobotsTxtManager(make_fetch_fn(content="")) + + assert await mgr.get_request_rate("https://example.com/", "s1") is None + + @pytest.mark.asyncio + async def test_returns_none_on_fetch_error(self): + async def failing_fetch(url: str, sid: str) -> MockResponse: + raise ConnectionError("network failure") + + mgr = RobotsTxtManager(failing_fetch) + + assert await mgr.get_request_rate("https://example.com/", "s1") is None + + @pytest.mark.asyncio + async def test_returns_none_for_non_200_response(self): + mgr = RobotsTxtManager(make_fetch_fn(status=404)) + + assert await mgr.get_request_rate("https://example.com/", "s1") is None + + +# --------------------------------------------------------------------------- +# Tests: get_sitemaps +# --------------------------------------------------------------------------- + + +# --------------------------------------------------------------------------- +# Tests: caching behaviour +# --------------------------------------------------------------------------- + + +class TestCachingBehaviour: + @pytest.mark.asyncio + async def test_second_call_same_domain_uses_cache(self): + fetch_fn = make_fetch_fn(content=ROBOTS_BASIC) + mgr = RobotsTxtManager(fetch_fn) + + await mgr.can_fetch("https://example.com/page1", "s1") + await mgr.can_fetch("https://example.com/page2", "s1") + + assert len(fetch_fn.calls) == 1 + + @pytest.mark.asyncio + async def test_all_methods_share_cache(self): + fetch_fn = make_fetch_fn(content=ROBOTS_BASIC) + mgr = RobotsTxtManager(fetch_fn) + + await mgr.can_fetch("https://example.com/", "s1") + await mgr.get_crawl_delay("https://example.com/", "s1") + await mgr.get_request_rate("https://example.com/", "s1") + + assert len(fetch_fn.calls) == 1 + + @pytest.mark.asyncio + async def test_different_sids_use_separate_cache_entries(self): + fetch_fn = make_fetch_fn(content=ROBOTS_BASIC) + mgr = RobotsTxtManager(fetch_fn) + + await mgr.can_fetch("https://example.com/", "s1") + await mgr.can_fetch("https://example.com/", "s2") + + assert len(fetch_fn.calls) == 2 + + @pytest.mark.asyncio + async def test_different_domains_use_separate_cache_entries(self): + fetch_fn = make_fetch_fn(content=ROBOTS_BASIC) + mgr = RobotsTxtManager(fetch_fn) + + await mgr.can_fetch("https://example.com/", "s1") + await mgr.can_fetch("https://other.com/", "s1") + + assert len(fetch_fn.calls) == 2 + + @pytest.mark.asyncio + async def test_cache_keyed_by_domain_not_path(self): + fetch_fn = make_fetch_fn(content=ROBOTS_BASIC) + mgr = RobotsTxtManager(fetch_fn) + + await mgr.can_fetch("https://example.com/a/b/c", "s1") + await mgr.can_fetch("https://example.com/x/y/z", "s1") + await mgr.can_fetch("https://example.com/admin/", "s1") + + assert len(fetch_fn.calls) == 1 + + @pytest.mark.asyncio + async def test_sid_is_passed_to_fetch_fn(self): + fetch_fn = make_fetch_fn(content=ROBOTS_BASIC) + mgr = RobotsTxtManager(fetch_fn) + + await mgr.can_fetch("https://example.com/", "my_session") + + _, received_sid = fetch_fn.calls[0] + assert received_sid == "my_session" + + +# --------------------------------------------------------------------------- +# Tests: robots.txt URL construction +# --------------------------------------------------------------------------- + + +class TestRobotsTxtUrlConstruction: + @pytest.mark.asyncio + async def test_http_scheme_preserved(self): + fetch_fn = make_fetch_fn(content="") + mgr = RobotsTxtManager(fetch_fn) + + await mgr.can_fetch("http://example.com/page", "s1") + + fetched_url, _ = fetch_fn.calls[0] + assert fetched_url == "http://example.com/robots.txt" + + @pytest.mark.asyncio + async def test_https_scheme_preserved(self): + fetch_fn = make_fetch_fn(content="") + mgr = RobotsTxtManager(fetch_fn) + + await mgr.can_fetch("https://example.com/page", "s1") + + fetched_url, _ = fetch_fn.calls[0] + assert fetched_url == "https://example.com/robots.txt" + + @pytest.mark.asyncio + async def test_fetched_at_domain_root_regardless_of_request_path(self): + fetch_fn = make_fetch_fn(content="") + mgr = RobotsTxtManager(fetch_fn) + + await mgr.can_fetch("https://example.com/deep/nested/path/page.html", "s1") + + fetched_url, _ = fetch_fn.calls[0] + assert fetched_url == "https://example.com/robots.txt" + + @pytest.mark.asyncio + async def test_port_included_in_url(self): + fetch_fn = make_fetch_fn(content="") + mgr = RobotsTxtManager(fetch_fn) + + await mgr.can_fetch("http://example.com:8080/page", "s1") + + fetched_url, _ = fetch_fn.calls[0] + assert fetched_url == "http://example.com:8080/robots.txt" + + @pytest.mark.asyncio + async def test_different_ports_treated_as_different_domains(self): + fetch_fn = make_fetch_fn(content="") + mgr = RobotsTxtManager(fetch_fn) + + await mgr.can_fetch("http://example.com:8000/page", "s1") + await mgr.can_fetch("http://example.com:9000/page", "s1") + + assert len(fetch_fn.calls) == 2 + urls = [call[0] for call in fetch_fn.calls] + assert "http://example.com:8000/robots.txt" in urls + assert "http://example.com:9000/robots.txt" in urls + + +# --------------------------------------------------------------------------- +# Tests: encoding +# --------------------------------------------------------------------------- + + +class TestEncoding: + @pytest.mark.asyncio + async def test_non_utf8_body_decoded_with_response_encoding(self): + content = "User-agent: *\nDisallow: /admin/\nCrawl-delay: 3" + body = content.encode("latin-1") + + async def fetch_fn(url: str, sid: str) -> MockResponse: + return MockResponse(status=200, body=body, encoding="latin-1") + + mgr = RobotsTxtManager(fetch_fn) + delay = await mgr.get_crawl_delay("https://example.com/", "s1") + + assert delay == 3.0 + + @pytest.mark.asyncio + async def test_bytes_body_decoded_correctly(self): + content = "User-agent: *\nDisallow: /private/" + body = content.encode("utf-8") + + async def fetch_fn(url: str, sid: str) -> MockResponse: + return MockResponse(status=200, body=body, encoding="utf-8") + + mgr = RobotsTxtManager(fetch_fn) + + assert await mgr.can_fetch("https://example.com/private/", "s1") is False + assert await mgr.can_fetch("https://example.com/public/", "s1") is True + + +# --------------------------------------------------------------------------- +# Tests: clear_cache +# --------------------------------------------------------------------------- + + +class TestClearCache: + @pytest.mark.asyncio + async def test_clear_all_forces_refetch(self): + fetch_fn = make_fetch_fn(content=ROBOTS_BASIC) + mgr = RobotsTxtManager(fetch_fn) + + await mgr.can_fetch("https://example.com/", "s1") + mgr.clear_cache() + await mgr.can_fetch("https://example.com/", "s1") + + assert len(fetch_fn.calls) == 2 + + @pytest.mark.asyncio + async def test_clear_by_domain_only_invalidates_that_domain(self): + fetch_fn = make_fetch_fn(content=ROBOTS_BASIC) + mgr = RobotsTxtManager(fetch_fn) + + await mgr.can_fetch("https://example.com/", "s1") + await mgr.can_fetch("https://other.com/", "s1") + assert len(fetch_fn.calls) == 2 + + mgr.clear_cache(domain="example.com") + + await mgr.can_fetch("https://example.com/", "s1") # refetched + await mgr.can_fetch("https://other.com/", "s1") # still cached + + assert len(fetch_fn.calls) == 3 + + @pytest.mark.asyncio + async def test_clear_by_sid_only_invalidates_that_sid(self): + fetch_fn = make_fetch_fn(content=ROBOTS_BASIC) + mgr = RobotsTxtManager(fetch_fn) + + await mgr.can_fetch("https://example.com/", "s1") + await mgr.can_fetch("https://example.com/", "s2") + assert len(fetch_fn.calls) == 2 + + mgr.clear_cache(sid="s1") + + await mgr.can_fetch("https://example.com/", "s1") # refetched + await mgr.can_fetch("https://example.com/", "s2") # still cached + + assert len(fetch_fn.calls) == 3 + + @pytest.mark.asyncio + async def test_clear_by_domain_and_sid_targets_exact_entry(self): + fetch_fn = make_fetch_fn(content=ROBOTS_BASIC) + mgr = RobotsTxtManager(fetch_fn) + + await mgr.can_fetch("https://example.com/", "s1") + await mgr.can_fetch("https://example.com/", "s2") + assert len(fetch_fn.calls) == 2 + + mgr.clear_cache(domain="example.com", sid="s1") + + await mgr.can_fetch("https://example.com/", "s1") # refetched + await mgr.can_fetch("https://example.com/", "s2") # still cached + + assert len(fetch_fn.calls) == 3 + + def test_clear_nonexistent_domain_does_not_raise(self): + mgr = RobotsTxtManager(make_fetch_fn()) + mgr.clear_cache(domain="nevervisited.com") # should not raise + + def test_clear_empty_cache_does_not_raise(self): + mgr = RobotsTxtManager(make_fetch_fn()) + mgr.clear_cache() # should not raise + + @pytest.mark.asyncio + async def test_clear_all_empties_cache_completely(self): + fetch_fn = make_fetch_fn(content=ROBOTS_BASIC) + mgr = RobotsTxtManager(fetch_fn) + + await mgr.can_fetch("https://a.com/", "s1") + await mgr.can_fetch("https://b.com/", "s1") + await mgr.can_fetch("https://c.com/", "s1") + assert len(fetch_fn.calls) == 3 + + mgr.clear_cache() + + await mgr.can_fetch("https://a.com/", "s1") + await mgr.can_fetch("https://b.com/", "s1") + await mgr.can_fetch("https://c.com/", "s1") + + assert len(fetch_fn.calls) == 6 + + +# --------------------------------------------------------------------------- +# Tests: concurrent access (double-checked locking) +# --------------------------------------------------------------------------- + + +class TestConcurrency: + @pytest.mark.asyncio + async def test_concurrent_calls_same_domain_same_sid_deduplicated(self): + """Multiple concurrent tasks for the same domain+sid trigger only one robots.txt fetch.""" + fetch_count = 0 + + async def slow_fetch(url: str, sid: str) -> MockResponse: + nonlocal fetch_count + fetch_count += 1 + await asyncio.sleep(0.02) # simulate network latency + return MockResponse(status=200, body=ROBOTS_BASIC.encode(), encoding="utf-8") + + mgr = RobotsTxtManager(slow_fetch) + + results = await asyncio.gather(*[ + mgr.can_fetch(f"https://example.com/page{i}", "s1") + for i in range(8) + ]) + + # Concurrent calls for the same domain+sid are deduplicated to a single fetch + assert fetch_count == 1 + assert all(isinstance(r, bool) for r in results) + + @pytest.mark.asyncio + async def test_concurrent_calls_different_domains_fetch_independently(self): + fetch_count = 0 + + async def slow_fetch(url: str, sid: str) -> MockResponse: + nonlocal fetch_count + fetch_count += 1 + await asyncio.sleep(0.01) + return MockResponse(status=200, body=b"", encoding="utf-8") + + mgr = RobotsTxtManager(slow_fetch) + + await asyncio.gather( + mgr.can_fetch("https://alpha.com/", "s1"), + mgr.can_fetch("https://beta.com/", "s1"), + mgr.can_fetch("https://gamma.com/", "s1"), + ) + + assert fetch_count == 3 + + @pytest.mark.asyncio + async def test_concurrent_calls_consistent_results(self): + """All concurrent callers should see the same allow/disallow result.""" + mgr = RobotsTxtManager(make_fetch_fn(content=ROBOTS_BASIC)) + + results = await asyncio.gather(*[ + mgr.can_fetch("https://example.com/admin/", "s1") + for _ in range(6) + ]) + + assert all(r is False for r in results) + + @pytest.mark.asyncio + async def test_different_sids_concurrent_fetch_independently(self): + fetch_count = 0 + + async def slow_fetch(url: str, sid: str) -> MockResponse: + nonlocal fetch_count + fetch_count += 1 + await asyncio.sleep(0.01) + return MockResponse(status=200, body=b"", encoding="utf-8") + + mgr = RobotsTxtManager(slow_fetch) + + await asyncio.gather( + mgr.can_fetch("https://example.com/", "s1"), + mgr.can_fetch("https://example.com/", "s2"), + mgr.can_fetch("https://example.com/", "s3"), + ) + + assert fetch_count == 3 From 9dcfe47da77cbe48a633ee91888be56cee0f1b45 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Fri, 3 Apr 2026 17:18:15 +0200 Subject: [PATCH 08/25] build: pump up deps --- pyproject.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index bb69194..a805818 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -63,7 +63,7 @@ classifiers = [ dependencies = [ "lxml>=6.0.2", "cssselect>=1.4.0", - "orjson>=3.11.7", + "orjson>=3.11.8", "tld>=0.13.2", "w3lib>=2.4.1", "typing_extensions", @@ -72,11 +72,11 @@ dependencies = [ [project.optional-dependencies] fetchers = [ "click>=8.3.0", - "curl_cffi>=0.14.0", + "curl_cffi>=0.15.0", "playwright==1.58.0", "patchright==1.58.2", "browserforge>=1.2.4", - "apify-fingerprint-datapoints>=0.11.0", + "apify-fingerprint-datapoints>=0.12.0", "msgspec>=0.20.0", "anyio>=4.12.1" ] From eaf7ec2cc06f07a50ae711c3b3f60e24ca0f1fc6 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Fri, 3 Apr 2026 17:26:52 +0200 Subject: [PATCH 09/25] fix(requests): handle missing type hint returned by curl-cffi --- scrapling/engines/static.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scrapling/engines/static.py b/scrapling/engines/static.py index 1f4b09b..a962ccf 100644 --- a/scrapling/engines/static.py +++ b/scrapling/engines/static.py @@ -250,6 +250,7 @@ class _SyncSessionLogic(_ConfigurationLogic): request_args = self._merge_request_args(stealth=stealth, proxy=proxy, **kwargs) try: response = session.request(method, **request_args) + assert response is not None result = ResponseFactory.from_http_request(response, selector_config, meta={"proxy": proxy}) return result except CurlError as e: # pragma: no cover From 07129ce4b1645f73df25943eb88c215c05e1f14d Mon Sep 17 00:00:00 2001 From: Abdullah <52079299+AbdullahY36@users.noreply.github.com> Date: Fri, 3 Apr 2026 17:51:00 +0200 Subject: [PATCH 10/25] feat(deps): move protego to fetchers optional dependency protego is only used by the spider framework for robots.txt compliance. Moving it from core dependencies to the optional 'fetchers' group reduces the dependency footprint for users who don't need the spider framework. for pyproject.toml file --- pyproject.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 41fb14c..4333d7e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -66,8 +66,7 @@ dependencies = [ "orjson>=3.11.8", "tld>=0.13.2", "w3lib>=2.4.1", - "typing_extensions", - "protego>=0.4.0", + "typing_extensions" ] [project.optional-dependencies] @@ -79,7 +78,8 @@ fetchers = [ "browserforge>=1.2.4", "apify-fingerprint-datapoints>=0.12.0", "msgspec>=0.20.0", - "anyio>=4.12.1" + "anyio>=4.12.1", + "protego>=0.4.0", ] ai = [ "mcp>=1.26.0", From e2b293f41c289a496195b547df89482a23a44739 Mon Sep 17 00:00:00 2001 From: Abdullah <52079299+AbdullahY36@users.noreply.github.com> Date: Sat, 4 Apr 2026 03:00:15 +0200 Subject: [PATCH 11/25] refactor(spiders): simplify robots.txt cache to domain-only key MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit robots.txt is a domain-level document and does not vary by session. Keying the cache by (domain, sid) was both wasteful and incorrect — it caused redundant fetches when the same domain was accessed by different sessions. - Cache is now keyed by domain string only; all sessions share one entry - Removed asyncio.Event inflight-deduplication mechanism (superseded by the prefetch approach added in the next commit) - clear_cache() loses the `sid` parameter (breaking change); clearing a domain now evicts the single shared entry for all sessions - Updated tests to reflect shared-cache semantics Files: scrapling/spiders/robotstxt.py, tests/spiders/test_robotstxt.py --- scrapling/spiders/robotstxt.py | 108 +++++++++++++------------- tests/spiders/test_robotstxt.py | 131 ++++++++++++++++++-------------- 2 files changed, 128 insertions(+), 111 deletions(-) diff --git a/scrapling/spiders/robotstxt.py b/scrapling/spiders/robotstxt.py index 4e8612f..c64c66e 100644 --- a/scrapling/spiders/robotstxt.py +++ b/scrapling/spiders/robotstxt.py @@ -1,6 +1,6 @@ -from asyncio import Event from urllib.parse import urlparse +from anyio import create_task_group from protego import Protego from scrapling.core._types import Dict, Optional, Callable, Awaitable @@ -21,56 +21,40 @@ class RobotsTxtManager: - Allow/Disallow directives (including wildcards and $ anchors) - Crawl-delay directives - Deduplicates concurrent robots.txt fetches for the same domain — if multiple - requests for the same domain arrive before the first fetch completes, they - all wait for that single fetch instead of triggering redundant requests. + robots.txt is a domain-level document and does not vary by session, so the + cache is keyed by domain only. The ``sid`` parameter on public methods + controls which session is used for the initial fetch if the domain is not + yet cached, but all sessions share the same parsed result afterwards. """ def __init__(self, fetch_fn: Callable[[str, str], Awaitable]): self._fetch_fn = fetch_fn - self._cache: Dict[tuple[str, str], Protego] = {} - self._inflight: Dict[tuple[str, str], Event] = {} + self._cache: Dict[str, Protego] = {} async def _get_parser(self, url: str, sid: str) -> Protego: parsed = urlparse(url) domain = parsed.netloc + + if domain in self._cache: + return self._cache[domain] + scheme = parsed.scheme or "https" - cache_key = (domain, sid) - - # Return cached parser if available - if cache_key in self._cache: - return self._cache[cache_key] - - # If a fetch is already in-flight for this domain, wait for it to complete - if cache_key in self._inflight: - await self._inflight[cache_key].wait() - return self._cache[cache_key] - - # Mark fetch as in-flight to deduplicate concurrent requests - event = Event() - self._inflight[cache_key] = event + robots_url = f"{scheme}://{domain}/robots.txt" + content = "" + try: + response = await self._fetch_fn(robots_url, sid) + if response.status == 200: + content = response.body.decode(response.encoding, errors="replace") + except Exception as e: + log.warning(f"Failed to fetch robots.txt for {domain}: {e}") try: - robots_url = f"{scheme}://{domain}/robots.txt" - content = "" - try: - response = await self._fetch_fn(robots_url, sid) - if response.status == 200: - content = response.body.decode(response.encoding, errors="replace") - except Exception as e: - log.warning(f"Failed to fetch robots.txt for {domain}: {e}") - - try: - parser = Protego.parse(content) - except Exception as e: - log.warning(f"Failed to parse robots.txt for {domain}: {e}") - parser = Protego.parse("") - - self._cache[cache_key] = parser - finally: - event.set() - del self._inflight[cache_key] + parser = Protego.parse(content) + except Exception as e: + log.warning(f"Failed to parse robots.txt for {domain}: {e}") + parser = Protego.parse("") + self._cache[domain] = parser return parser async def can_fetch(self, url: str, sid: str) -> bool: @@ -88,7 +72,7 @@ class RobotsTxtManager: Args: url: The full URL to check - sid: Session ID for fetching robots.txt + sid: Session ID for fetching robots.txt if not yet cached Returns: True if the URL can be fetched, False otherwise @@ -104,7 +88,7 @@ class RobotsTxtManager: Args: url: Any URL on the domain to check - sid: Session ID for fetching robots.txt + sid: Session ID for fetching robots.txt if not yet cached Returns: The crawl delay in seconds, or None if not specified @@ -121,7 +105,7 @@ class RobotsTxtManager: Args: url: Any URL on the domain to check - sid: Session ID for fetching robots.txt + sid: Session ID for fetching robots.txt if not yet cached Returns: A tuple of (requests, seconds) if specified, or None if not specified @@ -137,7 +121,7 @@ class RobotsTxtManager: Args: url: Any URL on the domain to check - sid: Session ID for fetching robots.txt + sid: Session ID for fetching robots.txt if not yet cached Returns: A tuple of (crawl_delay, request_rate) where crawl_delay is in seconds @@ -151,19 +135,35 @@ class RobotsTxtManager: (rate.requests, rate.seconds) if rate is not None else None, ) - def clear_cache(self, domain: Optional[str] = None, sid: Optional[str] = None) -> None: - """Clear the robots.txt cache. + async def prefetch(self, urls: list[str], sid: str) -> None: + """Pre-warm the robots.txt cache for a list of seed URLs concurrently. + + Callers are responsible for deduplicating URLs by domain before calling + this method — passing multiple URLs for the same domain will trigger + redundant fetches since no inflight deduplication exists here. Args: - domain: If specified, only clear cache for this domain - sid: If specified, only clear cache for this session ID - If both are None, clears the entire cache + urls: Seed URLs whose domains should be pre-fetched (one per domain). + sid: Session ID to use for the robots.txt fetch requests. """ - if domain is None and sid is None: + if not urls: + return + log.debug(f"Pre-fetching robots.txt for {len(urls)} domain(s)") + async with create_task_group() as tg: + for url in urls: + tg.start_soon(self._get_parser, url, sid) + + def clear_cache(self, domain: Optional[str] = None) -> None: + """Clear the robots.txt cache. + + Note: the ``sid`` parameter was removed — the cache is now keyed by + domain only, so clearing a domain evicts all sessions at once. + + Args: + domain: If specified, only clear cache for this domain. + If None, clears the entire cache. + """ + if domain is None: self._cache.clear() else: - keys_to_remove = [ - key for key in self._cache if (domain is None or key[0] == domain) and (sid is None or key[1] == sid) - ] - for key in keys_to_remove: - del self._cache[key] + self._cache.pop(domain, None) diff --git a/tests/spiders/test_robotstxt.py b/tests/spiders/test_robotstxt.py index 5a447c0..efb8328 100644 --- a/tests/spiders/test_robotstxt.py +++ b/tests/spiders/test_robotstxt.py @@ -5,7 +5,6 @@ import asyncio import pytest from scrapling.spiders.robotstxt import RobotsTxtManager -from scrapling.core._types import List, Optional # --------------------------------------------------------------------------- @@ -28,7 +27,7 @@ def make_fetch_fn(status: int = 200, content: str = "", encoding: str = "utf-8") Attaches a `.calls` list so tests can assert how many times it was invoked and with which arguments. """ - calls: List[tuple] = [] + calls: list[tuple] = [] async def _fetch(url: str, sid: str) -> MockResponse: calls.append((url, sid)) @@ -275,11 +274,6 @@ class TestGetRequestRate: assert await mgr.get_request_rate("https://example.com/", "s1") is None -# --------------------------------------------------------------------------- -# Tests: get_sitemaps -# --------------------------------------------------------------------------- - - # --------------------------------------------------------------------------- # Tests: caching behaviour # --------------------------------------------------------------------------- @@ -308,14 +302,15 @@ class TestCachingBehaviour: assert len(fetch_fn.calls) == 1 @pytest.mark.asyncio - async def test_different_sids_use_separate_cache_entries(self): + async def test_different_sids_share_cache_entry(self): + """robots.txt is domain-level — different sessions share the same cached parser.""" fetch_fn = make_fetch_fn(content=ROBOTS_BASIC) mgr = RobotsTxtManager(fetch_fn) await mgr.can_fetch("https://example.com/", "s1") await mgr.can_fetch("https://example.com/", "s2") - assert len(fetch_fn.calls) == 2 + assert len(fetch_fn.calls) == 1 @pytest.mark.asyncio async def test_different_domains_use_separate_cache_entries(self): @@ -476,37 +471,21 @@ class TestClearCache: assert len(fetch_fn.calls) == 3 @pytest.mark.asyncio - async def test_clear_by_sid_only_invalidates_that_sid(self): + async def test_clear_by_domain_invalidates_all_sessions(self): + """Clearing a domain evicts the single shared cache entry for all sessions.""" fetch_fn = make_fetch_fn(content=ROBOTS_BASIC) mgr = RobotsTxtManager(fetch_fn) await mgr.can_fetch("https://example.com/", "s1") - await mgr.can_fetch("https://example.com/", "s2") + assert len(fetch_fn.calls) == 1 + + mgr.clear_cache(domain="example.com") + + await mgr.can_fetch("https://example.com/", "s1") # refetched — cache was cleared + await mgr.can_fetch("https://example.com/", "s2") # hits the newly warm cache, no fetch + assert len(fetch_fn.calls) == 2 - mgr.clear_cache(sid="s1") - - await mgr.can_fetch("https://example.com/", "s1") # refetched - await mgr.can_fetch("https://example.com/", "s2") # still cached - - assert len(fetch_fn.calls) == 3 - - @pytest.mark.asyncio - async def test_clear_by_domain_and_sid_targets_exact_entry(self): - fetch_fn = make_fetch_fn(content=ROBOTS_BASIC) - mgr = RobotsTxtManager(fetch_fn) - - await mgr.can_fetch("https://example.com/", "s1") - await mgr.can_fetch("https://example.com/", "s2") - assert len(fetch_fn.calls) == 2 - - mgr.clear_cache(domain="example.com", sid="s1") - - await mgr.can_fetch("https://example.com/", "s1") # refetched - await mgr.can_fetch("https://example.com/", "s2") # still cached - - assert len(fetch_fn.calls) == 3 - def test_clear_nonexistent_domain_does_not_raise(self): mgr = RobotsTxtManager(make_fetch_fn()) mgr.clear_cache(domain="nevervisited.com") # should not raise @@ -535,32 +514,30 @@ class TestClearCache: # --------------------------------------------------------------------------- -# Tests: concurrent access (double-checked locking) +# Tests: concurrent access # --------------------------------------------------------------------------- -class TestConcurrency: +class TestCacheAndConcurrency: @pytest.mark.asyncio - async def test_concurrent_calls_same_domain_same_sid_deduplicated(self): - """Multiple concurrent tasks for the same domain+sid trigger only one robots.txt fetch.""" + async def test_cached_domain_not_refetched(self): + """Once a domain is cached, subsequent calls return the cached parser without fetching.""" fetch_count = 0 - async def slow_fetch(url: str, sid: str) -> MockResponse: + async def counting_fetch(url: str, sid: str) -> MockResponse: nonlocal fetch_count fetch_count += 1 - await asyncio.sleep(0.02) # simulate network latency return MockResponse(status=200, body=ROBOTS_BASIC.encode(), encoding="utf-8") - mgr = RobotsTxtManager(slow_fetch) + mgr = RobotsTxtManager(counting_fetch) - results = await asyncio.gather(*[ - mgr.can_fetch(f"https://example.com/page{i}", "s1") - for i in range(8) - ]) + # First call fetches and caches + await mgr.can_fetch("https://example.com/page1", "s1") + # Subsequent calls hit the cache + for i in range(7): + await mgr.can_fetch(f"https://example.com/page{i + 2}", "s1") - # Concurrent calls for the same domain+sid are deduplicated to a single fetch assert fetch_count == 1 - assert all(isinstance(r, bool) for r in results) @pytest.mark.asyncio async def test_concurrent_calls_different_domains_fetch_independently(self): @@ -595,21 +572,61 @@ class TestConcurrency: assert all(r is False for r in results) @pytest.mark.asyncio - async def test_different_sids_concurrent_fetch_independently(self): + async def test_different_sids_share_cache_after_first_fetch(self): + """After the first fetch, all sessions share the cached parser regardless of sid.""" fetch_count = 0 - async def slow_fetch(url: str, sid: str) -> MockResponse: + async def counting_fetch(url: str, sid: str) -> MockResponse: nonlocal fetch_count fetch_count += 1 - await asyncio.sleep(0.01) return MockResponse(status=200, body=b"", encoding="utf-8") - mgr = RobotsTxtManager(slow_fetch) + mgr = RobotsTxtManager(counting_fetch) - await asyncio.gather( - mgr.can_fetch("https://example.com/", "s1"), - mgr.can_fetch("https://example.com/", "s2"), - mgr.can_fetch("https://example.com/", "s3"), - ) + # First call fetches and caches + await mgr.can_fetch("https://example.com/", "s1") + # s2 and s3 hit the cache — no additional fetches + await mgr.can_fetch("https://example.com/", "s2") + await mgr.can_fetch("https://example.com/", "s3") - assert fetch_count == 3 + assert fetch_count == 1 + + +# --------------------------------------------------------------------------- +# Tests: prefetch +# --------------------------------------------------------------------------- + + +class TestPrefetch: + @pytest.mark.asyncio + async def test_prefetch_fetches_all_domains(self): + fetch_fn = make_fetch_fn(content=ROBOTS_BASIC) + mgr = RobotsTxtManager(fetch_fn) + + await mgr.prefetch(["https://a.com/", "https://b.com/", "https://c.com/"], "s1") + + assert len(fetch_fn.calls) == 3 + fetched = {url for url, _ in fetch_fn.calls} + assert fetched == {"https://a.com/robots.txt", "https://b.com/robots.txt", "https://c.com/robots.txt"} + + @pytest.mark.asyncio + async def test_prefetch_warms_cache_for_subsequent_calls(self): + fetch_fn = make_fetch_fn(content=ROBOTS_BASIC) + mgr = RobotsTxtManager(fetch_fn) + + await mgr.prefetch(["https://example.com/"], "s1") + assert len(fetch_fn.calls) == 1 + + # Any subsequent call for the same domain hits the cache + await mgr.can_fetch("https://example.com/products", "s1") + await mgr.can_fetch("https://example.com/products", "s2") + assert len(fetch_fn.calls) == 1 + + @pytest.mark.asyncio + async def test_prefetch_empty_list_is_noop(self): + fetch_fn = make_fetch_fn(content=ROBOTS_BASIC) + mgr = RobotsTxtManager(fetch_fn) + + await mgr.prefetch([], "s1") + + assert len(fetch_fn.calls) == 0 From a86e9709ea0442fcf7b1c9b0459ac0716514d8a6 Mon Sep 17 00:00:00 2001 From: Abdullah <52079299+AbdullahY36@users.noreply.github.com> Date: Sat, 4 Apr 2026 03:00:15 +0200 Subject: [PATCH 12/25] feat(spiders): pre-warm robots.txt cache before crawl loop starts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previously robots.txt was fetched lazily on the first request per domain, causing early concurrent requests to each stall waiting for the same network fetch. The cache is now warmed before the crawl loop starts, making all subsequent robots.txt lookups a local read. - RobotsTxtManager gains a prefetch(urls, sid) method that fetches all domains concurrently via a task group - CrawlerEngine._prefetch_robots_txt() is called after on_start(): uses allowed_domains if configured, otherwise falls back to unique domains extracted from start_urls - Mid-crawl domain discovery (not covered by prefetch) still fetches lazily; two concurrent callbacks on the same new domain can each trigger a fetch — accepted tradeoff, documented in _get_domain_delay Files: scrapling/spiders/robotstxt.py, scrapling/spiders/engine.py, tests/spiders/test_engine.py --- scrapling/spiders/engine.py | 28 +++++++++++- tests/spiders/test_engine.py | 86 +++++++++++++++++++++++++++++++++++- 2 files changed, 112 insertions(+), 2 deletions(-) diff --git a/scrapling/spiders/engine.py b/scrapling/spiders/engine.py index cf1715e..163101b 100644 --- a/scrapling/spiders/engine.py +++ b/scrapling/spiders/engine.py @@ -97,7 +97,10 @@ class CrawlerEngine: if domain in self._domain_delays: return self._domain_delays[domain] - # Fetch both robots.txt directives in a single parser lookup + # For domains covered by _prefetch_robots_txt this is a local parser read. + # Domains discovered mid-crawl (not in start_urls/allowed_domains) will fetch here. + # Two concurrent callbacks hitting the same new domain can each trigger a fetch; + # the second write is a no-op in effect (same content), but the extra request is accepted. c_delay, r_rate = await robots_manager._get_delay_directives(request.url, request.sid) delay = self.spider.download_delay @@ -287,6 +290,27 @@ class CrawlerEngine: return True + async def _prefetch_robots_txt(self) -> None: + """Pre-warm the robots.txt cache before the crawl loop starts. + + Uses allowed_domains if configured, otherwise falls back to unique domains + extracted from start_urls via Request.domain. Both paths use https. + """ + if not self._robots_manager: + return + + if self._allowed_domains: + domains = self._allowed_domains + elif self.spider.start_urls: + # Deduplicate by domain so we spawn exactly one task per domain + domains = {Request(url).domain for url in self.spider.start_urls} + else: + return + + seed_urls = [f"https://{domain}/" for domain in domains] + + await self._robots_manager.prefetch(seed_urls, self.session_manager.default_session_id) + async def crawl(self) -> CrawlStats: """Run the spider and return CrawlStats.""" self._running = True @@ -310,6 +334,8 @@ class CrawlerEngine: await self.spider.on_start(resuming=resuming) + await self._prefetch_robots_txt() + try: if not resuming: async for request in self.spider.start_requests(): diff --git a/tests/spiders/test_engine.py b/tests/spiders/test_engine.py index e362036..e7382f8 100644 --- a/tests/spiders/test_engine.py +++ b/tests/spiders/test_engine.py @@ -8,6 +8,7 @@ import pytest from scrapling.spiders.engine import CrawlerEngine, _dump from scrapling.spiders.request import Request +from scrapling.spiders.robotstxt import RobotsTxtManager from scrapling.spiders.session import SessionManager from scrapling.spiders.result import CrawlStats, ItemList from scrapling.spiders.checkpoint import CheckpointData @@ -22,10 +23,11 @@ from scrapling.core._types import Any, Dict, Set, AsyncGenerator class MockResponse: """Minimal Response stand-in.""" - def __init__(self, status: int = 200, body: bytes = b"ok", url: str = "https://example.com"): + def __init__(self, status: int = 200, body: bytes = b"ok", url: str = "https://example.com", encoding: str = "utf-8"): self.status = status self.body = body self.url = url + self.encoding = encoding self.request: Any = None self.meta: Dict[str, Any] = {} @@ -84,6 +86,7 @@ class MockSpider: on_scraped_item_fn=None, retry_blocked_request_fn=None, robots_txt_obey: bool = False, + start_urls: list[str] | None = None, ): self.concurrent_requests = concurrent_requests self.concurrent_requests_per_domain = concurrent_requests_per_domain @@ -95,6 +98,7 @@ class MockSpider: self.fp_keep_fragments = fp_keep_fragments self.name = "test_spider" self.robots_txt_obey = robots_txt_obey + self.start_urls = start_urls or [] # Tracking lists self.on_start_calls: list[dict] = [] @@ -914,3 +918,83 @@ class TestPauseDuringCrawl: await engine.crawl() assert engine.paused is False + + +# --------------------------------------------------------------------------- +# Tests: _prefetch_robots_txt +# --------------------------------------------------------------------------- + + +class TestPrefetchRobotsTxt: + """_prefetch_robots_txt warms the robots.txt cache before the crawl loop.""" + + @staticmethod + def _make_counting_fetch(): + """Return (fetch_fn, calls_list) where calls_list records every (url, sid) pair.""" + calls: list[tuple[str, str]] = [] + + async def _fetch(url: str, sid: str): + calls.append((url, sid)) + return MockResponse(status=200, body=b"", url=url) + + return _fetch, calls + + @pytest.mark.asyncio + async def test_prefetch_uses_allowed_domains_when_set(self): + fetch_fn, calls = self._make_counting_fetch() + spider = MockSpider(allowed_domains={"a.com", "b.com"}, robots_txt_obey=True) + engine = _make_engine(spider=spider) + engine._robots_manager = RobotsTxtManager(fetch_fn) + + await engine._prefetch_robots_txt() + + fetched_domains = {Request(url).domain for url, _ in calls} + assert fetched_domains == {"a.com", "b.com"} + + @pytest.mark.asyncio + async def test_prefetch_falls_back_to_start_urls_when_no_allowed_domains(self): + fetch_fn, calls = self._make_counting_fetch() + spider = MockSpider(robots_txt_obey=True, start_urls=["https://example.com/page1"]) + engine = _make_engine(spider=spider) + engine._robots_manager = RobotsTxtManager(fetch_fn) + + await engine._prefetch_robots_txt() + + assert len(calls) == 1 + assert calls[0][0] == "https://example.com/robots.txt" + + @pytest.mark.asyncio + async def test_prefetch_noop_when_robots_disabled(self): + fetch_fn, calls = self._make_counting_fetch() + spider = MockSpider(robots_txt_obey=False) + engine = _make_engine(spider=spider) + + assert engine._robots_manager is None + + await engine._prefetch_robots_txt() + + assert calls == [] + + @pytest.mark.asyncio + async def test_prefetch_noop_when_start_urls_empty(self): + fetch_fn, calls = self._make_counting_fetch() + spider = MockSpider(robots_txt_obey=True, start_urls=[]) + engine = _make_engine(spider=spider) + engine._robots_manager = RobotsTxtManager(fetch_fn) + + await engine._prefetch_robots_txt() + + assert calls == [] + + @pytest.mark.asyncio + async def test_prefetch_deduplicates_same_domain_in_start_urls(self): + fetch_fn, calls = self._make_counting_fetch() + spider = MockSpider(robots_txt_obey=True, start_urls=["https://example.com/a", "https://example.com/b"]) + engine = _make_engine(spider=spider) + engine._robots_manager = RobotsTxtManager(fetch_fn) + + await engine._prefetch_robots_txt() + + # set of Request.domain values deduplicates to one task per domain + assert len(calls) == 1 + assert calls[0][0] == "https://example.com/robots.txt" From a134fdb8cce853f66713e115b81d195ec6e7b0c3 Mon Sep 17 00:00:00 2001 From: Abdullah <52079299+AbdullahY36@users.noreply.github.com> Date: Sat, 4 Apr 2026 03:10:17 +0200 Subject: [PATCH 13/25] feat(spiders): enable robots.txt compliance by default robots_txt_obey now defaults to True. Spiders must explicitly opt out with robots_txt_obey = False rather than opt in, making ethical crawling the default behaviour. File: scrapling/spiders/spider.py --- scrapling/spiders/spider.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapling/spiders/spider.py b/scrapling/spiders/spider.py index 6aaa24f..52afcbd 100644 --- a/scrapling/spiders/spider.py +++ b/scrapling/spiders/spider.py @@ -73,7 +73,7 @@ class Spider(ABC): allowed_domains: Set[str] = set() # Robots.txt compliance - robots_txt_obey: bool = False + robots_txt_obey: bool = True # Concurrency settings concurrent_requests: int = 4 From 854daac7941a66e01b96cbac54eae44a29b30063 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Sat, 4 Apr 2026 21:06:35 +0200 Subject: [PATCH 14/25] style(spiders robots feat): Adjustments for maintainability --- scrapling/spiders/engine.py | 7 ++++--- scrapling/spiders/robotstxt.py | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/scrapling/spiders/engine.py b/scrapling/spiders/engine.py index 163101b..f395a70 100644 --- a/scrapling/spiders/engine.py +++ b/scrapling/spiders/engine.py @@ -7,9 +7,9 @@ from anyio import Path as AsyncPath from anyio import create_task_group, CapacityLimiter, create_memory_object_stream, EndOfStream from scrapling.core.utils import log -from scrapling.spiders.request import Request from scrapling.spiders.scheduler import Scheduler from scrapling.spiders.session import SessionManager +from scrapling.spiders.request import Request, Response from scrapling.spiders.robotstxt import RobotsTxtManager from scrapling.spiders.result import CrawlStats, ItemList from scrapling.spiders.checkpoint import CheckpointManager, CheckpointData @@ -44,7 +44,7 @@ class CrawlerEngine: if self.spider.robots_txt_obey: - async def _fetch_robots(url: str, sid: str): + async def _fetch_robots(url: str, sid: str) -> Response: return await self.session_manager.fetch(Request(url, sid=sid)) self._robots_manager: Optional[RobotsTxtManager] = RobotsTxtManager(_fetch_robots) @@ -101,7 +101,7 @@ class CrawlerEngine: # Domains discovered mid-crawl (not in start_urls/allowed_domains) will fetch here. # Two concurrent callbacks hitting the same new domain can each trigger a fetch; # the second write is a no-op in effect (same content), but the extra request is accepted. - c_delay, r_rate = await robots_manager._get_delay_directives(request.url, request.sid) + c_delay, r_rate = await robots_manager.get_delay_directives(request.url, request.sid) delay = self.spider.download_delay robots_enforced_delay = False @@ -133,6 +133,7 @@ class CrawlerEngine: """Get or create a per-domain concurrency limiter if enabled, otherwise use the global limiter.""" if self.spider.concurrent_requests_per_domain: self._domain_limiters.setdefault(domain, CapacityLimiter(self.spider.concurrent_requests_per_domain)) + # robots.txt-created limiters always apply even with `concurrent_requests_per_domain = 0` (if enabled) return self._domain_limiters.get(domain, self._global_limiter) def _normalize_request(self, request: Request) -> None: diff --git a/scrapling/spiders/robotstxt.py b/scrapling/spiders/robotstxt.py index c64c66e..867698b 100644 --- a/scrapling/spiders/robotstxt.py +++ b/scrapling/spiders/robotstxt.py @@ -116,7 +116,7 @@ class RobotsTxtManager: return (rate.requests, rate.seconds) return None - async def _get_delay_directives(self, url: str, sid: str) -> tuple[Optional[float], Optional[tuple[int, int]]]: + async def get_delay_directives(self, url: str, sid: str) -> tuple[Optional[float], Optional[tuple[int, int]]]: """Return both crawl-delay and request-rate in a single parser lookup. Args: From af83a11aa7873dc8342edf1279a2cb00aaf677a9 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Sun, 5 Apr 2026 00:09:17 +0200 Subject: [PATCH 15/25] fix(spider robots): solve multiple issues with cache prefetch --- scrapling/spiders/engine.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/scrapling/spiders/engine.py b/scrapling/spiders/engine.py index f395a70..3b41fb5 100644 --- a/scrapling/spiders/engine.py +++ b/scrapling/spiders/engine.py @@ -1,6 +1,7 @@ import json import pprint from pathlib import Path +from urllib.parse import urlparse import anyio from anyio import Path as AsyncPath @@ -294,22 +295,28 @@ class CrawlerEngine: async def _prefetch_robots_txt(self) -> None: """Pre-warm the robots.txt cache before the crawl loop starts. - Uses allowed_domains if configured, otherwise falls back to unique domains - extracted from start_urls via Request.domain. Both paths use https. + Uses allowed_domains if configured (defaults to https since bare domains + have no scheme), otherwise falls back to unique domains extracted from + start_urls preserving the original scheme. """ if not self._robots_manager: return if self._allowed_domains: - domains = self._allowed_domains + # allowed_domains are bare strings like "example.com", no scheme available + seed_urls = [f"http://{domain}/" for domain in self._allowed_domains] elif self.spider.start_urls: - # Deduplicate by domain so we spawn exactly one task per domain - domains = {Request(url).domain for url in self.spider.start_urls} + # Deduplicate by netloc, preserving the scheme from the first URL per domain + seen: set[str] = set() + seed_urls = [] + for url in self.spider.start_urls: + parsed = urlparse(url) + if parsed.netloc not in seen: + seen.add(parsed.netloc) + seed_urls.append(f"{parsed.scheme}://{parsed.netloc}/") else: return - seed_urls = [f"https://{domain}/" for domain in domains] - await self._robots_manager.prefetch(seed_urls, self.session_manager.default_session_id) async def crawl(self) -> CrawlStats: From ec487d37e87f9a17d8df917cafd02e8fe26a3229 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Sun, 5 Apr 2026 01:39:05 +0200 Subject: [PATCH 16/25] fix(spider): Make delay in robots file don't affect user's concurrency settings --- scrapling/spiders/engine.py | 24 ++---------------------- 1 file changed, 2 insertions(+), 22 deletions(-) diff --git a/scrapling/spiders/engine.py b/scrapling/spiders/engine.py index 3b41fb5..3d75564 100644 --- a/scrapling/spiders/engine.py +++ b/scrapling/spiders/engine.py @@ -85,8 +85,6 @@ class CrawlerEngine: Takes the max of the spider's configured delay and any robots.txt directives (Crawl-delay / Request-rate). Result is cached per domain. - Also pre-creates a per-domain concurrency limiter of 1 when robots.txt - enforces any delay, before the caller acquires it via _rate_limiter(). """ robots_manager = self._robots_manager if robots_manager is None: @@ -94,48 +92,32 @@ class CrawlerEngine: domain = request.domain - # Return cached delay if available if domain in self._domain_delays: return self._domain_delays[domain] # For domains covered by _prefetch_robots_txt this is a local parser read. # Domains discovered mid-crawl (not in start_urls/allowed_domains) will fetch here. - # Two concurrent callbacks hitting the same new domain can each trigger a fetch; - # the second write is a no-op in effect (same content), but the extra request is accepted. c_delay, r_rate = await robots_manager.get_delay_directives(request.url, request.sid) delay = self.spider.download_delay - robots_enforced_delay = False if r_rate: req_count, period = r_rate if req_count > 0: delay = max(delay, period / req_count) - robots_enforced_delay = True if c_delay is not None: delay = max(delay, c_delay) - robots_enforced_delay = True self._domain_delays[domain] = delay - - # Enforce 1 concurrent request for this domain when robots.txt adds a delay - if robots_enforced_delay and delay > 0 and domain not in self._domain_limiters: - if self.spider.concurrent_requests_per_domain: - log.warning( - f"robots.txt for {domain} enforces a delay, overriding" - f" concurrent_requests_per_domain={self.spider.concurrent_requests_per_domain} with 1" - ) - self._domain_limiters[domain] = CapacityLimiter(1) - return delay def _rate_limiter(self, domain: str) -> CapacityLimiter: """Get or create a per-domain concurrency limiter if enabled, otherwise use the global limiter.""" if self.spider.concurrent_requests_per_domain: self._domain_limiters.setdefault(domain, CapacityLimiter(self.spider.concurrent_requests_per_domain)) - # robots.txt-created limiters always apply even with `concurrent_requests_per_domain = 0` (if enabled) - return self._domain_limiters.get(domain, self._global_limiter) + return self._domain_limiters[domain] + return self._global_limiter def _normalize_request(self, request: Request) -> None: """Normalize request fields before enqueueing. @@ -154,8 +136,6 @@ class CrawlerEngine: self.stats.robots_disallowed_count += 1 log.debug(f"Request disallowed by robots.txt: {request.url}") return - # Must be called before _rate_limiter: may create CapacityLimiter(1) in _domain_limiters - # when robots.txt enforces a delay, which _rate_limiter then picks up. delay = await self._get_domain_delay(request) else: delay = self.spider.download_delay From 556a90f645f4aaecbc95de899f6cc6d2807e184a Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Sun, 5 Apr 2026 01:43:06 +0200 Subject: [PATCH 17/25] refactor(spider): prefetch robots.txt from start_urls only Stop using allowed_domains for robots.txt prefetch since bare domain strings have no scheme info. Domains discovered mid-crawl via requests still fetch robots.txt lazily. --- scrapling/spiders/engine.py | 28 ++++++++++------------------ 1 file changed, 10 insertions(+), 18 deletions(-) diff --git a/scrapling/spiders/engine.py b/scrapling/spiders/engine.py index 3d75564..7729d62 100644 --- a/scrapling/spiders/engine.py +++ b/scrapling/spiders/engine.py @@ -275,27 +275,19 @@ class CrawlerEngine: async def _prefetch_robots_txt(self) -> None: """Pre-warm the robots.txt cache before the crawl loop starts. - Uses allowed_domains if configured (defaults to https since bare domains - have no scheme), otherwise falls back to unique domains extracted from - start_urls preserving the original scheme. + Extracts unique domains from start_urls, preserving the original scheme. """ - if not self._robots_manager: + if not self._robots_manager or not self.spider.start_urls: return - if self._allowed_domains: - # allowed_domains are bare strings like "example.com", no scheme available - seed_urls = [f"http://{domain}/" for domain in self._allowed_domains] - elif self.spider.start_urls: - # Deduplicate by netloc, preserving the scheme from the first URL per domain - seen: set[str] = set() - seed_urls = [] - for url in self.spider.start_urls: - parsed = urlparse(url) - if parsed.netloc not in seen: - seen.add(parsed.netloc) - seed_urls.append(f"{parsed.scheme}://{parsed.netloc}/") - else: - return + # Deduplicate by netloc, preserving the scheme from the first URL per domain + seen: set[str] = set() + seed_urls: list[str] = [] + for url in self.spider.start_urls: + parsed = urlparse(url) + if parsed.netloc not in seen: + seen.add(parsed.netloc) + seed_urls.append(f"{parsed.scheme}://{parsed.netloc}/") await self._robots_manager.prefetch(seed_urls, self.session_manager.default_session_id) From ea2dd7866b9cfe052279025de790892b15275096 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Sun, 5 Apr 2026 01:55:17 +0200 Subject: [PATCH 18/25] refactor(spiders): Make Robots.txt compliance turned off by default Scrapy is turning it off by default --- scrapling/spiders/spider.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapling/spiders/spider.py b/scrapling/spiders/spider.py index 52afcbd..6aaa24f 100644 --- a/scrapling/spiders/spider.py +++ b/scrapling/spiders/spider.py @@ -73,7 +73,7 @@ class Spider(ABC): allowed_domains: Set[str] = set() # Robots.txt compliance - robots_txt_obey: bool = True + robots_txt_obey: bool = False # Concurrency settings concurrent_requests: int = 4 From afaf68e7d59e7735e3c6e711f6460dab722a5161 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Sun, 5 Apr 2026 02:32:33 +0200 Subject: [PATCH 19/25] fix(spider robots): removing dead code --- scrapling/spiders/engine.py | 2 +- scrapling/spiders/robotstxt.py | 52 -------- tests/spiders/test_engine.py | 14 +-- tests/spiders/test_robotstxt.py | 208 +++++++------------------------- 4 files changed, 43 insertions(+), 233 deletions(-) diff --git a/scrapling/spiders/engine.py b/scrapling/spiders/engine.py index 7729d62..1924298 100644 --- a/scrapling/spiders/engine.py +++ b/scrapling/spiders/engine.py @@ -96,7 +96,7 @@ class CrawlerEngine: return self._domain_delays[domain] # For domains covered by _prefetch_robots_txt this is a local parser read. - # Domains discovered mid-crawl (not in start_urls/allowed_domains) will fetch here. + # Domains discovered mid-crawl (not in start_urls) will fetch here. c_delay, r_rate = await robots_manager.get_delay_directives(request.url, request.sid) delay = self.spider.download_delay diff --git a/scrapling/spiders/robotstxt.py b/scrapling/spiders/robotstxt.py index 867698b..10dc7ab 100644 --- a/scrapling/spiders/robotstxt.py +++ b/scrapling/spiders/robotstxt.py @@ -61,7 +61,6 @@ class RobotsTxtManager: """Check if a URL can be fetched according to the domain's robots.txt. Handles: - - User-agent specific rules (e.g., User-agent: SpinarakBot) - Wildcard user-agent rules (User-agent: *) - Allow/Disallow directives with wildcards (e.g., /*.pdf$) - Allow directives that override Disallow (e.g., Allow: /admin/public-docs/) @@ -80,42 +79,6 @@ class RobotsTxtManager: parser = await self._get_parser(url, sid) return parser.can_fetch(url, "*") - async def get_crawl_delay(self, url: str, sid: str) -> Optional[float]: - """Get the crawl delay for this crawler. - - Uses the wildcard user-agent (*) to get the general crawl delay - that applies to all bots. - - Args: - url: Any URL on the domain to check - sid: Session ID for fetching robots.txt if not yet cached - - Returns: - The crawl delay in seconds, or None if not specified - """ - parser = await self._get_parser(url, sid) - delay = parser.crawl_delay("*") - return float(delay) if delay is not None else None - - async def get_request_rate(self, url: str, sid: str) -> Optional[tuple[int, int]]: - """Get the request rate for this crawler. - - Uses the wildcard user-agent (*) to get the general request rate - that applies to all bots. - - Args: - url: Any URL on the domain to check - sid: Session ID for fetching robots.txt if not yet cached - - Returns: - A tuple of (requests, seconds) if specified, or None if not specified - """ - parser = await self._get_parser(url, sid) - rate = parser.request_rate("*") - if rate is not None: - return (rate.requests, rate.seconds) - return None - async def get_delay_directives(self, url: str, sid: str) -> tuple[Optional[float], Optional[tuple[int, int]]]: """Return both crawl-delay and request-rate in a single parser lookup. @@ -152,18 +115,3 @@ class RobotsTxtManager: async with create_task_group() as tg: for url in urls: tg.start_soon(self._get_parser, url, sid) - - def clear_cache(self, domain: Optional[str] = None) -> None: - """Clear the robots.txt cache. - - Note: the ``sid`` parameter was removed — the cache is now keyed by - domain only, so clearing a domain evicts all sessions at once. - - Args: - domain: If specified, only clear cache for this domain. - If None, clears the entire cache. - """ - if domain is None: - self._cache.clear() - else: - self._cache.pop(domain, None) diff --git a/tests/spiders/test_engine.py b/tests/spiders/test_engine.py index e7382f8..74cf875 100644 --- a/tests/spiders/test_engine.py +++ b/tests/spiders/test_engine.py @@ -940,19 +940,7 @@ class TestPrefetchRobotsTxt: return _fetch, calls @pytest.mark.asyncio - async def test_prefetch_uses_allowed_domains_when_set(self): - fetch_fn, calls = self._make_counting_fetch() - spider = MockSpider(allowed_domains={"a.com", "b.com"}, robots_txt_obey=True) - engine = _make_engine(spider=spider) - engine._robots_manager = RobotsTxtManager(fetch_fn) - - await engine._prefetch_robots_txt() - - fetched_domains = {Request(url).domain for url, _ in calls} - assert fetched_domains == {"a.com", "b.com"} - - @pytest.mark.asyncio - async def test_prefetch_falls_back_to_start_urls_when_no_allowed_domains(self): + async def test_prefetch_uses_start_urls(self): fetch_fn, calls = self._make_counting_fetch() spider = MockSpider(robots_txt_obey=True, start_urls=["https://example.com/page1"]) engine = _make_engine(spider=spider) diff --git a/tests/spiders/test_robotstxt.py b/tests/spiders/test_robotstxt.py index efb8328..c4050c9 100644 --- a/tests/spiders/test_robotstxt.py +++ b/tests/spiders/test_robotstxt.py @@ -53,14 +53,6 @@ Request-rate: 1/10 Disallow: /private/ """ -ROBOTS_WITH_SITEMAP = """\ -User-agent: * -Disallow: - -Sitemap: https://example.com/sitemap.xml -Sitemap: https://example.com/sitemap2.xml -""" - ROBOTS_ALLOW_OVERRIDE = """\ User-agent: * Disallow: /secret/ @@ -157,121 +149,79 @@ class TestCanFetch: # --------------------------------------------------------------------------- -# Tests: get_crawl_delay +# Tests: get_delay_directives # --------------------------------------------------------------------------- -class TestGetCrawlDelay: +class TestGetDelayDirectives: @pytest.mark.asyncio - async def test_returns_float_when_set(self): + async def test_returns_crawl_delay_when_set(self): mgr = RobotsTxtManager(make_fetch_fn(content=ROBOTS_BASIC)) - delay = await mgr.get_crawl_delay("https://example.com/", "s1") + c_delay, r_rate = await mgr.get_delay_directives("https://example.com/", "s1") - assert delay == 2.0 - assert isinstance(delay, float) + assert c_delay == 2.0 + assert isinstance(c_delay, float) + assert r_rate is None @pytest.mark.asyncio - async def test_returns_none_when_not_set(self): + async def test_returns_request_rate_when_set(self): + mgr = RobotsTxtManager(make_fetch_fn(content=ROBOTS_WITH_RATE)) + + c_delay, r_rate = await mgr.get_delay_directives("https://example.com/", "s1") + + assert c_delay is None + assert r_rate is not None + assert r_rate == (1, 1) + + @pytest.mark.asyncio + async def test_returns_both_none_when_not_set(self): content = "User-agent: *\nDisallow: /admin/" mgr = RobotsTxtManager(make_fetch_fn(content=content)) - assert await mgr.get_crawl_delay("https://example.com/", "s1") is None + c_delay, r_rate = await mgr.get_delay_directives("https://example.com/", "s1") + + assert c_delay is None + assert r_rate is None @pytest.mark.asyncio - async def test_returns_none_for_empty_robots(self): + async def test_returns_both_none_for_empty_robots(self): mgr = RobotsTxtManager(make_fetch_fn(content="")) - assert await mgr.get_crawl_delay("https://example.com/", "s1") is None + c_delay, r_rate = await mgr.get_delay_directives("https://example.com/", "s1") + + assert c_delay is None + assert r_rate is None @pytest.mark.asyncio - async def test_returns_none_on_fetch_error(self): + async def test_returns_both_none_on_fetch_error(self): async def failing_fetch(url: str, sid: str) -> MockResponse: raise ConnectionError("network failure") mgr = RobotsTxtManager(failing_fetch) - assert await mgr.get_crawl_delay("https://example.com/", "s1") is None + c_delay, r_rate = await mgr.get_delay_directives("https://example.com/", "s1") + + assert c_delay is None + assert r_rate is None @pytest.mark.asyncio - async def test_returns_none_for_non_200_response(self): - mgr = RobotsTxtManager(make_fetch_fn(status=404)) - - assert await mgr.get_crawl_delay("https://example.com/", "s1") is None - - @pytest.mark.asyncio - async def test_fractional_delay(self): + async def test_fractional_crawl_delay(self): content = "User-agent: *\nCrawl-delay: 0.5" mgr = RobotsTxtManager(make_fetch_fn(content=content)) - delay = await mgr.get_crawl_delay("https://example.com/", "s1") + c_delay, _ = await mgr.get_delay_directives("https://example.com/", "s1") - assert delay == 0.5 + assert c_delay == 0.5 @pytest.mark.asyncio async def test_url_path_does_not_affect_result(self): - """Any URL on the same domain should return the same delay.""" mgr = RobotsTxtManager(make_fetch_fn(content=ROBOTS_BASIC)) - d1 = await mgr.get_crawl_delay("https://example.com/", "s1") - d2 = await mgr.get_crawl_delay("https://example.com/deep/path/page.html", "s1") + r1 = await mgr.get_delay_directives("https://example.com/", "s1") + r2 = await mgr.get_delay_directives("https://example.com/deep/path/page.html", "s1") - assert d1 == d2 - - -# --------------------------------------------------------------------------- -# Tests: get_request_rate -# --------------------------------------------------------------------------- - - -class TestGetRequestRate: - @pytest.mark.asyncio - async def test_returns_tuple_when_set(self): - mgr = RobotsTxtManager(make_fetch_fn(content=ROBOTS_WITH_RATE)) - - rate = await mgr.get_request_rate("https://example.com/", "s1") - - assert rate is not None - assert isinstance(rate, tuple) - assert len(rate) == 2 - - @pytest.mark.asyncio - async def test_tuple_contains_integers(self): - mgr = RobotsTxtManager(make_fetch_fn(content=ROBOTS_WITH_RATE)) - - rate = await mgr.get_request_rate("https://example.com/", "s1") - - assert rate is not None - requests, seconds = rate - assert isinstance(requests, int) - assert isinstance(seconds, int) - - @pytest.mark.asyncio - async def test_returns_none_when_not_set(self): - mgr = RobotsTxtManager(make_fetch_fn(content=ROBOTS_BASIC)) - - assert await mgr.get_request_rate("https://example.com/", "s1") is None - - @pytest.mark.asyncio - async def test_returns_none_for_empty_robots(self): - mgr = RobotsTxtManager(make_fetch_fn(content="")) - - assert await mgr.get_request_rate("https://example.com/", "s1") is None - - @pytest.mark.asyncio - async def test_returns_none_on_fetch_error(self): - async def failing_fetch(url: str, sid: str) -> MockResponse: - raise ConnectionError("network failure") - - mgr = RobotsTxtManager(failing_fetch) - - assert await mgr.get_request_rate("https://example.com/", "s1") is None - - @pytest.mark.asyncio - async def test_returns_none_for_non_200_response(self): - mgr = RobotsTxtManager(make_fetch_fn(status=404)) - - assert await mgr.get_request_rate("https://example.com/", "s1") is None + assert r1 == r2 # --------------------------------------------------------------------------- @@ -296,8 +246,7 @@ class TestCachingBehaviour: mgr = RobotsTxtManager(fetch_fn) await mgr.can_fetch("https://example.com/", "s1") - await mgr.get_crawl_delay("https://example.com/", "s1") - await mgr.get_request_rate("https://example.com/", "s1") + await mgr.get_delay_directives("https://example.com/", "s1") assert len(fetch_fn.calls) == 1 @@ -419,9 +368,9 @@ class TestEncoding: return MockResponse(status=200, body=body, encoding="latin-1") mgr = RobotsTxtManager(fetch_fn) - delay = await mgr.get_crawl_delay("https://example.com/", "s1") + c_delay, _ = await mgr.get_delay_directives("https://example.com/", "s1") - assert delay == 3.0 + assert c_delay == 3.0 @pytest.mark.asyncio async def test_bytes_body_decoded_correctly(self): @@ -437,81 +386,6 @@ class TestEncoding: assert await mgr.can_fetch("https://example.com/public/", "s1") is True -# --------------------------------------------------------------------------- -# Tests: clear_cache -# --------------------------------------------------------------------------- - - -class TestClearCache: - @pytest.mark.asyncio - async def test_clear_all_forces_refetch(self): - fetch_fn = make_fetch_fn(content=ROBOTS_BASIC) - mgr = RobotsTxtManager(fetch_fn) - - await mgr.can_fetch("https://example.com/", "s1") - mgr.clear_cache() - await mgr.can_fetch("https://example.com/", "s1") - - assert len(fetch_fn.calls) == 2 - - @pytest.mark.asyncio - async def test_clear_by_domain_only_invalidates_that_domain(self): - fetch_fn = make_fetch_fn(content=ROBOTS_BASIC) - mgr = RobotsTxtManager(fetch_fn) - - await mgr.can_fetch("https://example.com/", "s1") - await mgr.can_fetch("https://other.com/", "s1") - assert len(fetch_fn.calls) == 2 - - mgr.clear_cache(domain="example.com") - - await mgr.can_fetch("https://example.com/", "s1") # refetched - await mgr.can_fetch("https://other.com/", "s1") # still cached - - assert len(fetch_fn.calls) == 3 - - @pytest.mark.asyncio - async def test_clear_by_domain_invalidates_all_sessions(self): - """Clearing a domain evicts the single shared cache entry for all sessions.""" - fetch_fn = make_fetch_fn(content=ROBOTS_BASIC) - mgr = RobotsTxtManager(fetch_fn) - - await mgr.can_fetch("https://example.com/", "s1") - assert len(fetch_fn.calls) == 1 - - mgr.clear_cache(domain="example.com") - - await mgr.can_fetch("https://example.com/", "s1") # refetched — cache was cleared - await mgr.can_fetch("https://example.com/", "s2") # hits the newly warm cache, no fetch - - assert len(fetch_fn.calls) == 2 - - def test_clear_nonexistent_domain_does_not_raise(self): - mgr = RobotsTxtManager(make_fetch_fn()) - mgr.clear_cache(domain="nevervisited.com") # should not raise - - def test_clear_empty_cache_does_not_raise(self): - mgr = RobotsTxtManager(make_fetch_fn()) - mgr.clear_cache() # should not raise - - @pytest.mark.asyncio - async def test_clear_all_empties_cache_completely(self): - fetch_fn = make_fetch_fn(content=ROBOTS_BASIC) - mgr = RobotsTxtManager(fetch_fn) - - await mgr.can_fetch("https://a.com/", "s1") - await mgr.can_fetch("https://b.com/", "s1") - await mgr.can_fetch("https://c.com/", "s1") - assert len(fetch_fn.calls) == 3 - - mgr.clear_cache() - - await mgr.can_fetch("https://a.com/", "s1") - await mgr.can_fetch("https://b.com/", "s1") - await mgr.can_fetch("https://c.com/", "s1") - - assert len(fetch_fn.calls) == 6 - # --------------------------------------------------------------------------- # Tests: concurrent access From 911d3af6324ce0902702207456c9ff5f3c3be6ca Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Sun, 5 Apr 2026 02:33:15 +0200 Subject: [PATCH 20/25] build: pumping up protego minimum version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 4333d7e..af8f37a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -79,7 +79,7 @@ fetchers = [ "apify-fingerprint-datapoints>=0.12.0", "msgspec>=0.20.0", "anyio>=4.12.1", - "protego>=0.4.0", + "protego>=0.6.0", ] ai = [ "mcp>=1.26.0", From 070338cf248d7bdbf5f70ee599cd5e7490bb07d9 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Sun, 5 Apr 2026 02:36:15 +0200 Subject: [PATCH 21/25] fix(spider): only allocate _domain_delays when robots_txt_obey is enabled --- scrapling/spiders/engine.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/scrapling/spiders/engine.py b/scrapling/spiders/engine.py index 1924298..f1baf76 100644 --- a/scrapling/spiders/engine.py +++ b/scrapling/spiders/engine.py @@ -54,9 +54,11 @@ class CrawlerEngine: self._global_limiter = CapacityLimiter(spider.concurrent_requests) self._domain_limiters: dict[str, CapacityLimiter] = {} - self._domain_delays: dict[str, float] = {} self._allowed_domains: set[str] = spider.allowed_domains or set() + if self.spider.robots_txt_obey: + self._domain_delays: dict[str, float] = {} + self._active_tasks: int = 0 self._running: bool = False self._items: ItemList = ItemList() @@ -300,7 +302,8 @@ class CrawlerEngine: self._force_stop = False self.stats = CrawlStats(start_time=anyio.current_time()) self._domain_limiters.clear() - self._domain_delays.clear() + if self._robots_manager: + self._domain_delays.clear() # Check for existing checkpoint resuming = (await self._restore_from_checkpoint()) if self._checkpoint_system_enabled else False From 9383bec14ea13c5351ab434ce6d16d50f88988a6 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Sun, 5 Apr 2026 03:43:21 +0200 Subject: [PATCH 22/25] style(spider): removing excessive docstrings and unifying the style with the rest of the repo --- scrapling/spiders/engine.py | 4 +-- scrapling/spiders/robotstxt.py | 54 +++++----------------------------- 2 files changed, 8 insertions(+), 50 deletions(-) diff --git a/scrapling/spiders/engine.py b/scrapling/spiders/engine.py index f1baf76..deac28f 100644 --- a/scrapling/spiders/engine.py +++ b/scrapling/spiders/engine.py @@ -136,7 +136,7 @@ class CrawlerEngine: can_fetch = await self._robots_manager.can_fetch(request.url, request.sid) if not can_fetch: self.stats.robots_disallowed_count += 1 - log.debug(f"Request disallowed by robots.txt: {request.url}") + log.info(f"Request disallowed by robots.txt: {request.url}") return delay = await self._get_domain_delay(request) else: @@ -310,11 +310,9 @@ class CrawlerEngine: self._last_checkpoint_time = anyio.current_time() async with self.session_manager: - # Set stats from spider configuration self.stats.concurrent_requests = self.spider.concurrent_requests self.stats.concurrent_requests_per_domain = self.spider.concurrent_requests_per_domain self.stats.download_delay = self.spider.download_delay - await self.spider.on_start(resuming=resuming) await self._prefetch_robots_txt() diff --git a/scrapling/spiders/robotstxt.py b/scrapling/spiders/robotstxt.py index 10dc7ab..18f4263 100644 --- a/scrapling/spiders/robotstxt.py +++ b/scrapling/spiders/robotstxt.py @@ -8,24 +8,7 @@ from scrapling.core.utils import log class RobotsTxtManager: - """Manages fetching, parsing, and caching of robots.txt files. - - Accepts a fetch callable ``(url: str, sid: str) -> Awaitable[Response]`` - so it stays decoupled from any specific session or transport layer. - - All public methods accept only ``(url, sid)`` — domain and scheme are - derived internally from the URL so callers don't pass redundant data. - - Handles all standard robots.txt directives including: - - User-agent specific rules - - Allow/Disallow directives (including wildcards and $ anchors) - - Crawl-delay directives - - robots.txt is a domain-level document and does not vary by session, so the - cache is keyed by domain only. The ``sid`` parameter on public methods - controls which session is used for the initial fetch if the domain is not - yet cached, but all sessions share the same parsed result afterwards. - """ + """Manages fetching, parsing, and caching of robots.txt files.""" def __init__(self, fetch_fn: Callable[[str, str], Awaitable]): self._fetch_fn = fetch_fn @@ -60,21 +43,8 @@ class RobotsTxtManager: async def can_fetch(self, url: str, sid: str) -> bool: """Check if a URL can be fetched according to the domain's robots.txt. - Handles: - - Wildcard user-agent rules (User-agent: *) - - Allow/Disallow directives with wildcards (e.g., /*.pdf$) - - Allow directives that override Disallow (e.g., Allow: /admin/public-docs/) - - Uses the wildcard user-agent (*) which matches standard robots.txt directives - that apply to all bots. This is the conservative approach — if a URL is - disallowed for all bots, we respect that. - - Args: - url: The full URL to check - sid: Session ID for fetching robots.txt if not yet cached - - Returns: - True if the URL can be fetched, False otherwise + :param url: The full URL to check + :param sid: Session ID for fetching robots.txt if not yet cached """ parser = await self._get_parser(url, sid) return parser.can_fetch(url, "*") @@ -82,13 +52,8 @@ class RobotsTxtManager: async def get_delay_directives(self, url: str, sid: str) -> tuple[Optional[float], Optional[tuple[int, int]]]: """Return both crawl-delay and request-rate in a single parser lookup. - Args: - url: Any URL on the domain to check - sid: Session ID for fetching robots.txt if not yet cached - - Returns: - A tuple of (crawl_delay, request_rate) where crawl_delay is in seconds - or None, and request_rate is (requests, seconds) or None. + :param url: Any URL on the domain to check + :param sid: Session ID for fetching robots.txt if not yet cached """ parser = await self._get_parser(url, sid) c_delay = parser.crawl_delay("*") @@ -101,13 +66,8 @@ class RobotsTxtManager: async def prefetch(self, urls: list[str], sid: str) -> None: """Pre-warm the robots.txt cache for a list of seed URLs concurrently. - Callers are responsible for deduplicating URLs by domain before calling - this method — passing multiple URLs for the same domain will trigger - redundant fetches since no inflight deduplication exists here. - - Args: - urls: Seed URLs whose domains should be pre-fetched (one per domain). - sid: Session ID to use for the robots.txt fetch requests. + :param urls: Seed URLs whose domains should be pre-fetched (one per domain). + :param sid: Session ID to use for the robots.txt fetch requests. """ if not urls: return From c54081ff1a4d5d3a0111dafd6435602e3c9bd9f7 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Sun, 5 Apr 2026 05:26:34 +0200 Subject: [PATCH 23/25] docs(agent): update skill with the latest changes --- agent-skill/Scrapling-Skill.zip | Bin 79160 -> 79799 bytes agent-skill/Scrapling-Skill/SKILL.md | 5 ++-- .../references/spiders/advanced.md | 2 ++ .../references/spiders/architecture.md | 3 ++- .../references/spiders/getting-started.md | 25 ++++++++++++++++++ 5 files changed, 32 insertions(+), 3 deletions(-) diff --git a/agent-skill/Scrapling-Skill.zip b/agent-skill/Scrapling-Skill.zip index c89b5d7f7eacb36b382512f54825ce58c9c17374..b03d956754082a4dc96681054135ab4cc3b6c73a 100644 GIT binary patch delta 17125 zcmYg%V{m3c*KMBIwr$(S#I}=(ZRbfQwr$(CHOa)z#LmQa?#%nW-@W%#b?wvDy?6Kd zv8wi2d$0C0@U|;(1SMH8aCDG=&p=3U0sv+&|Vv+U*Dc_hr?k|Bu@ulf4Ag1gZXlUanXf@&}({0Lz4 z5qQq6I>Dth-~}=bhjdHk|2B@2lJHf$jSju3K)!qNC85F4Sc@Srn5BG8jMKc@B1BMt zq+$ZU5Y8NDmWz(67q6(i3T+N$8%cKcvq?|w7HGj}qj|$fs>bS4U z+Oge6t1WQNZ2Wxll)=xYeU(MGLuS-VNKB=7{i@A1$ODgs97!b?%q6;Ee;H7#A}g&ENrz$K7AbmDFjUKeEL{#O zA9Wmg|CbL%>WMM*n=S1Qxh!0)BoZouqgz!5bVG3a!Sg=jQ!ttFlM@t zq7FuaW z7rHE!m8@fhc4o2mX)kq{n2MJ8+A5boX)bo@YG6(EVobwIp^}1wwR(6<11Xo=mQIy* zAF{fxx8F&!%lcyXd<5P_o5|AIJkgmBd!yUgIPHptOna+U^g{*rorzh&FpLA|PY2_l zi=C1>Qk3Q1MK%x{`8qGYvVy1Ucq9_Ke_BGNgF5O2vm>%<7+HF#PeO*?3riYCEM2$e zEPxL39ndB$RDJgF!6pWou?*+o`G8H8L7*{ z;FBYRS1KarJ)Q$u?|UJZ)impR9r?hT8ladXMIptM&UdV{Ck>uFn5I_*VaE#3O=GbY zRNXe@^&w6Q0Zcsb_1lB`85z;Rv8X&`#{?dx8pGCB*ELX|KS9Pk%%*~LHpWsy6f%xY zUZa*)Ib12!za)>XZa<9puH>1wEA>ui>hlfpI7?~FTq{W7$h#Lut+!$1tkr+_SOVX0 zS=2Gvtg9(on&~lw&Jc)QxU2Cow9czwf566SWxkiRT?BK&>3ZV!LxGia9j#ujBVbG7 z7}smG^{&+SL#+`14j!U;l+m|3$+MGbN~GX}g@aR-)}I-A8vMOjlJJ}_vu4~kT_3ev z9ql0t{F7#0(ivS}Y~4Yt-T=LBS_%Z`Zfc-ML*8$xpt>{a1WQCAm8hRcfvs@%gihzv z2K@y+Zl7Ka;D+0c*KT3zQp;ZHh}><*sx)S@@`Z*}TK*QsDw2oXian{txiz64T-CI3!%>vkCnda;DVJ=d(MbO8f0P55_0@ zDLeol*3azU^d7|vYZm)Hs_3?h0Isvtr`@rhiiR1={R z%T5I0r2t>qTE_E68)Ba_M~d&V1NvR4-81-7)e^5VR8_EbLaE*=ZEPfJ>=su-1}_)^ zMZzaBdDnTKZv_5>PAw4t#Z&-;0FOX^0-qIv>mvUkmS1pWy_srRR#Wza%A9H|l}Ygk zF0!ds8INjy!uEuo+ZR|ci|SZrP$VsRgktccL+%|BBb?@2G*>1aXn0$Rj?ewXgJ`?W zCi}r4SWYw@II>-sC&HVURdqjpIB#}3EHnf%zMmha7xrB^ejUS$I_P}?aF8QJ_}R#V z)_;I*rPh;wzl52fCzBNq?p%QgQPONU89hTQP$J{UMxhmGw-vB%)_2((>~#f8k&#nu zgYiL!g)qT#1=0Vz_OffdI#u*kZfXf*qKrODSX3{AcwXB*jX1sZYKRAcV#ZD1ixcVf zVl#(Ocu=KoV}H=Kw<~;59%Abj*X8rcx6PlGH0qgIXFDt8Q{Ph~%b_1PE1Y|d_}L0*w^lB?!PE%(8LEVT!`%w!Q!1>^6+ z?!^Pe6mBUmLCc{9bu1a`pU3?+hLu740!?k(1Vlc?09}4Q?)0y;#~6{etZw+;Rl?|! zX(;ShEw?~v1lwRanCfoWU+GX0LivkX=7C<}BsUFj-_3y{R-4~0E5u$EX>JUPETSPF zQ7VNf%-6661+!!C3YZ^U`m(5#TfSMdgY>^KQ-@C^LLxuB@b-#p*NvhBe8wDc8uJ1I zhhmE;zwhBTmodP?)MeC{5mIyp|uqr-N>x?JYTFORY?TuAcA&E`b%FQ{c zNnQ@N*sB1|pHIZ;%Tf_yNs5Z}GSn11z)W$YsPe>#bA>n3m;~d4=OseCO4|D$Ll8q6 zPl6h|yrgd#Nr%DtBO=SmDIP4lYe}%lQG7gQ!oqj`^qNO$j+A`8C#gEijoL1t?30U5 zXRMCsWD2QvEjsNb{hEA`gHTf+Y{ZFe9NS7CoNOJzrFO`l<$H zJ+py~5C7Yxq$rNZi=rO*hWbU=EaT8dGsAP8M7wDC!9cj_s#<1^5;3kwwy}(3d5O zlLdgRD;V`Qtv`>K_bX6FNMK4y>3E6;9H#2hN~clSp?U0eKgj>t|Bx+C?q-&v0=S+K zDBC9F7@ttdk3D3F`;EjRMiUE@-I>2+u{{9e-)?-QnHKFVry8bysa}Hf9mAi9KOjX5 z;8!*=AEL4eQ=dqZoxwJzGRn+?hM%Q`YM~B96&B4QP)1AXAfT=G@0g~M@8nQd&jiI` zY*Q9=-kpfrzT5`YDiii4S1KJT(Ol+(jTGN3#QI`ML`da!DPNkZ=6YTxb`3%0LKy=6 zhn^RZ(538078Rk;2aD*H8W>}y9K|K?xx1v62s2*W$M&KZNmN7OEe4*TZxqVeD0vLC zACmWydatga{i6>d)`Iq!roClubQhi^_FNcgw~oDuON*MzM*F5sW!#KKtc0Njf5LQh#z&N9I={3OSLmvi({M%wS5gFhfFZ;Tkc7^+4TuM#`cBJ-$j87QgI5? z+j+j-?c9FAmTvBkm9d4%s>x`?<6M^b7|k~^T#K7-nKJSo&^s^ZxS@)`-Jyn3{==*h zifw7p6NB)7-?^``*D?S@^pUD0@?K?!= zkON=*z>lMI6UE)(`WppEkue}j$uEITmNkUp#W-Ov#hcR$fyJV1n*NxGez2unI;=!0 z1bp(G`iLTZMizt3Cn}y$EohFf@CK^%E>&P7bc}_x9%I!0kXyemf0^G9+PVq%&x`V2 zw5vD+KP&BR@{{ky0$=GnkBXs+H2!U>4!tT)T8qg^S!qgLussHJc#L>(U64H6$osqg zeN?-^e<`)5&O41QEC|RrUXy|#E!=;yUms~1;J^Hq;lJ7Mrq-sz1~)2j#ZZJAsRc|| z(s{JR+tDlrwKakS-m9oBvzwH0D)=0--guZT!J*~{FUV`sWiR3!-KF+`Vqidb zp!2B=*t#Bo_A1W6t_lb}{n zXS9Sm$zpBcz=?JYjD&PT@k`jrg}FXVYOGj;D(^K7K*AL^`-J*{ENZytSQE@G9zAk~ zQyOG%TUsJlBg*Ii)t+JG78E6Kf6zmhdk-Y2(=s`^OB=;OvxC3^VXSy!kn7ZY+Zax6 zxyTPAf2Bw59EYsham0I7U15mXSF-sf0(vJACNezrkx!835|qdn>e4rf$@XL7XL;eM zrYv_&$My3tN%c$NP*P1qUx)X$ZPlH|p2Qdb3{nJk$kgov0}Tt`jj#pH2vWS@*Ng8G zPJ`%|P{P#5MW?&mj9-_9B6ocT-At!8QPL{Sj@3HcE1B(1HP$B2l!xPzF@RN*R1UC< zk{7L@=4;Zzpr?|gtCM}v7TqjrpGuyZ`_d!z_vw=TWCvSWot%8mt0neVdWpWwjFduL zpzS84-@Rf0`lCwt39JW7L{^OWmILuJ>A|)1_-Q=6(3Y4(pC? ze!q;@RZ*doox8w~CU_2{n2<$vX&y{R3*d?Ep$CTnQH@Ya2s;>EVe2ETuH_paO`5Rs zVhAfW(SBYwEGYX8WJVADlgC%?zI@E`&U2o3rGJ z{n!0~!`|~Qf2i@TZolmtAGfg!#g-*IL+$NcV{W7wA_bs(wRuh_-rQQXuAb)ReT5t8 z=Hx}KoG|C|RGucpa-Fp=xtc7iK2hA>{CsIX%?t}Y+XSOfEcA)8qH*UIBBs+Xd_7cL zA|RYBb4y{q8ZKRFF*H$ZNEa!)syPvX`SBnGto?d1Wfg#;@%UKFaK~(Nl_WZasYO;K zK!1FrSmD+?^9UD3nw=NEJ{x^JOa3iO$ZAolzRTyzS*RLqR-esVB* zNT5=<(0QMFaMlNJ`NJ>XYC`fB>V{9}7lCg2^1fo#%CRpilTVq}*I(cm*1r#p2*u>T zj7%?c3B3O-52crIoUBi7E9dlPZjrXv0^fe!gM*Z>k~$;x52J-gAu){!d)asKtC5tt zr2D?gbFR%`k!~B!bL$MfL-~F-s$&Dgti{-*M;t>W)1AxjBy08&xy{pNk1{Js->Z?b z4zd@w#(o?zA1q`kl5GfFhgM1AjiSui!h?IcLjB&e&iphYfamNu?BRG^HV=yT22xO; z@d)F3c`#chJkU&EByn%3#%xoAhzZDl6-zMhz)XF8xBh*k+kTb4$0q`OCH&$l{_^lN z3oh5NXEtLjE!{rtB;0cFZ?v9Zf<=?cM@<}McR(7C0m03+gQOCO5yNwnb!D59ZAhtn zx_jP1qZG`QgX-Rc_L5B$8-yPN0oJ3jA2_t^_m6`yv$D{aJI{a534gdozkz?9t2m^l zFcA`D(3cPn3p0>5rkHTX9LGvWz>>7ujp4-sy+%rD2pK9ADBkFlY2+TsDK;v`_ z={W9QJaNXEWwnVB8ePOEm=O?@**JZG`jGQPs(8Nf{^s)zA{N7$mnxstvHv|;H8y$^ zvvH?6qs%FvJu)|XHZ7kDN(3Gf4vHP?jiidNK@XCzDQw~rq;H#MNw6R>cYbol5`XRd z3-B+0VIt;o3xfs$iNgB-DM8A=yqlr>e|R@?AOZ*!^?zx%fA&zv#$iJe^P|>~g>EP& zS*CfvbW=3IV~*Po+%A*YT^U`3IA%P7onvIz`P+u|`B(4p7N74+Li4G=Zjq&}v-#!Q z?=eS3za+gAdd`T-ZUa0a?DLBXw6qa*<#hV3xhu2X`GFl<)8rXD6WBwKUUs1FIkQ1Z z)*v=U*}6~3WMlngW$~E8NTaIV?~?gHZpPhhSd^PIx|Qv-X%%@0kSk|d-bvm&$2tOB zbowO}Uq}0cMpXZrgSs8N~2OXFm zQZ_YoQ`9#ok}otWw+d57&24}wSd5{p#%%hcU(8Wkxa#XQR^PkR_V~F*_?>j^njn<< zyX6};7ZQgFDSQ@BlnAP(=8CL5lhS{Fj zZuh^vb4o9r3l+v`%BDWR@zSHL3S`;Le*}$~BwYQBnS|WCtkbuab@|D{k_a4a0w_<9 z6OUP^-$jxL-IXx@^inS-bl5`VGRt^d8wl)}-{9v^K5rfJ05j+)%f)F4#ZW0?wCg~@ z%i0@6fBz(0*c&fH>iGjqCzy=&)ZW(cd?)86AM8tH$qm5HfYP90ikZN*Kz?lAHfxZX zk~!?WHb0Q>%ruIb(LeeXtuEgOp+7q_oyo3z6;=`=H>BOf9oa)Pv9MOnu6gK)Un9?h zi#Wzrd^xbQf^Y&{%&38yJQRPY3XgQ*&mW}=U$2A+9~FeF8^s0Ykf%0)`+t)$D6_9n z)kdjkLYN)__IIeKRLYu!1V;UEgcELL)+2_3q!*aaa7uw1mtUGzr(Gr%-74X9@2Qx% za(3xx;3fy>?1Ez%`msvifW=u+FBUEyG_XHr!I>xEUkx356^suLIzzTgjB~A3bBsn~g^VrFn?r3SL|lFH9&#hb_co##1g`9ilbq?);WBNg+kQ zIw!Rs0!oDk6(4Duls5beG&6g0ov|(+iz)qC#>e|s1G5KjNx4xHTTK==F(}R3Zht{# z>=>%i+V}c#-W;iR#?}|nmfr))cq!c#B)n)FL$-A!FY_H}RCq3P3TLO<6D;d4M#v~j zbE4JrJ0zS)iqwj3@kU!Yqk)@|4=IBP9o9o+52IGc;a0}=b#G_M)%1RZ@nG=V$5Ew} zRV6D}lnBLj1!^r=2}DSYf1|dWTm8~a)k4D{N)$1&!3V6NQ<{G?GjUJqwa1bh>t>l9(`!d}I6qSe6UQpghRR{Ar)2d8OO6&tsOVg! zv3BS%CS1rDz=FV+epoFZGVjSud9EtgVCYY*!Jnqsu=frAxe~$E%cDkTZhLtW#UD7M zL4hS|(pRLZg`?)#S6zwuri9q0(tuw@_hQe7=K6u)4^;{I5%bNSiLeulTaM4V1`vc&5PjH@aJI)jcJVMQzpwYUHu zXKG}%!FmMbTZ1#9lt90X$eO*GA3eoGcAQ>uw$6l zsGq9f=o&S!Z4|Cd*w-peq%|ZZFD{d?(DBQ_VnQQbc2Ak}ykbcUb%oz9g|Ml1z9U6? zw5|0g3NfsjkTXH+Z}>yhYLS-AAjDKes-3ifLnSd*Z2Obd zSxft;qQP(b)Rfp)jwT7r94gDlP*^97u64)NZX}0zJn6>scZ84T4pNc_e3@*Y7HPf# zJkKtNM6+jYdYKO__0ovYQ@@Pd1EG?)0yLNl{7m&(@|hN%yJD;jsduC#EInz-IIm#Ze>&ueEDSA_XQNAr-S=YzER|9{96kG2T<`>6-Mqr+|G9RxzFFRP!Jq+LL%m6ZPX`^~Lhx`3qks zW?SV#Sffrq{Fm^Y_(_`x(7SNDF)O+H9it0euw7gTO&PTiGwoc{2Z@4H!B-x_c_WFk z5xw_=Z={C2vs5D?_mHZK^@dS0>IO_D{fdKOQFixlwMyCHB)KWu!P|~m1Km^6IUhoU zB^(4P71G|41|}JyG5LFOi`A=vxe=vTtHO_pw)6e@Y6rZ_9t!H9wv)# ztO8C9PmNg`mF8spjhppXJnv>#BEX|EfKq%AQAw^Eax6M|PwZjGyZ?UWwZfk2hQGY*Ux!~raJgxvQ)7`3n z#yQPVW}qn-V-P~NCkPFygt(g1+I>Imd&Y9BIT_cUU2x$^G3pr=s`1R##v0ci$75Yd z23S({fD4{Z0llB!62&({RHZ0&q|ioLW$43C>@G)Sns+Yh&2`g;JlLHDdHG+hy`RCq zCRTZnjo9k_c+tew;D3{|(*R&HcC+tCT(bKm()CfMF!>dK+1VriS=1y>YqB@as_B=` z;uPJOcgTYZ(&UiYwyTCfAm;@|=O}39)YeyagT4Cc0n~BDtz#mrzhTfJpkGt1Gt9~5 za3vNP3{~0siCO-%4&x5pp={dwHu?;OKDFjJPoHSM>-kNbHx4E`b{nbS+JswGEXUR= z4R>BHhR_pj$a5&m9V(?=c6g5^jI-!6J7OZysZHjr`(riOAL-Hqhu63#iLJmsk22R%Ca|n^_9IDEkJ>>oGgw65DQAvb1QE)lloBuGlvD_Jo4x3p z7jlaIXP3vCB{cSU6>Fi8XJvwLmN6$h>5Scc25@?@;$|CREs0l~RqpOZ{*yfOp>u_G z-xgEa;BeicuFA-YH1^M2d2(r6l?*L4Q91da68X#ckVL2>q&i?KZ*h$Kkjm$0f2?Lz zN*P>yiu`VOPK|Bdfk)BV zeqfR8f_f`|?%n=n=y3|Me$!$xZH2WR$CDV^g}s$AQ1k7^kX^(H!8JJz@!uTQ@%O#H^%dEtIU=i|6vA_2Ou z*WEhR?4y@Yaf3-kLqk$*4+oP&5tT#&0zltLDMMExUlAMo@N+g5TbE6mrg_Jar6PR$ zn%3guX3zP{n3WC>EB#F{Oi!Q3BI;1{9-{6HV#khl0K2lK(BVeQs zLkegA)3k(6#@%Fp3r+CVFntvVc1VgNdk6HVk*@yP4@l<$B|Xi=fM#yZ5s&vxZSiNsl&-j+Bsm` z;N~EVFT{B$$Q##4^<$GPI9PT@a=&#rV52t6$#chXsLM)6UEwqf!#-k={1Je1 zk@o_1o3Q|E0*)<2MZ;YmWATN)EbD!RL{MVqSAj--+~yFDPS_Wc#bv{riSRpKsrDfZWsB5rt@auY(Mp+#6e` zRR;>O4*PHT?aR34I2^I}t~DU_)ZWnnBA024kO}Gi3U-G-5L~Sw?$4(mm#!xC5 zg-@H``3}qovctbV9Tq+=T!ob1gr*)1rymu=fI4E=c(3JT^&|tzY{x>Qzp`r?e2Q!T zguX5K*Dl$}lZ+<0w;kCib96rBAo_jVxz>w3clsKtW@b~cL61jj$ut3eZN)XO?_TG9 zDMVpD`eNAF5c%EQdDNXjRFjha*!JTKrOd|hzgmM(G^s4nXgaiVTP1_V>q(t$b)|{OXD%}6H`TT1J^2v8({=XB$xgWfsq#&~&sA>~R z+Vk9E)$q-S$n>ekoyhpEyiyRvz*O5NI0zHh%}j&P(A@07k{wp1L&H>AykiPAhf&JB zVKyOE8kx?syh$a?m7ems3y7L#&B)rJ7E%stIThEC)xLNrGv2&d)Il@in>0~~{2`u4 z=aigIYe3GbxmW#jp>Qz*7hXb{Hbb~9CDExc|AC(?jhps8`BdvLMC+iwT{bBXDI}VL z{!sKrrXoeo^oGidgsNFCp$eUMfT`fQGoYkajrO}!t%)-3Wl1RLC(t7UK4c=Klhy?% zg*3G!IARJng3h%flIg~9A%wOlMeAEme~>c-+K__@MO5Zbdajblc#&1Av28`J!=J^Z ztPPcAn`PR4!`dAwB~0*)J4`oiPt@Gr+J*Uc6}N6AuC!a-g1r7* zJcyH3tXfqndxK+v$UwR#jdm*L@k;t8lkUDN54Nemj{4_Saf$L@y)sW|FyyIkxan>f zwLjhFgIMHB=cyCYxO0)M?RpoOe}8=*1K(G^-kE<;7GJL!daZbc7Cl~KWaC^Sfaz$m#W z*_x?mw$O9jD{?YBnvfo(t)|10vnku83>uLSYHYHu+GD_G2a%&K4OwTTjhTS)LJT_B zEW@a#4AOtgJ_a7qqKDOEB8W8#b8>?gN>!oAnWT^`B}f<1LJ81?ndnn6+W@?)l(!Nf zAn*d^G(y(H1ZtNn<>_gsFyd`wB4ohcpbm;G!T6Ra==mUIjJ>MG3%I2*__L2$x8X;S zXvZ+BjiXyay#jY`e{pMEAKNIFtW86^|BN+W3`qFxWwOWT2k>zkr&0pIYxoEl-m zU7>@%rlvyfm3|2Pu03c7`M@Nil>E=ZT;P@&F?Dq5VzRfurXs* zP}n3rEudStaRIZDedXa#2L$>iW>SttWGg6d(`HTOT9Z`=EsF-A=WC}UEPfdCSdvoq zz)uA$%_>mv6i@g2ceUy{kiDsd7E|J*P+u6O=$O>}*oq>Ajz5_BMl!37kG~(9_c}mz zG00fzEw?kw$2bHa3LxEo`f4|zCa~UHC}+!8@dGmuM&SA6=DZb5bN0b7Wxt=>3P<(E zM@agC9ex}5EG;gHwMe=k0IjK<(8{0{aT?Ok0YhUn7Eg>`aZps;$|LCI^Lcip@6|34 zY{VMr?JwiNC7uOb?fn)*=2shOGgO>*jIDW$)ndVmDJD$0Cg3KVQL?CsGf@TSYk^dJ z;RSxgZZ13Y+yB^Aqd4&(O4ddAu>-vbUY&xD#mhvu%0M?t8v;dH;?@0JjORoX=!UgJ zdB_idl0DBI*fN;Jq&g9b^OviI10T3GHq0DAd0aXGl=5m<=(26N`q2 z4VE(2FT}JkJf40Ig~Jw%NZjT_N_hj~Pzs!z0B>5eWsFi+s!GYSV+XB?KyQWq(T1v# z16r2tmi>arcCXw%nL!?mwt#t(2npI2CXiVZF;~$ce)C-kJgFlF-EY$#;l96l)%}sF zEX<(D9Q z;R-_-*iE6XV(_Hyw+?s(@o@B+D+2Pm=kpRX4Q5*E$blKW6$oW#=k^GCalKH-Ds1%% z_&(|Ra{GQc-?G3LJS7}*ff6a1&An$=;%mj!9~|CfjFBJgDj|Dzd9J~$^-l*gE0{+* zKXI2)CHWwz>9^%C)LpSDR6}oHVjsWUWbHTY`@oNA^?VY4rcF~DsoL+nCIWRj4e=O} zsdUKCWM)uk6Gkz(yJOTi$@q|vu5!0n)?R7Amf4q#A3;8$OAD46Q=Emp_EFAEoY%K3;=>C=Qk_QLE=}so~{iw$kc`gG)h}W z?n$7KMrn^{=NDO&GuuDk@qy|i>+b8A@_o-s_h~00mNK)(&T_+B1J9YKe*#Q$)6Fcg z2nI8~g<*FjpBrN>np-21f|dt~sqZMKYxW=4bkInqO1Vo|;?S8n(#7#gc(#=WE&Z+E z;G@A}uh}q-poXvQAxRwvHxx_|^974Y!oqD)8lpO)r!NSvajWQq4S-?thC$2t2%mG` zLF;J&S!T{A3A%C#72lYU)4;Oeq+l0}1^9_0%c&)6-{1wNY}KavzX5)bXj`~g=S->q zKqT7PLznV%`F**?V0@nATMA~sHMEuN_!v#X!H%oCX}5nc4tDaLCKG`Kn&qmHikjm* zeQ%Nz8p}c#_CM&*MglHqmnUaM?l~Q&#Hg7ryzt3&cYU8)=#5ac{F!vgs#==M$SPBQMfGG$pnD@m>#X=}|MK zn517rSRSrbv~46jnGD431p}t^1%*I`Db}{X!}jvm-6?sTeZm3#;r;RBE>v7a~P_&#~~J|Cm|X6?ITK`?6K&H|BRMhwkVDsoS_SSTnEl)hLv znIQB0Edt`l(}6DipB|Q5lt<%_bU(9)z!eeAzY!7X5evlip+8^10Db1)hS6b7$T`uP174Y#_~QZA4dg^lc}K_iw#2L+=?7NqjLp=$LBP=<^j^)vVLye+u< zt{>jGe#wgv!7`$gk^D3=r97WMxFH%)O4*+}IAm(!5eLcz@-Ey*dr*{~Mti*OX-w_D zvrv9^6Ac8QdLq~K%lo66u#!VE2k32=auS$)s?fkjv)F05NB4KE*Qfdl;`N)fGE18< zO6_D*E6@!K^2_~tbRj(=C>;vNc05D3CW|gBEwtC6`=xDAF;mDX2QRGHv&xNn-j?&L zj$+h3Qw|8i1Xe9YQlxixTpSSrIENB7QCWk3tEDK#Y_f0*Si&Ynm!j7Y6uAOz=!H!q zA?E3?w;_gC#t%Z7)j-z}_laQSmYe_-29!1^kd9hD5JqAVgTlFR1jieZxClTqun>Ck zKv_On{_(P~K#sK!iLJdjPGh4iP2lJ*;rXnH-~qNSM3wz6HW$u@vTkP>3I`jJiA3rZp6z1n^7(Dx_plW76TQll@)e>v#b1yXYJ)Bso z@3Q1>qn2jg+}b5^&V*NZh>ue^EX>^Lf&;gj$|9kh2?hLb?`35?h70!pOM|ZOX}{=dag$d7EG1rD+0uwymhR;I+Zx8{?JziSM;|p5Th)<5CbVJ zPf_USE@pK?no@<5oG^mI++#{K^-S95nts*xP6`5UWd&RNYuo)K;&30-6wq$wmzQr~ zRh#xLxoe|#g}n)x4P|z;<)_E`Ut9>{$`9h=*DG_2z6W@15#BxdTWMi{ds&(nm-PJ4i)wm{>^yq zZnW0b~#3*Z9OrkUXa_<$+rglwh6 zU9$<$u}kacD>|F&1q>-y=DUR7QL}CM9|~o=MjH~g9Q&Ve|961K067;+rT|1x|MyO%YcW{GFrq++TfS)Kb>gDXbHi@xSitX~0S}XUSro!CUY)oRqUG$$J1e)pn?4|^3R{_rPtjyk~WcmDV zk9cu}&%gqY8JRC`^~SyD(`;7JSwybVctOR&+^}YB(7pR{Q5vjg809|DU&z$lIn2AX zx<#9hhhQ~$xMS&XqX4cC9e_fj_at%W6T%cQY}yqrn^>O${4G53maPkwjPsVruBp-d z^>~sDM`$cZ(_6=WJq4^mI0j!CYSS?KrRRIpF?KsS2GC>rU9H#H^8M0TR@n?YAb;oX zp*X-~+?yRpB30K=AZrM}DMZPyggje+5%w_}aWp_!ru7J^HD_5y1UCRX^R_`Rc|Z=Q z1bj6Af%x|qkAB5U8mpMvRmia@IN;;WM={ZoW4RBcXPD`QWn^Ajx;NiGuwiL3>T6HE8qgw#X1?rS8a>@+sB(- zC2aGj?!C3g_DIBG=(hY*eoszLncwkrxR+S35?c)e|6OxeA5eqV#^8HCoL3>Jct-rL zX>+5Eq16({mp#ytzkWd0G&Ld@Xf`Fr!0&tqM8=N__34RA;dkcc9$ z6P4+pg|vg?tSb*%T}Y-H5mL010F zSksaoG1D7=*oVW_(JeCI7Av*b#unUU%*u9oMS2FqAPIPS6r^9%r+R}wWx`lrpya2 znM{(vcQ-D^o)`S(46@^V=K0pOyQ9O&n+MiEj9PrEQifr*w#Z+~-;q5;W<~L+KSf<# zngqWb`rClcEV@v|Pw8yF6KS-CH?0IZ=SV$S=__yF1gv?>wxEc@$Ksx+>oc+%(?Lyg zGkNmMz#>JJT^%z#J7RQ^OQ-2($<8E!-y5XKLYU}@O(s}9tcd6@wWt_ zqxB|_S}!?p_brMr&bNegX0C@+gpLNzLG%MpkV1hQAJ35%WmH1l6<3Ni4xh5H1f?3C zdZjM9Z6X#hyzWYA{R@ zp3#A{&Roov3?EAPh2r)iFd9v5?e=fylxAeF_KPi&8DR@7oJ;vI`L=L9;N=1>=QPzi zAl0EYQ+RuJdBHnuZvx%sn}2?+WBOmP_+KDEc1hI>SGQ%s*)eg07?J^3|EA&3sq2ME?vSABOYO9S9=AY!VNq>OI2Hye-4Brp?_LKa&A=)ghf{&7 zbc}kzR={)lKG;mFNL3W*mrtC?`Q> z=>;ulB@s{af=2CE^ysQHqLXt<>7{+9Uj@h}w4~;5wboRL{2P-TqI*$)b;!->%Ozmw zkJLm{ct5qHiK|#{DB-6VXB-;WY&>5)ar@LoSR`=b=gV#j5_AXQZz5r*J+SxAV4a-{ zT2*>72)r^dR31Rdso$^W=caBGfsJM`hKM6rU^`66wdObmaa2%OV)3d*?6-$)dpX^+ zlRly{RnE!vL$aEWTc9WzYmvquzh!1> zrz_|pA)ET`-jHyd`u(l~7?)w~Gg5$6?c$==gqh+d9-P(pe8_ya z*f@XTePoY4tjtaH&OdYIwR%XBOt$;|wAB$;a>zBo)|96KQ(L%7ut+-1fGV z+4lI_u2l~mB;TSwoe^a~^d*1>7+D_@wTrz!U6PQ_Wou!16NH2*mJRyFm`8~fKf;=O z2lP{e-`vRArlO)8412aMmW25I8f{yKQW*k#Mf#RVv(C#+*lc}nOtjEjH z%Z=Vp{9^r5L$6}q{q@*%a_r372X0q&RLnKg z_e|H5KT70rk!Fck?`};Lcjl-f!~E6;+*MsfcrroA@*h;6fR4(Q$WzVph1IbD>-jbs z8as<>It_#f*qYygAPTucDa57m@4;qI!O^1Ve4$F>z?ADsqcX!Ft~7zDw(qGc>Y()w z11Ull%o^5=-SKG%t{NRJgyhxeFWHl=_&w?L`(@uC5VdL9mh+pep$2Q5i?{sWW;Xyd zK+=g^;iMO?&tcx|RK)#pb{BPhe?i#vC8*AGH30}UOu)y6rm?qniB3w$_~s#R^2<`D zE_kooiO+Rd?2>!bW4WJJ0(f+ZEPpy-+b<1C-_SJ2^604m-{f#Qd2p0y+ge~6f{_X* zF=aVcNePtPBOol`mDB>}o?|_iVtJ>Lq@cVL{i{a!-cb{4;DCS(s{ZRz1c0K0{Z;+{ z6*2$QE~lAL0QkZF%J+BmxBHv@8k$T3Q!1fDrJOwoVN&1vNzd=Vu2> zkCs+T15gA_$4vX80qB6KV1XpnViKg%|J@%=3m^o|!b+>A1@QgTgruF(0`S1putAdQ zu_!SA9`iq{=>PeWfA{*g=kLAnKufUGjOYMtV1GBHrP2ZDK+SQ|hUftHV9ul zX$JHF7SK|bv;=y9CfE}TND=@WE6JJ#J8ha3fRskV03ZczXHEM*V>D(Io2+RhIQ@n$ z&^hh8jB?Ce#$dJ6_v#~}8uFfb@FNPy2#01-s}CCcxd+ z(|76t!(@dW$O9tNf9o+?DgvDxgS>oH6~%~So?xYu=X)v}WAQI=g=q|8j){Rm4n^~U zAdvkg%)s@g)4%BhEq>gZwpN+xJerK+(^nWWN--_(0EL|B^qYo^e$32^;OZinE-r$|tTSQ^ zWa`-u3c^I@mzTgQChxwWI(@z|Fyyyh28ZqR7siY}Ow-ITdzVI&mDvayrhvB) z|Bl!tS4ChpNI;WA{U-qy4)EtC@=xA`=LQA-kYW0_fs>#}~v7l|-Vg`H8!om> zvrb{!o)6aj5=$}Y8W|Gs8?O{jw6%RQM(V^z74IqWi(Zemx5Ke5i=|U{7=QwlAXf{^ z<+6wdb1P?MK!(jJU3|SRfEj3o#9A#Uvnf5ja5;0Z9Xm^USvcohjjq<-;ed)^p<&o$ zdI~;vo#;z2)gD++_Ys3-yEUimVzY`=&=@)@Wgd(OZr+fPw22XL?MRv&86Z=u3W&)f zGe*>E6FG6`ZI}LR*Re5RHPawl~&MWYy&(8~R~;32L(&}J;N+RZ74xcP0(d5u`#d+VbY5BEh; z1fN1>FisEGl5qd~sokJc%g&W&YHob$k$K1`fyM60f!&K8FS8uApL1NGsn;Y}l7NGB z`kS;T8A`L3f@|oy0E+w0u|`c%*4=Cdj}7M)?gk#k0~rZeGhvOp#d=0^UGZ#|v>2(B zQhoCD@oy6kb1Khf-I`#%ruYz}voboxRRA0$|gOa2@n1K>1n*C2%ti@;tnNasp%-dj)R(U449aft$b%+GF z!w;!$Op>K+N|Z8zd^q!7hGAv^aMvOO5D*j+5_}7ahU!-6-LWes9KQPXjQUk|`|E2g znj5I2vyE$3KD!~0W0^vDzR8m1M}Xy-O`u9yedlijC0STFpUm9TX$Syd5(WVHM}_`s zz#kRT1vJbc-~3lG{-r6BAMzppttn2)x;SzF`G&k8^iMXz@%+r|AWr9qhu{XZ&d2lKZH4TQ-X`9#T=(s0SI z>>ok9^Z)4LUs|N4r)@`ZVD27;S3zY^5x36qkVbQrH5z7(cHPAzh-Wo3ZYlu!cBqd;Mbui*XIRw# zPOfngQ?FV#!NB@-TNmB|9;6y|ru(`{d&q9I*h7Y{OQ(MaXGx3Ro-#w+PJ8UM+&h~Z zDV~)A6&`;mJg`IVqOT2FSQ#^;A6$eTa^8=5tgH+rhLqml#vYGpD012SkZAJk`sAv>aSE$NWJ z7QQzs9U!5rZm3o}GPlnhY)_5J5OSsdS%`*agY(&4)1lC4ADxTf_Yf+E#>1|5YvzsP zH=mSh&V6RClrQ5^$f04OWkvR>J03>%8DpP)l~c~pYwqQ%^B?%jW4jApm3i`4Wy0$V z9hBHlh{u0b+3triXn~ZwNWXv9?-9==D9!X?#k|wzdw@4ES4rqdpO>J*hFn-VW|M#8 z^JJ|Sld9I$s0-aUEK)y`{_I67ls%wc>~yjukwFY??0HVJg)c5hS`|&rr9$Y8!fe8% zSbY8*ni8=AS&XZrVE`^Dp?Jv`Wv_mBJX*zJCKc9K2{ zk36Yj!G$!6;hLxmoEbq}jsyZxPNZf12eulK&_5+`olQW*0^t-$NE~OR>`)am7a33C zy?}A`6?I&L>=Kxd*wmXE(~7C|Hx2eP(dO&YcHbG6{qpLRcc-d`m7l_?E_CQE|qed{F%;o(LD`FBI7t ze&`-&ChHDT|NO*KW0(>2(RYR)>wP%sbg?t%TDdfC<#S3bq`SohbP<~vRg_v~ptf9} z$@fOGE?N!CyP{2iJ@RC7<|Gdly&m0^6y|DEFwvLPCP7b5c#EHaU=4WS%%IP#sOazF3N8TKXOTc}owsJ%bvAS5_6(J^(y*xLzQEwB_DlRo` zwPWR%&6(jTPF{z_9(3a5O3(Qm&r#|sp=#z;(`}0d9v-Pcz%vLGD2+D zi_eS#LP>ouYIT}B$7ZHBAVeLW+ZV@wT6RlJ#&1KXqs-mrLo(Th-T_j!YdQxuc>igr`xZ6d2>tZv)xo~UX8qmZN%?)kcZX4HO8NU zu9YHn2J#+$WjH7)_g*ZcF4K(67+@SA(&f(+tGN0zd-eJ7|H9E99dmeweYiE}Nqp&V z$p^CJt)RWwYTTP$zDFv^mY=#B^7R)bwDEAt*LvC&xYb467f|wcCV?zX-;8-MpsxiH zT|jt{9&AOu5}$#lZtQVkJ7PypTLYdT2IU=RJ@kzIVeG-c@RaG{hs1Pu$AkrN@{18zZpkN*oy4Pnwd$H+;DQD_{H$0ROBOCCjNs+nZLfuC-p z@gz9~rYWowuf20cm;DHZuQD1=b0X1GVz<7#=AJe8xO*QAhmtB}^NB|~U<1S+?l#&a zdSYe#EL?8vi4JjBn)w){xFyDGVq%rix%X9LLs`RiM&g*MQcOQsG zh6gEF9oh-|48aEyV!E*RQM0EBeQ~{c+ZY|Wz16&As+lr)J>9E_9*>S;I{fBBn)z*R zgl%fqFU4Nk8+rfMGG?GEa6a5m-c3dVBKA=aUG84@b>Y>)<#+fMs2VhN`SfuQ)D80T zb{E2HWA^mM4?J(Bb!{stA?^Tze+I^^et0<<&)yf;RF2@A-t);|;zrwd#LpGsBRG-j z_~_?dN6e%~c&s$?;@&fo7Cs2L5`XU@|C}=Pdaco&RV}-94L?u2Uh4j7E6PKPx!c|L zXu(x<_S@FX#vAmxB5R+6$JiSIbhx-3rw;(0v2ez$%+|BvROhR{Y;W+$1#$R0@UJ~f z!U6XY3IPDt*A0f=V#1YnF+^ToS2%>3E%kC)DicRkGF|$gzJ2| z_A`Tzv}kuCd||1zIvgf9ok|sf3CJxswxi^z^yZ|S(#&Qqr$7@6PEGBHXG@V!7Yfyh z(o9d|(zI4Uv0<}bMP8W;S(;k~ti(72{doP_P;5hbE$sIT?PYC>C=o+9|3)X#B|g_F zm~6Ar?)N*C)wy$qS8fCIka%?+d1jh-JsM};WI(0&7R-9*I5(@ik(XEcXHeA9r!FSC zmdP!P{=EJv4MM=lS5$9o*ENRks2U8^J}^ZuR$0`EpNAE`N?`Rf)J&F$Ro@7Dzn#U{ z+dMft4(AAv4?ePu%Cd=-*!GQ#7wJw#^AlNk2^k0+Xb6g(#y_-_+eN$H+M%OnWi$A^ zNFfO8?g0GoYX&jRZ|2@(DBZZXY~8Ch4mjmLk9zIz#p>1Tg z=4GPq(!_aii=Dqbe^J)0SIIK4PpP91vPqF3lMYjTR?P50zD z@2?ULu<1r)mUU_9{{^dZCqb5>mgyUllI4p`4#A#ajI){**plg3FLL zcL{X?uSBrmwhB{hwXvaxG1)oO!&yrzLG;zP^cpMeXHC#Pv=j(UDV8Ed{j@Z$NUz8%$FF*{%)krzD^Ub}!Ctj*@A&GBwt!U3ZRi8;| zaMQw`&VnUWEUmuZ?0OFH; z_Er}+T>7hHUUBQssu0C*5kCYFkckyQL6?o0Vje|KHyf`ZJhTU6xcn|ZDN3?bzWd$g6bY9Bh&+5UZD7oG=63>HFkX=V8E9s=HU@Nr7gZ1zI@Gm{kwsBaK#D1+dV5J|3{i@x# zN3|gq;4#bN2W6K*>W+pfN*en;k%Mz+$NBS`_1Sy(;U>TDV`Aa4zYbXhrL+0P^Ezx> zLATnE9_FT>*``RQ2U>8R=IqO3 z$I2%m#@uDq9h4DG8dAnUh&bZuwdqWGpfm}G7`pRe4z^8cr_@a2)YjPMl`I6BCpYu`MZVqnqdJ5{J*8Zq3H5s^V! zX+`ugY<&7lg~DbdeZE4|EeES+kk{RsOie|+MmR*HIoGZt(d=`e=|udWoyOmne9NMp zHL%J?vONaezWnrW?{Q%IO{FN0*)e(b8lZVmS|VAM`}IdDqE5BMK`{?E^j44fJkQOw z-#wp3Z@swwq9zfOa1=_!mCoGcN3@T2qS6PUtY1tBj1-^kuqy@`)Tv zx9W)*z&^%Pq;yl)5xzB~))m+Qqz<$!BpcTxEfl*}Xj-q7_D}^gi9&@X9-Mao|2UmG z&LF}b+|VG*gP~EY2nv!`dm;NC9qL4SX;p<2Qnkm&o=fE>w{wGTT&27{306Zt# zo21rs@nrTdptS?6n_ZX>2G)w2o1^*n1$;6ZaKYB1qV?}$HMYJgCl9s;8nO3Vddp{< zztdx<0qPd^!j<^RC;>db^y*9rUViMu`iwE92rY3fAhw!v@9)%qMUVRtZ5Q$2M3K}i zMwE@6f)~A4f1B_qn&)^(p}$kUz*qCBWcuDfVNo*20VHl6_2aVIJE544y%tvuB8P5= zPf1&8`A9QU9sEo{)FXgLPsoZ2v9qZ25Y=&I5&}GK`m|^QS*Dqx_KXE>*2MZ+sD02z zZ7W7fuRaW8 zgu@1NHTZXhpZVB-KCLqQLBkC|%-X1Y$43l^#9ZhO-f{gthRKjla!HGuS^1UL1s*29 zk4*ZifZo_qqg|j!2Tjict{T{jUlm&7AZNVt$m77<$~P+ETW9KIMT^!b3|Zt5fze@> z3ljZ##1o@cJPG=$q9Yvr`4oNvJ=xx|M;+#nyfr^b!7(BYGXKRvq9DG z%{`_UL0p0c&dmWhKY*CdHRI;;PkQ_wVM-CPSgmO|M(4VWs$D1P4OPb(A2RCW?&TKx z^P2cng!?Z~=9sn_w;ZemS80ZfIzZ8>Sn848HQ6bYLkywkY zA@|$G@$a8YZci&mgX`-XUoYw^>XVvLwOG+BImu(8GT)2=O73kOYGg|f>kF5K7ExoW zP8D#khV1%s>fAQ*BE)vk?h#rbsI2uE!=iB{QL@)r!T=qFQqpx|SSO@?@h~#Y@7p-| z3pFJJ*1bqf?3Yn10f?M2{YdZ>zrPZ=>MG`#MNmv*KGxh%jA>`)a& zd$C61w*$=%j%zN-=xcTrP}(IY;}6*##NW@Q+^%C^rntMfaOn}&sd;B0DsVpqz#>9& zH-3d0gZU!EHaEuM$IQ->BA`Rv*SD>uM9DENf8WDGZwTs<<}yo>b&AFn5-*7v&RVP= zPp&Zg{=nA+lF7c%lSqFDC4^mmPsvha9s4_+o z0r}o6Z#0ykE`DX!nw6H>6yRPCeW!%1PD784^*E&0-(RFV3+F=?>OZe&mFXR*Ag_lx z$>(UR$R{L{LPItk0b)(ke8E^q->b(9T7K|9ehe7KN@JN^=|>8`R9HaeJm=@2fMp05 z4$}EGbRy0}SswLETmA>EzT?I<22(SCOr{vVBDPj%xS6bS4z>pkBk4+vW7RAXeK7Pz zs+OCple6fLv6oNAu&iwo**N3c@{P{hW5H%!&LW&eHN$WEf*GLsfW_f z{R`}Q_MjZ9ZGRM|rDi`WT$Z$Uo zbl7r0A?>T<&7yv@rkF8LLk|AVp6Oc5?q>bsA%54Ykcu?#>z^zfY2I~_GQpE6k)+vn zw{}dYl@U?k%|TamJXxNEFy=!y>TQ+GZ<@CsxB_}T%_&B)D%hw^*dkfEc5eIi3t(!c z%Hv>BkxW3cf-hs^6k)zqmEYcz<#>8`EiGs^xYSrGJG|t|=zHwEFk!zirp>|k1 zCHJ$~A74;g|9L7bdEFKFSa3gyu``&|dduT*dfGZ%*r>rh==f8+Sj$7OW0ng~{8Izn zf*L4>$%<`M;BNTZgID&dWK8xor?>6_o{tL_Xn!o+aW9fr-_1!LM*pXKn?3t z-u>%go0Z9LzUvCPs*S2%QMzt&UIx!Cj9N`_Y=>;U(gB@pBhZ}h{wi}svHO+!`|-Nu ztB>m*fAMqsGso)uudA@qos6EHB(cOKh-453>nv8PP0}@M&TF|Nc+qpTk-MZxki4z3 zP<-n;?=y$1-LvJ>MxO^Ps-yS)NuI8s;-}I8S!L@Ym7(X&w(kh+-FU5Bp;y^y5={^V zwHR?!IDM$i3!zMw->{K}%*fy)E?HyE1D_tedUvz6zx{S2tgPXg>Q@_&$-H}%H)jT| z-4x&EsyjDJdsC+|9G|~Gz~|vf+qcTzw;iX&^Bh>@IO~m?I5p4EgfI+P-%DU|`2Mhn zAoCg7x^8VZ)4+W@zDqXruYW3Y;fwDY(QR$4kz5yw#!Xd`>G%GG8j_&e7Gv{moYH)~ zWK4%AOdQGyD-hwsRT@}`2e<^yUhWMl63jbQT!!>--S0!jc{<2AJmUZr-?T}=JtkGh z4%>wSTJ$*Q&w6(e{Ja&KBLDP=xWcfg2W~Hev8mn_l z6FEHu;V&PYgR~GYc3Kv6)cZ!eP`uZCV&Z_>RD~moVv=+HwJq!W<6F~j>y$l zul8r-b|1DETjV~6qwJX8FeoX|WY#+-H@o-A@<%uI`*DN2HkBHVP+Zipji?;!^4ZuW z+ap0h_^Har_~%RY&_oCAf`Cqog&!v#-6jiD?!xb!^xDg}>XlV~zaAZgyDM>OqaxtC zpWUP6c-UWd!Fl%b)kMSKuCT39Z%T0zcv4O+{MnnC{<=>BZQo zRK3F%s50KHz2HP|8F@`Rfqpq5`O0Mhvc2$MyRoyNf1QX6e6qCiK?ML#XdBLrl_>w? z>C{ja(T?}u$Wp#5%YP#s^I?1tME+oL6d)rg^WQ)Ef9a&|sWZVy%A2s_Y+$pP2r#VO zvqi=62th&_{0RduFfLm~O~j-a*vOkkJRM3w_>6bA^tL&&>{qcNwd`_y(FW)os$ZnR zVMKazuYS#*FR|djhc45tsH4f(HIGyPe{9e?`aDh%fhF+cOPM?$_JM+4&yi($-4ka@Q|*)4^{>X=qo&Fc{`u1>JTh`v%oyqPP8tVR$|s+Y!}4iTI_G&7{g5fHdr0m~_@G_uObe7?*Pe#_3w-6rL@J!ppUEce=?69#A;U?0zcFoi>1uUFcd0m zi?#(t?r_&7NywX}W~xYhCp}BvyDZ6cHd8@Q*IJmrn6EV0ty~|Q!-2uM0za(zVEUsA zokK^)sc8p@#7*$YVm30WcQb`33}r4!ja*WV7td{A(=CY?$2x_u32JGTQf;af(#kQ-MNb5{HS<6`Ax)OJNm%uL=t_dx zv4$8GNM&(y`0BjNUwAQ3=>oqX*6PGQ#WBz0dIFndWXG`KaTksE(Sa}#PmR&MEFL&0H*aV!d&MO-BbBqw(x z7%;SK3I8*VZ#ALA9*9s3mDqGx{l_T7dML+@43v1D;^417rlsMmRsA_06&n$gCmg+N zt9>wPOJfdNHj3JG!9OJ=snsSov4GV66)MNEN`Nbc@WxF^? z#0N{I+aFPgSF`o)Tc5 zlgD7xiY3MG(vZvEYtJw)n|m5}3DPpov#&Mtm!4~&C*%it$Eq-?*LzRq?3dMGGFou=~aIG+ly0#Ca!-8}z8ZTWIfq}twBHBI!-_3aw7ODeN~RqLL`5WKGh9#$hDJ6-NtQ>}BMSR^fzntyMaYL_I9F-NzUFYj zjH<}lvbImZO_iXMc}1hT?OR~H4Yh6gpTYA34Vn$25o}(h-!hMM5BkVn5QIin<7T3@ zXN?P@Tv6|<5aapfULRD78`awuurCl%dJ7OiT`(1R+OMJ5+rmm_D-WtOUfEnJ$iEU| zF_n>S>LcA}U>9_YcSJbx$s@d`khm0l^Zf7DO z7ZLps&e_H^n9e(K5D%}QkKc{`jWS_&r=QQmMcaep!`s2W39i`X$8iTJA;Yo!M_zT_ zT709?@jdot3~)mT zf{9Hy#@hbS$Nz>8N@#t?Cf2hhfI0zlBmCe_+b9eGddm8^-QCG2A3B)%#}ylc{P-g! z=9&FDUnwt0IVhJ|?BoVF`yMimPXmmfW~vIwGgcj2tY+XRNI!U!kmzmJ6rz?6m}BN_oTww0SoWD24GfuyD1|U@ zjOtNZ!({P$-trEP}3l0tg2PL<4a{7QVDO(Gwtq=3$93 z8fH_@$uxFogsqS=4p|20Bz2r{QHlWjEc$!f13crn?ACW@7)h6DOurT^pQlPVAr-{I z?oN;Gxc1U50&yVMKWqKx`+3{xyhHXWq=aS;76sgVpXZXVGA zbjNR0s#gbI+2QS~_XZEQCRcl&uJ?)!*AM!F!XvN|=mnbMfPu8of!<5=)$8e%>lTZb z(86;*2(?BZHPo+`3~zMFGQU6dMAFh)1>D8gd9gnHN&kNd!*Nzj}$eF&*FiA^OvaRA3f9W{?ohl;A&@R}9j6@JV@4#VY! z(1bp$s;bOg75kH+=7$9<0ToPciM}l%{N38r&m{~KetD_@5E!^vkfdDw=rT7r2)Ki! zWG%A|^IO7DtypQ`>$OHgkNT)VpOfnmsAUj3j+Q`ZxXg(bMibWur$tXkpWpQfhp=ED zpxisROoeXJ=#4srs1z95hA;NHHld9;Bo!C6a~;Ogtnqi}N4$^%oxB2@=f~eMkyfV2 z4pwm>YvZ^o(0#t?toBJG$|#McqFWhA$+&0_mnC0#x%Sc8Qb)~3(w*~w*;M5vx$qqq ziW)oLeft<`o&=i>vN>xus-yQ8J8&8$=tT}L5sIjwq~ybNTx3`81mbIeWU8bz7CrdY z#naz@H}*&tHjy-_4L_RL3j`bQFS{imfM4tF7vA6xg4I~d{;imt>~Wv#aAlCF1jb z2S;tSvWP9PogA!T30j+2GS^I~WXd{T+46R6EWg5?jt(g^@AO?OoMbmXAwqW^UHf(W zf*p7dIxE`^%#w=F5qM3S+ko9lSC@(AjWJOOfMk+!Fl(7`6%Q-5A+y>&N;wLopOOma z8Pn?A5D1=maC-PYy%QMaHr1kXVsYI@k#WEhr3S-pxN4cB&`%IATV>VWYJJ_aeFpjOQQ`6M#=#7(_VZF0~Zvut1OzGGD{r_&Cjed5v-rQmOzkP z%4&k>=r3*@oFtFKcKQ0uG^4G%gS-WYVM23x^hESGb&{1#u25GQSOHUrZTX1zZ#cR; z5CmWp0w4oYy>UL1a^l5It30MHSdsupPW@S>Vb9PLp z>z)*EVPGNCmC%y0i1w{zvg%lt*2g76R3kGTQ!)9%fE6G3F)MgZ8JVD2K-1GU7fhR- zP-IpQAB5AUVI`-@vFMpXNn(<9D&vUQZW$1o+q~GEpymE?-lhDxF7Jv^wW)N=bsXwx z@!TvX)P4JRST7rsnSL^)ZW-t(%f|9?Y*NsEebARDYyvT8*w{)%M-cfgn_ioi(KXIH=zy)M8z-x56qeEY7 z&LniZZt{At2dKue*I-)FX7gw(DrG|wQM!KgRuN$}70l>KC|FvaDPj)0Ao)N{182E> zH{ieb>DDk_mB9zH(z0(k-GW3^RNZJx{{PpD~pal5k z4)&->;b;}9G_+qf^jlk?Ui!fRkp>ssg`gl`*?|24ccz4iwKz3fxGP>cm@v@oS zcr$_dhnx-Sw7~h;{j`kPWHgd1nW9<%%?D@S7!CK)6b-%G%P>$ZEL10ld3vc8kmsW| z5zE(n(PO0Q(Tys zPqg)TmLC~7o;!veJTDjCw^_1EU8~ENc2!owXLPYcqN?%1S{mZ8nk8^FAS3zqVJTAg ztYLG%>^t2Z&UDZlv8{T#obU^)jNzW(QJ&YG(8b>SQ!vbk)6mv88Rur~o8>AYldRnF z=}J4hC%+!%qY=A(uc{PcqhG`Xb+0^rlhb`gQ{FrXR}bMB-X5%Utuy?azbp z>)(8*Oj`V!QU>AGwrKClrf43bGhz^!AhPzhFFEhr+d81Ov?|Csp9#F~eOato&ovam zyYNluQ zY{x8WW^siHszi|9Ed>WqD=yPu8z5aN*GQ0Bp0D*L6%_qFRkw1qw_d=Qd~#J8?%(@z zW>OxT4JAO`k4)r#ASSod=V20JKP0m+KfT|K9H6;c@6lfbPgGU<8yN+0LnthjEX9ak zCE{zQAP~t6Ph3irOcq|YsiR4~gL5AVzZyl2T>DiByK-%rAc{^j=qx;r)iORAtG{8k z`*W*V=-nw1=1bZeK{SI~TJ4|Ds4B>w>^U+dyTU;1*mrG)@P?d-T~HGOO=q-~UjRZd z%0H2RH!F%QzdQ+cGH;yb&ts#Vv!b0N!*od1MwGW?BA%by0y4v1jOGE!!Xvq&9EWRn z3?n9Lt3uYcE+DIlH^^&Za)pJ~+dlBAo~h`uYBUz3l+J5ksdu*ns%BsXp2Lb@W}14f zP%{WfSh@!}-7H!W9TfYEI9Tg=rvmZ{dj`nHL_s{~US2KY_i=e!7m}@5bsgs{*1bI~ z2e;E;y_H>&)xw+uCej7Yvx377JaM)LJhEoKD3dp_@q)rh@iMee;x6)@qocIp+QV&_hLkH~tundfx2yETby7DPcJa}U<#eyIND z2LkQnf(e4nZ}k>bu_vm!n%eO%ii0vNce3j35wXnj3-&)IgD;s`oSghjo1|8M$d!Ox zSG@~yP7zUml}cak>BpY&bpafQR2wG+k6=7+NPK>{I56CKuI7}UvO{Am<)VI2HTwwR zot7m|J9d%ZAwCx{56Q33?fI)pSm`$2!+y&utfJ6_1+o>>;amufnPZcvnS4l-fP)F) z0^hAW9fQ1C9O6^TIHdMMK=e%I6z4Bcpted)Hu=EDGoP=s&Xx_Icb0!6L~_67>JkBy ztNvHjFs-FZD#J!dVQ%~ny6MKlcy^{ZbP!$CED;j}UpgSKrqw8|RZ zY><^nDStoasDeJN5XXiXtpRd=BB0H>*{um{gYUQ-5Q=!AO3jym?>Fg+^{*C<7g@WBRm z)%K6l8eZygN9TlvOf$13Wun*jcB7?#ByTTiLf`;LM0**Yx`|LzV-L4 zaD>*uc?mT<#<8xB?Nu+39C(e|m}JB;MNT9=NS>eLxbb7M%!esdy;l@LAd$_J!7f|# zL9oS6$(VuD9rxV>aCNG>p5Dt%rE7Tie$M(9m}=RKR~jwD=2(BMl|OM)yglp zg8}})je=dyG1|ePYs+F@Sin!7r3%eSq6NjTYax}CAHE=BBtyUICb;v<*^7tf&Z|e# zr-#}WMvGJWv|^h%VPq$e_NxSMzblQD`|r*W+$9lrcZNU;-yLiVHo^cL^SGMKYcVqg z(GoOM!9gau&nJ=>dtDY9BnUP_v|FlrntfSdOv0-zzU;y8VrD z3Ya^<q0H0rVNvVJf2I0Fo&Q8ZBPPSc(Vu56+0aP<)gz#(*Fk8s=y%TQVs#+} zD8rGk?RJP?L2LG>sGGP!WTS4YT%W64FmhjzI@O58ez9>UFfC%HPaom!y37To0YU&J zJQ&ojPN4>b$8+bo{64Ka7^{0Gf+ml_m)0UVfesa8$hLjZL`emMQxhSZ_UI!Ii@3P@ zzcwhp(F)>fI@$wd*KL)e&vQOx-!Xcnuj?jNaDH7hFltj6d~w3$8pV(F@C-R#bZFTz z?dj#Nn(2T##(jVP0~&>e^UQ%9rN#pQ#8m$}4gw)CA^(fS247PH1;E+#K(rP?8lW-+ z+#hhY?qAR;IF$}40{I784HlpW(nA7}0m+-lEZ_`!AS;j$JV6gMf%t|FW@P}{LD*n` za~Xh&5KCC#YX+b;WFj^o*#UP~e`5sl|AnA}*BOBXkjXfJWG7roupm3| z1H=U(xSJVB2)X|!ku)C*Y{>*90*kN%(ZDH8Kw^j@TyPT;&>pgm43Hd8P6t+E{v+{# z1?@D$6REH4(| zACM#4KQ+~`0GY`C;tu{7kw6Z>{VxikFc6XbZ%Z-%fDcGHpaHk;~=NBpLH!F~r><^GZ_kSHcDgTc& zd+C3Q9Octe`cG78^}WPXKnuPT88^S9bH5OgSdgj_ Date: Sun, 5 Apr 2026 05:27:07 +0200 Subject: [PATCH 24/25] docs: updating pages with the new feature --- README.md | 1 + docs/index.md | 1 + docs/spiders/advanced.md | 2 ++ docs/spiders/architecture.md | 3 ++- docs/spiders/getting-started.md | 25 +++++++++++++++++++++++++ 5 files changed, 31 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index d577a00..d6b6e41 100644 --- a/README.md +++ b/README.md @@ -214,6 +214,7 @@ MySpider().start() - 💾 **Pause & Resume**: Checkpoint-based crawl persistence. Press Ctrl+C for a graceful shutdown; restart to resume from where you left off. - 📡 **Streaming Mode**: Stream scraped items as they arrive via `async for item in spider.stream()` with real-time stats - ideal for UI, pipelines, and long-running crawls. - 🛡️ **Blocked Request Detection**: Automatic detection and retry of blocked requests with customizable logic. +- 🤖 **Robots.txt Compliance**: Optional `robots_txt_obey` flag that respects `Disallow`, `Crawl-delay`, and `Request-rate` directives with per-domain caching. - 📦 **Built-in Export**: Export results through hooks and your own pipeline or the built-in JSON/JSONL with `result.items.to_json()` / `result.items.to_jsonl()` respectively. ### Advanced Websites Fetching with Session Support diff --git a/docs/index.md b/docs/index.md index 01d9921..9ff2d04 100644 --- a/docs/index.md +++ b/docs/index.md @@ -99,6 +99,7 @@ MySpider().start() - 💾 **Pause & Resume**: Checkpoint-based crawl persistence. Press Ctrl+C for a graceful shutdown; restart to resume from where you left off. - 📡 **Streaming Mode**: Stream scraped items as they arrive via `async for item in spider.stream()` with real-time stats - ideal for UI, pipelines, and long-running crawls. - 🛡️ **Blocked Request Detection**: Automatic detection and retry of blocked requests with customizable logic. +- 🤖 **Robots.txt Compliance**: Optional `robots_txt_obey` flag that respects `Disallow`, `Crawl-delay`, and `Request-rate` directives with per-domain caching. - 📦 **Built-in Export**: Export results through hooks and your own pipeline or the built-in JSON/JSONL with `result.items.to_json()` / `result.items.to_jsonl()` respectively. ### Advanced Websites Fetching with Session Support diff --git a/docs/spiders/advanced.md b/docs/spiders/advanced.md index c35c2b1..9b1f5b9 100644 --- a/docs/spiders/advanced.md +++ b/docs/spiders/advanced.md @@ -17,6 +17,7 @@ The spider system uses three class attributes to control how aggressively it cra | `concurrent_requests` | `4` | Maximum number of requests being processed at the same time | | `concurrent_requests_per_domain` | `0` | Maximum concurrent requests per domain (0 = no per-domain limit) | | `download_delay` | `0.0` | Seconds to wait before each request | +| `robots_txt_obey` | `False` | Respect robots.txt rules (Disallow, Crawl-delay, Request-rate) | ```python class PoliteSpider(Spider): @@ -234,6 +235,7 @@ print(f"Requests: {stats.requests_count}") print(f"Failed: {stats.failed_requests_count}") print(f"Blocked: {stats.blocked_requests_count}") print(f"Offsite filtered: {stats.offsite_requests_count}") +print(f"Robots.txt disallowed: {stats.robots_disallowed_count}") print(f"Items scraped: {stats.items_scraped}") print(f"Items dropped: {stats.items_dropped}") print(f"Response bytes: {stats.response_bytes}") diff --git a/docs/spiders/architecture.md b/docs/spiders/architecture.md index 4ccfad2..82d388f 100644 --- a/docs/spiders/architecture.md +++ b/docs/spiders/architecture.md @@ -19,7 +19,7 @@ Here's what happens step by step when you run a spider without many details: 1. The **Spider** produces the first batch of `Request` objects. By default, it creates one request for each URL in `start_urls`, but you can override `start_requests()` for custom logic. 2. The **Scheduler** receives requests and places them in a priority queue, and creates fingerprints for them. Higher-priority requests are dequeued first. -3. The **Crawler Engine** asks the **Scheduler** to dequeue the next request, respecting concurrency limits (global and per-domain) and download delays. Once the **Crawler Engine** receives the request, it passes it to the **Session Manager**, which routes it to the correct session based on the request's `sid` (session ID). +3. The **Crawler Engine** asks the **Scheduler** to dequeue the next request, respecting concurrency limits (global and per-domain) and download delays. If `robots_txt_obey` is enabled, the engine checks the domain's robots.txt rules before proceeding -- disallowed requests are dropped silently. Once the **Crawler Engine** receives the request, it passes it to the **Session Manager**, which routes it to the correct session based on the request's `sid` (session ID). 4. The **session** fetches the page and returns a [Response](../fetching/choosing.md#response-object) object to the **Crawler Engine**. The engine records statistics and checks for blocked responses. If the response is blocked, the engine retries the request up to `max_blocked_retries` times. Of course, the blocking detection and the retry logic for blocked requests can be customized. 5. The **Crawler Engine** passes the [Response](../fetching/choosing.md#response-object) to the request's callback. The callback either yields a dictionary, which gets treated as a scraped item, or a follow-up request, which gets sent to the scheduler for queuing. 6. The cycle repeats from step 2 until the scheduler is empty and no tasks are active, or the spider is paused. @@ -91,6 +91,7 @@ If you're coming from Scrapy, here's how Scrapling's spider system maps: | Blocked detection | Through custom middlewares | Built-in `is_blocked()` + `retry_blocked_request()` hooks | | Concurrency | `CONCURRENT_REQUESTS` setting | `concurrent_requests` class attribute | | Domain filtering | `allowed_domains` | `allowed_domains` | +| Robots.txt | `ROBOTSTXT_OBEY` setting | `robots_txt_obey` class attribute | | Pause/Resume | `JOBDIR` setting | `crawldir` constructor argument | | Export | Feed exports | `result.items.to_json()` / `to_jsonl()` or custom through hooks | | Running | `scrapy crawl spider_name` | `MySpider().start()` | diff --git a/docs/spiders/getting-started.md b/docs/spiders/getting-started.md index e619268..9847c98 100644 --- a/docs/spiders/getting-started.md +++ b/docs/spiders/getting-started.md @@ -149,6 +149,31 @@ Subdomains are matched automatically, so setting `allowed_domains = {"example.co When a request is filtered out, it's counted in `stats.offsite_requests_count` so you can see how many were dropped. +## Robots.txt Compliance + +Set `robots_txt_obey = True` to make the spider respect robots.txt rules before crawling any domain: + +```python +class PoliteSpider(Spider): + name = "polite" + start_urls = ["https://example.com"] + robots_txt_obey = True + + async def parse(self, response: Response): + for link in response.css("a::attr(href)").getall(): + yield response.follow(link, callback=self.parse) +``` + +When enabled, the spider will: + +1. **Pre-fetch robots.txt** for all domains in `start_urls` before the crawl begins (concurrently). +2. **Check every request** against the domain's robots.txt `Disallow` rules. Disallowed requests are silently dropped and counted in `stats.robots_disallowed_count`. +3. **Respect `Crawl-delay` and `Request-rate` directives** by taking the maximum of the directive and your configured `download_delay`. This means robots.txt delays never reduce your configured delay, only increase it when needed. + +Robots.txt files are fetched using the spider's default session and cached per domain for the entire crawl. Domains discovered mid-crawl (not in `start_urls`) have their robots.txt fetched on the first request to that domain. + +**Note:** `robots_txt_obey` is turned off by default to avoid surprising behavior. If you enable it, it does not affect your concurrency settings (`concurrent_requests`, `concurrent_requests_per_domain`) -- only the delay between requests is adjusted. + ## What's Next Now that you have the basics, you can explore: From 087634ab67883edbb3d80a4f6a949c5c525a1e94 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Sun, 5 Apr 2026 05:28:57 +0200 Subject: [PATCH 25/25] docs: updated the translated files --- docs/README_AR.md | 1 + docs/README_CN.md | 1 + docs/README_DE.md | 1 + docs/README_ES.md | 1 + docs/README_FR.md | 1 + docs/README_JP.md | 1 + docs/README_KR.md | 1 + docs/README_RU.md | 1 + 8 files changed, 8 insertions(+) diff --git a/docs/README_AR.md b/docs/README_AR.md index 01b6d4b..7cc7193 100644 --- a/docs/README_AR.md +++ b/docs/README_AR.md @@ -209,6 +209,7 @@ MySpider().start() - 💾 **إيقاف واستئناف**: استمرارية الزحف القائمة على Checkpoint. اضغط Ctrl+C للإيقاف بسلاسة؛ أعد التشغيل للاستئناف من حيث توقفت. - 📡 **وضع Streaming**: بث العناصر المستخرجة فور وصولها عبر `async for item in spider.stream()` مع إحصائيات فورية - مثالي لواجهات المستخدم وخطوط الأنابيب وعمليات الزحف الطويلة. - 🛡️ **كشف الطلبات المحظورة**: كشف تلقائي وإعادة محاولة للطلبات المحظورة مع منطق قابل للتخصيص. +- 🤖 **الامتثال لـ robots.txt**: خيار `robots_txt_obey` الاختياري الذي يحترم توجيهات `Disallow` و `Crawl-delay` و `Request-rate` مع التخزين المؤقت لكل نطاق. - 📦 **تصدير مدمج**: صدّر النتائج عبر الخطافات وخط الأنابيب الخاص بك أو JSON/JSONL المدمج مع `result.items.to_json()` / `result.items.to_jsonl()` على التوالي. ### جلب متقدم للمواقع مع دعم الجلسات diff --git a/docs/README_CN.md b/docs/README_CN.md index 61cd255..bfc0b6f 100644 --- a/docs/README_CN.md +++ b/docs/README_CN.md @@ -209,6 +209,7 @@ MySpider().start() - 💾 **暂停与恢复**：基于 Checkpoint 的爬取持久化。按 Ctrl+C 优雅关闭；重启后从上次停止的地方继续。 - 📡 **Streaming 模式**：通过 `async for item in spider.stream()` 以实时统计 Streaming 抓取的数据--非常适合 UI、管道和长时间运行的爬取。 - 🛡️ **被阻止请求检测**：自动检测并重试被阻止的请求，支持自定义逻辑。 +- 🤖 **robots.txt 合规**：可选的 `robots_txt_obey` 标志，支持 `Disallow`、`Crawl-delay` 和 `Request-rate` 指令，并按域名缓存。 - 📦 **内置导出**：通过钩子和您自己的管道导出结果，或使用内置的 JSON/JSONL，分别通过 `result.items.to_json()`/`result.items.to_jsonl()`。 ### 支持 Session 的高级网站获取 diff --git a/docs/README_DE.md b/docs/README_DE.md index 1548260..fbbd7e9 100644 --- a/docs/README_DE.md +++ b/docs/README_DE.md @@ -209,6 +209,7 @@ MySpider().start() - 💾 **Pause & Resume**: Checkpoint-basierte Crawl-Persistenz. Drücken Sie Strg+C für ein kontrolliertes Herunterfahren; starten Sie neu, um dort fortzufahren, wo Sie aufgehört haben. - 📡 **Streaming-Modus**: Gescrapte Elemente in Echtzeit streamen über `async for item in spider.stream()` mit Echtzeit-Statistiken -- ideal für UI, Pipelines und lang laufende Crawls. - 🛡️ **Erkennung blockierter Anfragen**: Automatische Erkennung und Wiederholung blockierter Anfragen mit anpassbarer Logik. +- 🤖 **robots.txt-Konformität**: Optionales `robots_txt_obey`-Flag, das `Disallow`-, `Crawl-delay`- und `Request-rate`-Direktiven mit domainbasiertem Caching respektiert. - 📦 **Integrierter Export**: Ergebnisse über Hooks und Ihre eigene Pipeline oder den integrierten JSON/JSONL-Export mit `result.items.to_json()` / `result.items.to_jsonl()` exportieren. ### Erweitertes Website-Abrufen mit Session-Unterstützung diff --git a/docs/README_ES.md b/docs/README_ES.md index 746da96..fba23a0 100644 --- a/docs/README_ES.md +++ b/docs/README_ES.md @@ -209,6 +209,7 @@ MySpider().start() - 💾 **Pause & Resume**: Persistencia de rastreo basada en Checkpoint. Presiona Ctrl+C para un cierre ordenado; reinicia para continuar desde donde lo dejaste. - 📡 **Modo Streaming**: Transmite elementos extraídos a medida que llegan con `async for item in spider.stream()` con estadísticas en tiempo real - ideal para UI, pipelines y rastreos de larga duración. - 🛡️ **Detección de Solicitudes Bloqueadas**: Detección automática y reintento de solicitudes bloqueadas con lógica personalizable. +- 🤖 **Cumplimiento de robots.txt**: Flag opcional `robots_txt_obey` que respeta las directivas `Disallow`, `Crawl-delay` y `Request-rate` con caché por dominio. - 📦 **Exportación Integrada**: Exporta resultados a través de hooks y tu propio pipeline o el JSON/JSONL integrado con `result.items.to_json()` / `result.items.to_jsonl()` respectivamente. ### Obtención Avanzada de Sitios Web con Soporte de Session diff --git a/docs/README_FR.md b/docs/README_FR.md index ec2fd97..d001bbb 100644 --- a/docs/README_FR.md +++ b/docs/README_FR.md @@ -209,6 +209,7 @@ MySpider().start() - 💾 **Pause & Reprise** : Persistance du crawl basée sur des checkpoints. Appuyez sur Ctrl+C pour un arrêt gracieux ; redémarrez pour reprendre là où vous vous étiez arrêté. - 📡 **Mode streaming** : Diffusez les éléments scrapés en temps réel via `async for item in spider.stream()` avec des statistiques en temps réel - idéal pour les UI, pipelines et crawls de longue durée. - 🛡️ **Détection des requêtes bloquées** : Détection automatique et réessai des requêtes bloquées avec une logique personnalisable. +- 🤖 **Conformité robots.txt** : Flag optionnel `robots_txt_obey` qui respecte les directives `Disallow`, `Crawl-delay` et `Request-rate` avec mise en cache par domaine. - 📦 **Export intégré** : Exportez les résultats via des hooks et votre propre pipeline ou l'export JSON/JSONL intégré avec `result.items.to_json()` / `result.items.to_jsonl()` respectivement. ### Récupération avancée de sites web avec support de sessions diff --git a/docs/README_JP.md b/docs/README_JP.md index 7b6632a..f5c6d37 100644 --- a/docs/README_JP.md +++ b/docs/README_JP.md @@ -209,6 +209,7 @@ MySpider().start() - 💾 **Pause & Resume**：Checkpoint ベースのクロール永続化。Ctrl+C で正常にシャットダウン；再起動すると中断したところから再開。 - 📡 **Streaming モード**：`async for item in spider.stream()` でリアルタイム統計とともにスクレイプされたアイテムを Streaming で受信 - UI、パイプライン、長時間実行クロールに最適。 - 🛡️ **ブロックされたリクエストの検出**：カスタマイズ可能なロジックによるブロックされたリクエストの自動検出とリトライ。 +- 🤖 **robots.txt 準拠**：オプションの `robots_txt_obey` フラグで `Disallow`、`Crawl-delay`、`Request-rate` ディレクティブをドメインごとのキャッシュで遵守。 - 📦 **組み込みエクスポート**：フックや独自のパイプライン、または組み込みの JSON/JSONL で結果をエクスポート。それぞれ`result.items.to_json()` / `result.items.to_jsonl()`を使用。 ### Session サポート付き高度なウェブサイト取得 diff --git a/docs/README_KR.md b/docs/README_KR.md index 1e53fe6..020a3ef 100644 --- a/docs/README_KR.md +++ b/docs/README_KR.md @@ -209,6 +209,7 @@ MySpider().start() - 💾 **일시정지 & 재개**: 체크포인트 기반의 크롤링 영속화. Ctrl+C로 정상 종료하고, 재시작하면 중단된 지점부터 이어갑니다. - 📡 **스트리밍 모드**: `async for item in spider.stream()`으로 스크레이핑된 아이템을 실시간 통계와 함께 스트리밍으로 수신 - UI, 파이프라인, 장시간 크롤링에 적합합니다. - 🛡️ **차단된 요청 감지**: 커스텀 로직을 통한 차단된 요청의 자동 감지 및 재시도를 지원합니다. +- 🤖 **robots.txt 준수**: 선택적 `robots_txt_obey` 플래그로 `Disallow`, `Crawl-delay`, `Request-rate` 지시문을 도메인별 캐싱과 함께 준수합니다. - 📦 **내장 내보내기**: 훅이나 자체 파이프라인, 또는 내장 JSON/JSONL로 결과를 내보냅니다. 각각 `result.items.to_json()` / `result.items.to_jsonl()`을 사용합니다. ### 세션을 지원하는 고급 웹사이트 가져오기 diff --git a/docs/README_RU.md b/docs/README_RU.md index 7233bd5..fdef497 100644 --- a/docs/README_RU.md +++ b/docs/README_RU.md @@ -212,6 +212,7 @@ MySpider().start() - 💾 **Pause & Resume**: Persistence обхода на основе Checkpoint'ов. Нажмите Ctrl+C для мягкой остановки; перезапустите, чтобы продолжить с того места, где вы остановились. - 📡 **Режим Streaming**: Стримьте извлечённые элементы по мере их поступления через `async for item in spider.stream()` со статистикой в реальном времени - идеально для UI, конвейеров и длительных обходов. - 🛡️ **Обнаружение заблокированных запросов**: Автоматическое обнаружение и повторная отправка заблокированных запросов с настраиваемой логикой. +- 🤖 **Соответствие robots.txt**: Опциональный флаг `robots_txt_obey`, который учитывает директивы `Disallow`, `Crawl-delay` и `Request-rate` с кэшированием по доменам. - 📦 **Встроенный экспорт**: Экспортируйте результаты через хуки и собственный конвейер или встроенный JSON/JSONL с `result.items.to_json()` / `result.items.to_jsonl()` соответственно. ### Продвинутая загрузка сайтов с поддержкой Session