From a86e9709ea0442fcf7b1c9b0459ac0716514d8a6 Mon Sep 17 00:00:00 2001 From: Abdullah <52079299+AbdullahY36@users.noreply.github.com> Date: Sat, 4 Apr 2026 03:00:15 +0200 Subject: [PATCH] feat(spiders): pre-warm robots.txt cache before crawl loop starts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previously robots.txt was fetched lazily on the first request per domain, causing early concurrent requests to each stall waiting for the same network fetch. The cache is now warmed before the crawl loop starts, making all subsequent robots.txt lookups a local read. - RobotsTxtManager gains a prefetch(urls, sid) method that fetches all domains concurrently via a task group - CrawlerEngine._prefetch_robots_txt() is called after on_start(): uses allowed_domains if configured, otherwise falls back to unique domains extracted from start_urls - Mid-crawl domain discovery (not covered by prefetch) still fetches lazily; two concurrent callbacks on the same new domain can each trigger a fetch — accepted tradeoff, documented in _get_domain_delay Files: scrapling/spiders/robotstxt.py, scrapling/spiders/engine.py, tests/spiders/test_engine.py --- scrapling/spiders/engine.py | 28 +++++++++++- tests/spiders/test_engine.py | 86 +++++++++++++++++++++++++++++++++++- 2 files changed, 112 insertions(+), 2 deletions(-) diff --git a/scrapling/spiders/engine.py b/scrapling/spiders/engine.py index cf1715e..163101b 100644 --- a/scrapling/spiders/engine.py +++ b/scrapling/spiders/engine.py @@ -97,7 +97,10 @@ class CrawlerEngine: if domain in self._domain_delays: return self._domain_delays[domain] - # Fetch both robots.txt directives in a single parser lookup + # For domains covered by _prefetch_robots_txt this is a local parser read. + # Domains discovered mid-crawl (not in start_urls/allowed_domains) will fetch here. + # Two concurrent callbacks hitting the same new domain can each trigger a fetch; + # the second write is a no-op in effect (same content), but the extra request is accepted. c_delay, r_rate = await robots_manager._get_delay_directives(request.url, request.sid) delay = self.spider.download_delay @@ -287,6 +290,27 @@ class CrawlerEngine: return True + async def _prefetch_robots_txt(self) -> None: + """Pre-warm the robots.txt cache before the crawl loop starts. + + Uses allowed_domains if configured, otherwise falls back to unique domains + extracted from start_urls via Request.domain. Both paths use https. + """ + if not self._robots_manager: + return + + if self._allowed_domains: + domains = self._allowed_domains + elif self.spider.start_urls: + # Deduplicate by domain so we spawn exactly one task per domain + domains = {Request(url).domain for url in self.spider.start_urls} + else: + return + + seed_urls = [f"https://{domain}/" for domain in domains] + + await self._robots_manager.prefetch(seed_urls, self.session_manager.default_session_id) + async def crawl(self) -> CrawlStats: """Run the spider and return CrawlStats.""" self._running = True @@ -310,6 +334,8 @@ class CrawlerEngine: await self.spider.on_start(resuming=resuming) + await self._prefetch_robots_txt() + try: if not resuming: async for request in self.spider.start_requests(): diff --git a/tests/spiders/test_engine.py b/tests/spiders/test_engine.py index e362036..e7382f8 100644 --- a/tests/spiders/test_engine.py +++ b/tests/spiders/test_engine.py @@ -8,6 +8,7 @@ import pytest from scrapling.spiders.engine import CrawlerEngine, _dump from scrapling.spiders.request import Request +from scrapling.spiders.robotstxt import RobotsTxtManager from scrapling.spiders.session import SessionManager from scrapling.spiders.result import CrawlStats, ItemList from scrapling.spiders.checkpoint import CheckpointData @@ -22,10 +23,11 @@ from scrapling.core._types import Any, Dict, Set, AsyncGenerator class MockResponse: """Minimal Response stand-in.""" - def __init__(self, status: int = 200, body: bytes = b"ok", url: str = "https://example.com"): + def __init__(self, status: int = 200, body: bytes = b"ok", url: str = "https://example.com", encoding: str = "utf-8"): self.status = status self.body = body self.url = url + self.encoding = encoding self.request: Any = None self.meta: Dict[str, Any] = {} @@ -84,6 +86,7 @@ class MockSpider: on_scraped_item_fn=None, retry_blocked_request_fn=None, robots_txt_obey: bool = False, + start_urls: list[str] | None = None, ): self.concurrent_requests = concurrent_requests self.concurrent_requests_per_domain = concurrent_requests_per_domain @@ -95,6 +98,7 @@ class MockSpider: self.fp_keep_fragments = fp_keep_fragments self.name = "test_spider" self.robots_txt_obey = robots_txt_obey + self.start_urls = start_urls or [] # Tracking lists self.on_start_calls: list[dict] = [] @@ -914,3 +918,83 @@ class TestPauseDuringCrawl: await engine.crawl() assert engine.paused is False + + +# --------------------------------------------------------------------------- +# Tests: _prefetch_robots_txt +# --------------------------------------------------------------------------- + + +class TestPrefetchRobotsTxt: + """_prefetch_robots_txt warms the robots.txt cache before the crawl loop.""" + + @staticmethod + def _make_counting_fetch(): + """Return (fetch_fn, calls_list) where calls_list records every (url, sid) pair.""" + calls: list[tuple[str, str]] = [] + + async def _fetch(url: str, sid: str): + calls.append((url, sid)) + return MockResponse(status=200, body=b"", url=url) + + return _fetch, calls + + @pytest.mark.asyncio + async def test_prefetch_uses_allowed_domains_when_set(self): + fetch_fn, calls = self._make_counting_fetch() + spider = MockSpider(allowed_domains={"a.com", "b.com"}, robots_txt_obey=True) + engine = _make_engine(spider=spider) + engine._robots_manager = RobotsTxtManager(fetch_fn) + + await engine._prefetch_robots_txt() + + fetched_domains = {Request(url).domain for url, _ in calls} + assert fetched_domains == {"a.com", "b.com"} + + @pytest.mark.asyncio + async def test_prefetch_falls_back_to_start_urls_when_no_allowed_domains(self): + fetch_fn, calls = self._make_counting_fetch() + spider = MockSpider(robots_txt_obey=True, start_urls=["https://example.com/page1"]) + engine = _make_engine(spider=spider) + engine._robots_manager = RobotsTxtManager(fetch_fn) + + await engine._prefetch_robots_txt() + + assert len(calls) == 1 + assert calls[0][0] == "https://example.com/robots.txt" + + @pytest.mark.asyncio + async def test_prefetch_noop_when_robots_disabled(self): + fetch_fn, calls = self._make_counting_fetch() + spider = MockSpider(robots_txt_obey=False) + engine = _make_engine(spider=spider) + + assert engine._robots_manager is None + + await engine._prefetch_robots_txt() + + assert calls == [] + + @pytest.mark.asyncio + async def test_prefetch_noop_when_start_urls_empty(self): + fetch_fn, calls = self._make_counting_fetch() + spider = MockSpider(robots_txt_obey=True, start_urls=[]) + engine = _make_engine(spider=spider) + engine._robots_manager = RobotsTxtManager(fetch_fn) + + await engine._prefetch_robots_txt() + + assert calls == [] + + @pytest.mark.asyncio + async def test_prefetch_deduplicates_same_domain_in_start_urls(self): + fetch_fn, calls = self._make_counting_fetch() + spider = MockSpider(robots_txt_obey=True, start_urls=["https://example.com/a", "https://example.com/b"]) + engine = _make_engine(spider=spider) + engine._robots_manager = RobotsTxtManager(fetch_fn) + + await engine._prefetch_robots_txt() + + # set of Request.domain values deduplicates to one task per domain + assert len(calls) == 1 + assert calls[0][0] == "https://example.com/robots.txt"