diff --git a/scrapling/spiders/engine.py b/scrapling/spiders/engine.py index cf1715e..163101b 100644 --- a/scrapling/spiders/engine.py +++ b/scrapling/spiders/engine.py @@ -97,7 +97,10 @@ class CrawlerEngine: if domain in self._domain_delays: return self._domain_delays[domain] - # Fetch both robots.txt directives in a single parser lookup + # For domains covered by _prefetch_robots_txt this is a local parser read. + # Domains discovered mid-crawl (not in start_urls/allowed_domains) will fetch here. + # Two concurrent callbacks hitting the same new domain can each trigger a fetch; + # the second write is a no-op in effect (same content), but the extra request is accepted. c_delay, r_rate = await robots_manager._get_delay_directives(request.url, request.sid) delay = self.spider.download_delay @@ -287,6 +290,27 @@ class CrawlerEngine: return True + async def _prefetch_robots_txt(self) -> None: + """Pre-warm the robots.txt cache before the crawl loop starts. + + Uses allowed_domains if configured, otherwise falls back to unique domains + extracted from start_urls via Request.domain. Both paths use https. + """ + if not self._robots_manager: + return + + if self._allowed_domains: + domains = self._allowed_domains + elif self.spider.start_urls: + # Deduplicate by domain so we spawn exactly one task per domain + domains = {Request(url).domain for url in self.spider.start_urls} + else: + return + + seed_urls = [f"https://{domain}/" for domain in domains] + + await self._robots_manager.prefetch(seed_urls, self.session_manager.default_session_id) + async def crawl(self) -> CrawlStats: """Run the spider and return CrawlStats.""" self._running = True @@ -310,6 +334,8 @@ class CrawlerEngine: await self.spider.on_start(resuming=resuming) + await self._prefetch_robots_txt() + try: if not resuming: async for request in self.spider.start_requests(): diff --git a/tests/spiders/test_engine.py b/tests/spiders/test_engine.py index e362036..e7382f8 100644 --- a/tests/spiders/test_engine.py +++ b/tests/spiders/test_engine.py @@ -8,6 +8,7 @@ import pytest from scrapling.spiders.engine import CrawlerEngine, _dump from scrapling.spiders.request import Request +from scrapling.spiders.robotstxt import RobotsTxtManager from scrapling.spiders.session import SessionManager from scrapling.spiders.result import CrawlStats, ItemList from scrapling.spiders.checkpoint import CheckpointData @@ -22,10 +23,11 @@ from scrapling.core._types import Any, Dict, Set, AsyncGenerator class MockResponse: """Minimal Response stand-in.""" - def __init__(self, status: int = 200, body: bytes = b"ok", url: str = "https://example.com"): + def __init__(self, status: int = 200, body: bytes = b"ok", url: str = "https://example.com", encoding: str = "utf-8"): self.status = status self.body = body self.url = url + self.encoding = encoding self.request: Any = None self.meta: Dict[str, Any] = {} @@ -84,6 +86,7 @@ class MockSpider: on_scraped_item_fn=None, retry_blocked_request_fn=None, robots_txt_obey: bool = False, + start_urls: list[str] | None = None, ): self.concurrent_requests = concurrent_requests self.concurrent_requests_per_domain = concurrent_requests_per_domain @@ -95,6 +98,7 @@ class MockSpider: self.fp_keep_fragments = fp_keep_fragments self.name = "test_spider" self.robots_txt_obey = robots_txt_obey + self.start_urls = start_urls or [] # Tracking lists self.on_start_calls: list[dict] = [] @@ -914,3 +918,83 @@ class TestPauseDuringCrawl: await engine.crawl() assert engine.paused is False + + +# --------------------------------------------------------------------------- +# Tests: _prefetch_robots_txt +# --------------------------------------------------------------------------- + + +class TestPrefetchRobotsTxt: + """_prefetch_robots_txt warms the robots.txt cache before the crawl loop.""" + + @staticmethod + def _make_counting_fetch(): + """Return (fetch_fn, calls_list) where calls_list records every (url, sid) pair.""" + calls: list[tuple[str, str]] = [] + + async def _fetch(url: str, sid: str): + calls.append((url, sid)) + return MockResponse(status=200, body=b"", url=url) + + return _fetch, calls + + @pytest.mark.asyncio + async def test_prefetch_uses_allowed_domains_when_set(self): + fetch_fn, calls = self._make_counting_fetch() + spider = MockSpider(allowed_domains={"a.com", "b.com"}, robots_txt_obey=True) + engine = _make_engine(spider=spider) + engine._robots_manager = RobotsTxtManager(fetch_fn) + + await engine._prefetch_robots_txt() + + fetched_domains = {Request(url).domain for url, _ in calls} + assert fetched_domains == {"a.com", "b.com"} + + @pytest.mark.asyncio + async def test_prefetch_falls_back_to_start_urls_when_no_allowed_domains(self): + fetch_fn, calls = self._make_counting_fetch() + spider = MockSpider(robots_txt_obey=True, start_urls=["https://example.com/page1"]) + engine = _make_engine(spider=spider) + engine._robots_manager = RobotsTxtManager(fetch_fn) + + await engine._prefetch_robots_txt() + + assert len(calls) == 1 + assert calls[0][0] == "https://example.com/robots.txt" + + @pytest.mark.asyncio + async def test_prefetch_noop_when_robots_disabled(self): + fetch_fn, calls = self._make_counting_fetch() + spider = MockSpider(robots_txt_obey=False) + engine = _make_engine(spider=spider) + + assert engine._robots_manager is None + + await engine._prefetch_robots_txt() + + assert calls == [] + + @pytest.mark.asyncio + async def test_prefetch_noop_when_start_urls_empty(self): + fetch_fn, calls = self._make_counting_fetch() + spider = MockSpider(robots_txt_obey=True, start_urls=[]) + engine = _make_engine(spider=spider) + engine._robots_manager = RobotsTxtManager(fetch_fn) + + await engine._prefetch_robots_txt() + + assert calls == [] + + @pytest.mark.asyncio + async def test_prefetch_deduplicates_same_domain_in_start_urls(self): + fetch_fn, calls = self._make_counting_fetch() + spider = MockSpider(robots_txt_obey=True, start_urls=["https://example.com/a", "https://example.com/b"]) + engine = _make_engine(spider=spider) + engine._robots_manager = RobotsTxtManager(fetch_fn) + + await engine._prefetch_robots_txt() + + # set of Request.domain values deduplicates to one task per domain + assert len(calls) == 1 + assert calls[0][0] == "https://example.com/robots.txt"