feat(spiders): pre-warm robots.txt cache before crawl loop starts

Previously robots.txt was fetched lazily on the first request per
domain, causing early concurrent requests to each stall waiting for
the same network fetch. The cache is now warmed before the crawl loop
starts, making all subsequent robots.txt lookups a local read.

- RobotsTxtManager gains a prefetch(urls, sid) method that fetches all domains concurrently via a task group
- CrawlerEngine._prefetch_robots_txt() is called after on_start():
  uses allowed_domains if configured, otherwise falls back to unique
  domains extracted from start_urls
- Mid-crawl domain discovery (not covered by prefetch) still fetches
  lazily; two concurrent callbacks on the same new domain can each
  trigger a fetch — accepted tradeoff, documented in _get_domain_delay

Files: scrapling/spiders/robotstxt.py, scrapling/spiders/engine.py, tests/spiders/test_engine.py

scrapling/spiders/engine.py

@@ -97,7 +97,10 @@ class CrawlerEngine:
         if domain in self._domain_delays:
             return self._domain_delays[domain]
 
-        # Fetch both robots.txt directives in a single parser lookup
+        # For domains covered by _prefetch_robots_txt this is a local parser read.
+        # Domains discovered mid-crawl (not in start_urls/allowed_domains) will fetch here.
+        # Two concurrent callbacks hitting the same new domain can each trigger a fetch;
+        # the second write is a no-op in effect (same content), but the extra request is accepted.
         c_delay, r_rate = await robots_manager._get_delay_directives(request.url, request.sid)
 
         delay = self.spider.download_delay
@@ -287,6 +290,27 @@ class CrawlerEngine:
 
         return True
 
+    async def _prefetch_robots_txt(self) -> None:
+        """Pre-warm the robots.txt cache before the crawl loop starts.
+
+        Uses allowed_domains if configured, otherwise falls back to unique domains
+        extracted from start_urls via Request.domain. Both paths use https.
+        """
+        if not self._robots_manager:
+            return
+
+        if self._allowed_domains:
+            domains = self._allowed_domains
+        elif self.spider.start_urls:
+            # Deduplicate by domain so we spawn exactly one task per domain
+            domains = {Request(url).domain for url in self.spider.start_urls}
+        else:
+            return
+
+        seed_urls = [f"https://{domain}/" for domain in domains]
+
+        await self._robots_manager.prefetch(seed_urls, self.session_manager.default_session_id)
+
     async def crawl(self) -> CrawlStats:
         """Run the spider and return CrawlStats."""
         self._running = True
@@ -310,6 +334,8 @@ class CrawlerEngine:
 
             await self.spider.on_start(resuming=resuming)
 
+            await self._prefetch_robots_txt()
+
             try:
                 if not resuming:
                     async for request in self.spider.start_requests():

tests/spiders/test_engine.py

@@ -8,6 +8,7 @@ import pytest
 
 from scrapling.spiders.engine import CrawlerEngine, _dump
 from scrapling.spiders.request import Request
+from scrapling.spiders.robotstxt import RobotsTxtManager
 from scrapling.spiders.session import SessionManager
 from scrapling.spiders.result import CrawlStats, ItemList
 from scrapling.spiders.checkpoint import CheckpointData
@@ -22,10 +23,11 @@ from scrapling.core._types import Any, Dict, Set, AsyncGenerator
 class MockResponse:
     """Minimal Response stand-in."""
 
-    def __init__(self, status: int = 200, body: bytes = b"ok", url: str = "https://example.com"):
+    def __init__(self, status: int = 200, body: bytes = b"ok", url: str = "https://example.com", encoding: str = "utf-8"):
         self.status = status
         self.body = body
         self.url = url
+        self.encoding = encoding
         self.request: Any = None
         self.meta: Dict[str, Any] = {}
 
@@ -84,6 +86,7 @@ class MockSpider:
         on_scraped_item_fn=None,
         retry_blocked_request_fn=None,
         robots_txt_obey: bool = False,
+        start_urls: list[str] | None = None,
     ):
         self.concurrent_requests = concurrent_requests
         self.concurrent_requests_per_domain = concurrent_requests_per_domain
@@ -95,6 +98,7 @@ class MockSpider:
         self.fp_keep_fragments = fp_keep_fragments
         self.name = "test_spider"
         self.robots_txt_obey = robots_txt_obey
+        self.start_urls = start_urls or []
 
         # Tracking lists
         self.on_start_calls: list[dict] = []
@@ -914,3 +918,83 @@ class TestPauseDuringCrawl:
         await engine.crawl()
 
         assert engine.paused is False
+
+
+# ---------------------------------------------------------------------------
+# Tests: _prefetch_robots_txt
+# ---------------------------------------------------------------------------
+
+
+class TestPrefetchRobotsTxt:
+    """_prefetch_robots_txt warms the robots.txt cache before the crawl loop."""
+
+    @staticmethod
+    def _make_counting_fetch():
+        """Return (fetch_fn, calls_list) where calls_list records every (url, sid) pair."""
+        calls: list[tuple[str, str]] = []
+
+        async def _fetch(url: str, sid: str):
+            calls.append((url, sid))
+            return MockResponse(status=200, body=b"", url=url)
+
+        return _fetch, calls
+
+    @pytest.mark.asyncio
+    async def test_prefetch_uses_allowed_domains_when_set(self):
+        fetch_fn, calls = self._make_counting_fetch()
+        spider = MockSpider(allowed_domains={"a.com", "b.com"}, robots_txt_obey=True)
+        engine = _make_engine(spider=spider)
+        engine._robots_manager = RobotsTxtManager(fetch_fn)
+
+        await engine._prefetch_robots_txt()
+
+        fetched_domains = {Request(url).domain for url, _ in calls}
+        assert fetched_domains == {"a.com", "b.com"}
+
+    @pytest.mark.asyncio
+    async def test_prefetch_falls_back_to_start_urls_when_no_allowed_domains(self):
+        fetch_fn, calls = self._make_counting_fetch()
+        spider = MockSpider(robots_txt_obey=True, start_urls=["https://example.com/page1"])
+        engine = _make_engine(spider=spider)
+        engine._robots_manager = RobotsTxtManager(fetch_fn)
+
+        await engine._prefetch_robots_txt()
+
+        assert len(calls) == 1
+        assert calls[0][0] == "https://example.com/robots.txt"
+
+    @pytest.mark.asyncio
+    async def test_prefetch_noop_when_robots_disabled(self):
+        fetch_fn, calls = self._make_counting_fetch()
+        spider = MockSpider(robots_txt_obey=False)
+        engine = _make_engine(spider=spider)
+
+        assert engine._robots_manager is None
+
+        await engine._prefetch_robots_txt()
+
+        assert calls == []
+
+    @pytest.mark.asyncio
+    async def test_prefetch_noop_when_start_urls_empty(self):
+        fetch_fn, calls = self._make_counting_fetch()
+        spider = MockSpider(robots_txt_obey=True, start_urls=[])
+        engine = _make_engine(spider=spider)
+        engine._robots_manager = RobotsTxtManager(fetch_fn)
+
+        await engine._prefetch_robots_txt()
+
+        assert calls == []
+
+    @pytest.mark.asyncio
+    async def test_prefetch_deduplicates_same_domain_in_start_urls(self):
+        fetch_fn, calls = self._make_counting_fetch()
+        spider = MockSpider(robots_txt_obey=True, start_urls=["https://example.com/a", "https://example.com/b"])
+        engine = _make_engine(spider=spider)
+        engine._robots_manager = RobotsTxtManager(fetch_fn)
+
+        await engine._prefetch_robots_txt()
+
+        # set of Request.domain values deduplicates to one task per domain
+        assert len(calls) == 1
+        assert calls[0][0] == "https://example.com/robots.txt"