refactor(spider): prefetch robots.txt from start_urls only

Stop using allowed_domains for robots.txt prefetch since bare domain strings have no scheme info.
Domains discovered mid-crawl via requests still fetch robots.txt lazily.

scrapling/spiders/engine.py

@@ -275,27 +275,19 @@ class CrawlerEngine:
     async def _prefetch_robots_txt(self) -> None:
         """Pre-warm the robots.txt cache before the crawl loop starts.
 
-        Uses allowed_domains if configured (defaults to https since bare domains
+        Extracts unique domains from start_urls, preserving the original scheme.
-        have no scheme), otherwise falls back to unique domains extracted from
-        start_urls preserving the original scheme.
         """
-        if not self._robots_manager:
+        if not self._robots_manager or not self.spider.start_urls:
             return
 
-        if self._allowed_domains:
+        # Deduplicate by netloc, preserving the scheme from the first URL per domain
-            # allowed_domains are bare strings like "example.com", no scheme available
+        seen: set[str] = set()
-            seed_urls = [f"http://{domain}/" for domain in self._allowed_domains]
+        seed_urls: list[str] = []
-        elif self.spider.start_urls:
+        for url in self.spider.start_urls:
-            # Deduplicate by netloc, preserving the scheme from the first URL per domain
+            parsed = urlparse(url)
-            seen: set[str] = set()
+            if parsed.netloc not in seen:
-            seed_urls = []
+                seen.add(parsed.netloc)
-            for url in self.spider.start_urls:
+                seed_urls.append(f"{parsed.scheme}://{parsed.netloc}/")
-                parsed = urlparse(url)
-                if parsed.netloc not in seen:
-                    seen.add(parsed.netloc)
-                    seed_urls.append(f"{parsed.scheme}://{parsed.netloc}/")
-        else:
-            return
 
         await self._robots_manager.prefetch(seed_urls, self.session_manager.default_session_id)