fix(spider robots): solve multiple issues with cache prefetch

scrapling/spiders/engine.py

@@ -1,6 +1,7 @@
 import json
 import pprint
 from pathlib import Path
+from urllib.parse import urlparse
 
 import anyio
 from anyio import Path as AsyncPath
@@ -294,22 +295,28 @@ class CrawlerEngine:
     async def _prefetch_robots_txt(self) -> None:
         """Pre-warm the robots.txt cache before the crawl loop starts.
 
-        Uses allowed_domains if configured, otherwise falls back to unique domains
+        Uses allowed_domains if configured (defaults to https since bare domains
-        extracted from start_urls via Request.domain. Both paths use https.
+        have no scheme), otherwise falls back to unique domains extracted from
+        start_urls preserving the original scheme.
         """
         if not self._robots_manager:
             return
 
         if self._allowed_domains:
-            domains = self._allowed_domains
+            # allowed_domains are bare strings like "example.com", no scheme available
+            seed_urls = [f"http://{domain}/" for domain in self._allowed_domains]
         elif self.spider.start_urls:
-            # Deduplicate by domain so we spawn exactly one task per domain
+            # Deduplicate by netloc, preserving the scheme from the first URL per domain
-            domains = {Request(url).domain for url in self.spider.start_urls}
+            seen: set[str] = set()
+            seed_urls = []
+            for url in self.spider.start_urls:
+                parsed = urlparse(url)
+                if parsed.netloc not in seen:
+                    seen.add(parsed.netloc)
+                    seed_urls.append(f"{parsed.scheme}://{parsed.netloc}/")
         else:
             return
 
-        seed_urls = [f"https://{domain}/" for domain in domains]
-
         await self._robots_manager.prefetch(seed_urls, self.session_manager.default_session_id)
 
     async def crawl(self) -> CrawlStats: