From af83a11aa7873dc8342edf1279a2cb00aaf677a9 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Sun, 5 Apr 2026 00:09:17 +0200 Subject: [PATCH] fix(spider robots): solve multiple issues with cache prefetch --- scrapling/spiders/engine.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/scrapling/spiders/engine.py b/scrapling/spiders/engine.py index f395a70..3b41fb5 100644 --- a/scrapling/spiders/engine.py +++ b/scrapling/spiders/engine.py @@ -1,6 +1,7 @@ import json import pprint from pathlib import Path +from urllib.parse import urlparse import anyio from anyio import Path as AsyncPath @@ -294,22 +295,28 @@ class CrawlerEngine: async def _prefetch_robots_txt(self) -> None: """Pre-warm the robots.txt cache before the crawl loop starts. - Uses allowed_domains if configured, otherwise falls back to unique domains - extracted from start_urls via Request.domain. Both paths use https. + Uses allowed_domains if configured (defaults to https since bare domains + have no scheme), otherwise falls back to unique domains extracted from + start_urls preserving the original scheme. """ if not self._robots_manager: return if self._allowed_domains: - domains = self._allowed_domains + # allowed_domains are bare strings like "example.com", no scheme available + seed_urls = [f"http://{domain}/" for domain in self._allowed_domains] elif self.spider.start_urls: - # Deduplicate by domain so we spawn exactly one task per domain - domains = {Request(url).domain for url in self.spider.start_urls} + # Deduplicate by netloc, preserving the scheme from the first URL per domain + seen: set[str] = set() + seed_urls = [] + for url in self.spider.start_urls: + parsed = urlparse(url) + if parsed.netloc not in seen: + seen.add(parsed.netloc) + seed_urls.append(f"{parsed.scheme}://{parsed.netloc}/") else: return - seed_urls = [f"https://{domain}/" for domain in domains] - await self._robots_manager.prefetch(seed_urls, self.session_manager.default_session_id) async def crawl(self) -> CrawlStats: