From 556a90f645f4aaecbc95de899f6cc6d2807e184a Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Sun, 5 Apr 2026 01:43:06 +0200 Subject: [PATCH] refactor(spider): prefetch robots.txt from start_urls only Stop using allowed_domains for robots.txt prefetch since bare domain strings have no scheme info. Domains discovered mid-crawl via requests still fetch robots.txt lazily. --- scrapling/spiders/engine.py | 28 ++++++++++------------------ 1 file changed, 10 insertions(+), 18 deletions(-) diff --git a/scrapling/spiders/engine.py b/scrapling/spiders/engine.py index 3d75564..7729d62 100644 --- a/scrapling/spiders/engine.py +++ b/scrapling/spiders/engine.py @@ -275,27 +275,19 @@ class CrawlerEngine: async def _prefetch_robots_txt(self) -> None: """Pre-warm the robots.txt cache before the crawl loop starts. - Uses allowed_domains if configured (defaults to https since bare domains - have no scheme), otherwise falls back to unique domains extracted from - start_urls preserving the original scheme. + Extracts unique domains from start_urls, preserving the original scheme. """ - if not self._robots_manager: + if not self._robots_manager or not self.spider.start_urls: return - if self._allowed_domains: - # allowed_domains are bare strings like "example.com", no scheme available - seed_urls = [f"http://{domain}/" for domain in self._allowed_domains] - elif self.spider.start_urls: - # Deduplicate by netloc, preserving the scheme from the first URL per domain - seen: set[str] = set() - seed_urls = [] - for url in self.spider.start_urls: - parsed = urlparse(url) - if parsed.netloc not in seen: - seen.add(parsed.netloc) - seed_urls.append(f"{parsed.scheme}://{parsed.netloc}/") - else: - return + # Deduplicate by netloc, preserving the scheme from the first URL per domain + seen: set[str] = set() + seed_urls: list[str] = [] + for url in self.spider.start_urls: + parsed = urlparse(url) + if parsed.netloc not in seen: + seen.add(parsed.netloc) + seed_urls.append(f"{parsed.scheme}://{parsed.netloc}/") await self._robots_manager.prefetch(seed_urls, self.session_manager.default_session_id)