freedak/Scrapling
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(spider): only allocate _domain_delays when robots_txt_obey is enabled

Browse Source
This commit is contained in:
Karim shoair
2026-04-05 02:36:15 +02:00
parent 911d3af632
commit 070338cf24
1 changed files with 5 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scrapling/spiders/engine.py
+4 -1
View File
@@ -54,9 +54,11 @@ class CrawlerEngine:
self._global_limiter = CapacityLimiter(spider.concurrent_requests) self._global_limiter = CapacityLimiter(spider.concurrent_requests)
self._domain_limiters: dict[str, CapacityLimiter] = {} self._domain_limiters: dict[str, CapacityLimiter] = {}
self._domain_delays: dict[str, float] = {}
self._allowed_domains: set[str] = spider.allowed_domains or set() self._allowed_domains: set[str] = spider.allowed_domains or set()
if self.spider.robots_txt_obey:
self._domain_delays: dict[str, float] = {}
self._active_tasks: int = 0 self._active_tasks: int = 0
self._running: bool = False self._running: bool = False
self._items: ItemList = ItemList() self._items: ItemList = ItemList()
@@ -300,6 +302,7 @@ class CrawlerEngine:
self._force_stop = False self._force_stop = False
self.stats = CrawlStats(start_time=anyio.current_time()) self.stats = CrawlStats(start_time=anyio.current_time())
self._domain_limiters.clear() self._domain_limiters.clear()
if self._robots_manager:
self._domain_delays.clear() self._domain_delays.clear()
# Check for existing checkpoint # Check for existing checkpoint