From ec487d37e87f9a17d8df917cafd02e8fe26a3229 Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Sun, 5 Apr 2026 01:39:05 +0200
Subject: [PATCH] fix(spider): Make delay in robots file don't affect user's
 concurrency settings

---
 scrapling/spiders/engine.py | 24 ++----------------------
 1 file changed, 2 insertions(+), 22 deletions(-)

diff --git a/scrapling/spiders/engine.py b/scrapling/spiders/engine.py
index 3b41fb5..3d75564 100644
--- a/scrapling/spiders/engine.py
+++ b/scrapling/spiders/engine.py
@@ -85,8 +85,6 @@ class CrawlerEngine:
 
         Takes the max of the spider's configured delay and any robots.txt
         directives (Crawl-delay / Request-rate). Result is cached per domain.
-        Also pre-creates a per-domain concurrency limiter of 1 when robots.txt
-        enforces any delay, before the caller acquires it via _rate_limiter().
         """
         robots_manager = self._robots_manager
         if robots_manager is None:
@@ -94,48 +92,32 @@ class CrawlerEngine:
 
         domain = request.domain
 
-        # Return cached delay if available
         if domain in self._domain_delays:
             return self._domain_delays[domain]
 
         # For domains covered by _prefetch_robots_txt this is a local parser read.
         # Domains discovered mid-crawl (not in start_urls/allowed_domains) will fetch here.
-        # Two concurrent callbacks hitting the same new domain can each trigger a fetch;
-        # the second write is a no-op in effect (same content), but the extra request is accepted.
         c_delay, r_rate = await robots_manager.get_delay_directives(request.url, request.sid)
 
         delay = self.spider.download_delay
-        robots_enforced_delay = False
 
         if r_rate:
             req_count, period = r_rate
             if req_count > 0:
                 delay = max(delay, period / req_count)
-                robots_enforced_delay = True
 
         if c_delay is not None:
             delay = max(delay, c_delay)
-            robots_enforced_delay = True
 
         self._domain_delays[domain] = delay
-
-        # Enforce 1 concurrent request for this domain when robots.txt adds a delay
-        if robots_enforced_delay and delay > 0 and domain not in self._domain_limiters:
-            if self.spider.concurrent_requests_per_domain:
-                log.warning(
-                    f"robots.txt for {domain} enforces a delay, overriding"
-                    f" concurrent_requests_per_domain={self.spider.concurrent_requests_per_domain} with 1"
-                )
-            self._domain_limiters[domain] = CapacityLimiter(1)
-
         return delay
 
     def _rate_limiter(self, domain: str) -> CapacityLimiter:
         """Get or create a per-domain concurrency limiter if enabled, otherwise use the global limiter."""
         if self.spider.concurrent_requests_per_domain:
             self._domain_limiters.setdefault(domain, CapacityLimiter(self.spider.concurrent_requests_per_domain))
-        # robots.txt-created limiters always apply even with `concurrent_requests_per_domain = 0` (if enabled)
-        return self._domain_limiters.get(domain, self._global_limiter)
+            return self._domain_limiters[domain]
+        return self._global_limiter
 
     def _normalize_request(self, request: Request) -> None:
         """Normalize request fields before enqueueing.
@@ -154,8 +136,6 @@ class CrawlerEngine:
                 self.stats.robots_disallowed_count += 1
                 log.debug(f"Request disallowed by robots.txt: {request.url}")
                 return
-            # Must be called before _rate_limiter: may create CapacityLimiter(1) in _domain_limiters
-            # when robots.txt enforces a delay, which _rate_limiter then picks up.
             delay = await self._get_domain_delay(request)
         else:
             delay = self.spider.download_delay