feat(spiders): Add logic to detect blocked requests and retry them

scrapling/spiders/engine.py

@@ -75,7 +75,22 @@ class CrawlerEngine:
                 await self.spider.on_error(request, e)
                 return
 
-        # Process response through callback
+        if await self.spider.is_blocked(response):
+            self.stats.blocked_requests_count += 1
+            if request._retry_count < self.spider.max_blocked_retries:
+                retry_request = request.copy()
+                retry_request._retry_count += 1
+                retry_request.priority -= 1  # Don't retry immediately
+                retry_request.dont_filter = True
+                new_request = await self.spider.retry_blocked_request(retry_request)
+                await self.scheduler.enqueue(new_request)
+                log.debug(
+                    f"Scheduled blocked request for retry ({retry_request._retry_count}/{self.spider.max_blocked_retries}): {request.url}"
+                )
+            else:
+                log.warning(f"Max retries exceeded for blocked request: {request.url}")
+            return
+
         callback = request.callback if request.callback else self.spider.parse
         try:
             async for result in callback(response):

scrapling/spiders/request.py

@@ -13,6 +13,7 @@ class Request:
         priority: int = 0,
         dont_filter: bool = False,
         meta: dict[str, Any] | None = None,
+        _retry_count: int = 0,
         **kwargs: Any,
     ) -> None:
         self.url: str = url
@@ -21,8 +22,22 @@ class Request:
         self.priority: int = priority
         self.dont_filter: bool = dont_filter
         self.meta: dict[str, Any] = meta if meta else {}
+        self._retry_count: int = _retry_count
         self._session_kwargs = kwargs if kwargs else {}
 
+    def copy(self) -> "Request":
+        """Create a copy of this request."""
+        return Request(
+            url=self.url,
+            sid=self.sid,
+            callback=self.callback,
+            priority=self.priority,
+            dont_filter=self.dont_filter,
+            meta=self.meta.copy(),
+            _retry_count=self._retry_count,
+            **self._session_kwargs,
+        )
+
     @property
     def domain(self) -> str:
         return urlparse(self.url).netloc

scrapling/spiders/spider.py

@@ -74,6 +74,7 @@ class Spider(ABC):
     concurrent_requests: int = 16
     concurrent_requests_per_domain: int = 0
     download_delay: float = 0.0
+    max_blocked_retries: int = 3
 
     # Logging settings
     logging_level: int = logging.DEBUG
@@ -155,14 +156,16 @@ class Spider(ABC):
         """
         self.logger.error(error, exc_info=error)
 
-    @staticmethod
+    async def is_blocked(self, response: "Response") -> bool:
-    async def is_blocked(response: "Response") -> bool:
+        """Check if the response is blocked. Users should override this for custom detection logic."""
-        """Check if the response is blocked."""
-        # TODO
         if response.status in BLOCKED_CODES:
             return True
         return False
 
+    async def retry_blocked_request(self, request: Request) -> Request:
+        """Users should override this to prepare the blocked request before retrying, if needed."""
+        return request
+
     def __repr__(self) -> str:
         """String representation of the spider."""
         return f"<{self.__class__.__name__} '{self.name}'>"