diff --git a/scrapling/spiders/engine.py b/scrapling/spiders/engine.py index dce7b08..83cafb3 100644 --- a/scrapling/spiders/engine.py +++ b/scrapling/spiders/engine.py @@ -75,7 +75,22 @@ class CrawlerEngine: await self.spider.on_error(request, e) return - # Process response through callback + if await self.spider.is_blocked(response): + self.stats.blocked_requests_count += 1 + if request._retry_count < self.spider.max_blocked_retries: + retry_request = request.copy() + retry_request._retry_count += 1 + retry_request.priority -= 1 # Don't retry immediately + retry_request.dont_filter = True + new_request = await self.spider.retry_blocked_request(retry_request) + await self.scheduler.enqueue(new_request) + log.debug( + f"Scheduled blocked request for retry ({retry_request._retry_count}/{self.spider.max_blocked_retries}): {request.url}" + ) + else: + log.warning(f"Max retries exceeded for blocked request: {request.url}") + return + callback = request.callback if request.callback else self.spider.parse try: async for result in callback(response): diff --git a/scrapling/spiders/request.py b/scrapling/spiders/request.py index 495c73f..e6b25d0 100644 --- a/scrapling/spiders/request.py +++ b/scrapling/spiders/request.py @@ -13,6 +13,7 @@ class Request: priority: int = 0, dont_filter: bool = False, meta: dict[str, Any] | None = None, + _retry_count: int = 0, **kwargs: Any, ) -> None: self.url: str = url @@ -21,8 +22,22 @@ class Request: self.priority: int = priority self.dont_filter: bool = dont_filter self.meta: dict[str, Any] = meta if meta else {} + self._retry_count: int = _retry_count self._session_kwargs = kwargs if kwargs else {} + def copy(self) -> "Request": + """Create a copy of this request.""" + return Request( + url=self.url, + sid=self.sid, + callback=self.callback, + priority=self.priority, + dont_filter=self.dont_filter, + meta=self.meta.copy(), + _retry_count=self._retry_count, + **self._session_kwargs, + ) + @property def domain(self) -> str: return urlparse(self.url).netloc diff --git a/scrapling/spiders/spider.py b/scrapling/spiders/spider.py index 5c51071..5333f50 100644 --- a/scrapling/spiders/spider.py +++ b/scrapling/spiders/spider.py @@ -74,6 +74,7 @@ class Spider(ABC): concurrent_requests: int = 16 concurrent_requests_per_domain: int = 0 download_delay: float = 0.0 + max_blocked_retries: int = 3 # Logging settings logging_level: int = logging.DEBUG @@ -155,14 +156,16 @@ class Spider(ABC): """ self.logger.error(error, exc_info=error) - @staticmethod - async def is_blocked(response: "Response") -> bool: - """Check if the response is blocked.""" - # TODO + async def is_blocked(self, response: "Response") -> bool: + """Check if the response is blocked. Users should override this for custom detection logic.""" if response.status in BLOCKED_CODES: return True return False + async def retry_blocked_request(self, request: Request) -> Request: + """Users should override this to prepare the blocked request before retrying, if needed.""" + return request + def __repr__(self) -> str: """String representation of the spider.""" return f"<{self.__class__.__name__} '{self.name}'>"