diff --git a/scrapling/spiders/engine.py b/scrapling/spiders/engine.py index 83cafb3..26fce1e 100644 --- a/scrapling/spiders/engine.py +++ b/scrapling/spiders/engine.py @@ -82,7 +82,7 @@ class CrawlerEngine: retry_request._retry_count += 1 retry_request.priority -= 1 # Don't retry immediately retry_request.dont_filter = True - new_request = await self.spider.retry_blocked_request(retry_request) + new_request = await self.spider.retry_blocked_request(retry_request, response) await self.scheduler.enqueue(new_request) log.debug( f"Scheduled blocked request for retry ({retry_request._retry_count}/{self.spider.max_blocked_retries}): {request.url}" diff --git a/scrapling/spiders/spider.py b/scrapling/spiders/spider.py index 5333f50..5f3268e 100644 --- a/scrapling/spiders/spider.py +++ b/scrapling/spiders/spider.py @@ -162,7 +162,7 @@ class Spider(ABC): return True return False - async def retry_blocked_request(self, request: Request) -> Request: + async def retry_blocked_request(self, request: Request, response: "Response") -> Request: """Users should override this to prepare the blocked request before retrying, if needed.""" return request