freedak/Scrapling
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: make retry_blocked_request take the response as well

Browse Source
This commit is contained in:
Karim shoair
2026-01-11 22:54:17 +02:00
parent 736873a7c6
commit 5a11c2466b
2 changed files with 2 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scrapling/spiders/engine.py
+1 -1
View File
@@ -82,7 +82,7 @@ class CrawlerEngine:
retry_request._retry_count += 1 retry_request._retry_count += 1
retry_request.priority -= 1 # Don't retry immediately retry_request.priority -= 1 # Don't retry immediately
retry_request.dont_filter = True retry_request.dont_filter = True
new_request = await self.spider.retry_blocked_request(retry_request) new_request = await self.spider.retry_blocked_request(retry_request, response)
await self.scheduler.enqueue(new_request) await self.scheduler.enqueue(new_request)
log.debug( log.debug(
f"Scheduled blocked request for retry ({retry_request._retry_count}/{self.spider.max_blocked_retries}): {request.url}" f"Scheduled blocked request for retry ({retry_request._retry_count}/{self.spider.max_blocked_retries}): {request.url}"
scrapling/spiders/spider.py
+1 -1
View File
@@ -162,7 +162,7 @@ class Spider(ABC):
return True return True
return False return False
async def retry_blocked_request(self, request: Request) -> Request: async def retry_blocked_request(self, request: Request, response: "Response") -> Request:
"""Users should override this to prepare the blocked request before retrying, if needed.""" """Users should override this to prepare the blocked request before retrying, if needed."""
return request return request