feat(proxy control): Force a proxy at request level at any given point

And merge request's meta with response's meta
2026-02-02 14:26:52 +02:00
parent 6e0277f23b
commit f88502718f
9 changed files with 54 additions and 15 deletions
@@ -113,6 +113,9 @@ class CrawlerEngine:
                retry_request._retry_count += 1
                retry_request.priority -= 1  # Don't retry immediately
                retry_request.dont_filter = True
+                retry_request._session_kwargs.pop("proxy", None)
+                retry_request._session_kwargs.pop("proxies", None)
+
                new_request = await self.spider.retry_blocked_request(retry_request, response)
                self._normalize_request(new_request)
                await self.scheduler.enqueue(new_request)