feat(proxy control): Force a proxy at request level at any given point

And merge request's meta with response's meta

scrapling/spiders/engine.py

@@ -113,6 +113,9 @@ class CrawlerEngine:
                 retry_request._retry_count += 1
                 retry_request.priority -= 1  # Don't retry immediately
                 retry_request.dont_filter = True
+                retry_request._session_kwargs.pop("proxy", None)
+                retry_request._session_kwargs.pop("proxies", None)
+
                 new_request = await self.spider.retry_blocked_request(retry_request, response)
                 self._normalize_request(new_request)
                 await self.scheduler.enqueue(new_request)

scrapling/spiders/session.py

@@ -124,7 +124,8 @@ class SessionManager:
                 response = await session.fetch(url=request.url, **request._session_kwargs)
 
             response.request = request
-            response.meta = request.meta
+            # Merge request meta into response meta (response meta takes priority)
+            response.meta = {**request.meta, **response.meta}
             return response
         raise RuntimeError("No session found with the request session id")