From f88502718fe06675c4b353db2a0d6f33fb2a8464 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Mon, 2 Feb 2026 14:26:52 +0200 Subject: [PATCH] feat(proxy control): Force a proxy at request level at any given point And merge request's meta with response's meta --- scrapling/engines/_browsers/_base.py | 4 ++-- scrapling/engines/_browsers/_controllers.py | 20 ++++++++++++++++---- scrapling/engines/_browsers/_stealth.py | 20 ++++++++++++++++---- scrapling/engines/_browsers/_types.py | 1 + scrapling/engines/static.py | 4 ++-- scrapling/engines/toolbelt/convertor.py | 8 +++++++- scrapling/engines/toolbelt/custom.py | 6 +++++- scrapling/spiders/engine.py | 3 +++ scrapling/spiders/session.py | 3 ++- 9 files changed, 54 insertions(+), 15 deletions(-) diff --git a/scrapling/engines/_browsers/_base.py b/scrapling/engines/_browsers/_base.py index 5936652..01bced5 100644 --- a/scrapling/engines/_browsers/_base.py +++ b/scrapling/engines/_browsers/_base.py @@ -175,7 +175,7 @@ class SyncSession: proxy: Optional[ProxyType] = None, ) -> Generator["PageInfo[Page]", None, None]: """Acquire a page - either from persistent context or fresh context with proxy.""" - if self._config.proxy_rotator: + if proxy: # Rotation mode: create fresh context with the provided proxy if not self.browser: # pragma: no cover raise RuntimeError("Browser not initialized for proxy rotation mode") @@ -344,7 +344,7 @@ class AsyncSession: proxy: Optional[ProxyType] = None, ) -> AsyncGenerator["PageInfo[AsyncPage]", None]: """Acquire a page - either from persistent context or fresh context with proxy.""" - if self._config.proxy_rotator: + if proxy: # Rotation mode: create fresh context with the provided proxy if not self.browser: # pragma: no cover raise RuntimeError("Browser not initialized for proxy rotation mode") diff --git a/scrapling/engines/_browsers/_controllers.py b/scrapling/engines/_browsers/_controllers.py index 34a0b25..b43807c 100644 --- a/scrapling/engines/_browsers/_controllers.py +++ b/scrapling/engines/_browsers/_controllers.py @@ -115,8 +115,11 @@ class DynamicSession(SyncSession, DynamicSessionMixin): :param network_idle: Wait for the page until there are no network connections for at least 500 ms. :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute. :param selector_config: The arguments that will be passed in the end while creating the final Selector's class. + :param proxy: Static proxy to override rotator and session proxy. A new browser context will be created and used with it. :return: A `Response` object. """ + static_proxy = kwargs.pop("proxy", None) + params = _validate(kwargs, self, PlaywrightConfig) if not self._is_alive: # pragma: no cover raise RuntimeError("Context manager has been closed") @@ -129,7 +132,10 @@ class DynamicSession(SyncSession, DynamicSessionMixin): ) for attempt in range(self._config.retries): - proxy = self._config.proxy_rotator.get_proxy() if self._config.proxy_rotator else None + if self._config.proxy_rotator and static_proxy is None: + proxy = self._config.proxy_rotator.get_proxy() + else: + proxy = static_proxy with self._page_generator( params.timeout, params.extra_headers, params.disable_resources, proxy @@ -162,7 +168,7 @@ class DynamicSession(SyncSession, DynamicSessionMixin): page.wait_for_timeout(params.wait) response = ResponseFactory.from_playwright_response( - page, first_response, final_response[0], params.selector_config + page, first_response, final_response[0], params.selector_config, meta={"proxy": proxy} ) return response @@ -276,8 +282,11 @@ class AsyncDynamicSession(AsyncSession, DynamicSessionMixin): :param network_idle: Wait for the page until there are no network connections for at least 500 ms. :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute. :param selector_config: The arguments that will be passed in the end while creating the final Selector's class. + :param proxy: Static proxy to override rotator and session proxy. A new browser context will be created and used with it. :return: A `Response` object. """ + static_proxy = kwargs.pop("proxy", None) + params = _validate(kwargs, self, PlaywrightConfig) if not self._is_alive: # pragma: no cover @@ -291,7 +300,10 @@ class AsyncDynamicSession(AsyncSession, DynamicSessionMixin): ) for attempt in range(self._config.retries): - proxy = self._config.proxy_rotator.get_proxy() if self._config.proxy_rotator else None + if self._config.proxy_rotator and static_proxy is None: + proxy = self._config.proxy_rotator.get_proxy() + else: + proxy = static_proxy async with self._page_generator( params.timeout, params.extra_headers, params.disable_resources, proxy @@ -324,7 +336,7 @@ class AsyncDynamicSession(AsyncSession, DynamicSessionMixin): await page.wait_for_timeout(params.wait) response = await ResponseFactory.from_async_playwright_response( - page, first_response, final_response[0], params.selector_config + page, first_response, final_response[0], params.selector_config, meta={"proxy": proxy} ) return response diff --git a/scrapling/engines/_browsers/_stealth.py b/scrapling/engines/_browsers/_stealth.py index fc76d26..5564376 100644 --- a/scrapling/engines/_browsers/_stealth.py +++ b/scrapling/engines/_browsers/_stealth.py @@ -204,8 +204,11 @@ class StealthySession(SyncSession, StealthySessionMixin): :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute. :param solve_cloudflare: Solves all types of the Cloudflare's Turnstile/Interstitial challenges before returning the response to you. :param selector_config: The arguments that will be passed in the end while creating the final Selector's class. + :param proxy: Static proxy to override rotator and session proxy. A new browser context will be created and used with it. :return: A `Response` object. """ + static_proxy = kwargs.pop("proxy", None) + params = _validate(kwargs, self, StealthConfig) if not self._is_alive: # pragma: no cover raise RuntimeError("Context manager has been closed") @@ -218,7 +221,10 @@ class StealthySession(SyncSession, StealthySessionMixin): ) for attempt in range(self._config.retries): - proxy = self._config.proxy_rotator.get_proxy() if self._config.proxy_rotator else None + if self._config.proxy_rotator and static_proxy is None: + proxy = self._config.proxy_rotator.get_proxy() + else: + proxy = static_proxy with self._page_generator( params.timeout, params.extra_headers, params.disable_resources, proxy @@ -256,7 +262,7 @@ class StealthySession(SyncSession, StealthySessionMixin): page.wait_for_timeout(params.wait) response = ResponseFactory.from_playwright_response( - page, first_response, final_response[0], params.selector_config + page, first_response, final_response[0], params.selector_config, meta={"proxy": proxy} ) return response @@ -454,8 +460,11 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin): :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute. :param solve_cloudflare: Solves all types of the Cloudflare's Turnstile/Interstitial challenges before returning the response to you. :param selector_config: The arguments that will be passed in the end while creating the final Selector's class. + :param proxy: Static proxy to override rotator and session proxy. A new browser context will be created and used with it. :return: A `Response` object. """ + static_proxy = kwargs.pop("proxy", None) + params = _validate(kwargs, self, StealthConfig) if not self._is_alive: # pragma: no cover @@ -469,7 +478,10 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin): ) for attempt in range(self._config.retries): - proxy = self._config.proxy_rotator.get_proxy() if self._config.proxy_rotator else None + if self._config.proxy_rotator and static_proxy is None: + proxy = self._config.proxy_rotator.get_proxy() + else: + proxy = static_proxy async with self._page_generator( params.timeout, params.extra_headers, params.disable_resources, proxy @@ -507,7 +519,7 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin): await page.wait_for_timeout(params.wait) response = await ResponseFactory.from_async_playwright_response( - page, first_response, final_response[0], params.selector_config + page, first_response, final_response[0], params.selector_config, meta={"proxy": proxy} ) return response diff --git a/scrapling/engines/_browsers/_types.py b/scrapling/engines/_browsers/_types.py index afce5d0..d298248 100644 --- a/scrapling/engines/_browsers/_types.py +++ b/scrapling/engines/_browsers/_types.py @@ -99,6 +99,7 @@ if TYPE_CHECKING: # pragma: no cover selector_config: Optional[Dict] extra_headers: Optional[Dict[str, str]] wait_selector_state: SelectorWaitStates + proxy: Optional[str | Dict[str, str]] class StealthSession(PlaywrightSession, total=False): allow_webgl: bool diff --git a/scrapling/engines/static.py b/scrapling/engines/static.py index e8a41aa..884fe0c 100644 --- a/scrapling/engines/static.py +++ b/scrapling/engines/static.py @@ -250,7 +250,7 @@ class _SyncSessionLogic(_ConfigurationLogic): request_args = self._merge_request_args(stealth=stealth, proxy=proxy, **kwargs) try: response = session.request(method, **request_args) - result = ResponseFactory.from_http_request(response, selector_config) + result = ResponseFactory.from_http_request(response, selector_config, meta={"proxy": proxy}) return result except CurlError as e: # pragma: no cover if attempt < max_retries - 1: @@ -466,7 +466,7 @@ class _ASyncSessionLogic(_ConfigurationLogic): request_args = self._merge_request_args(stealth=stealth, proxy=proxy, **kwargs) try: response = await session.request(method, **request_args) - result = ResponseFactory.from_http_request(response, selector_config) + result = ResponseFactory.from_http_request(response, selector_config, meta={"proxy": proxy}) return result except CurlError as e: # pragma: no cover if attempt < max_retries - 1: diff --git a/scrapling/engines/toolbelt/convertor.py b/scrapling/engines/toolbelt/convertor.py index ef28acf..9dff73d 100644 --- a/scrapling/engines/toolbelt/convertor.py +++ b/scrapling/engines/toolbelt/convertor.py @@ -85,6 +85,7 @@ class ResponseFactory: first_response: SyncResponse, final_response: Optional[SyncResponse], parser_arguments: Dict, + meta: Optional[Dict] = None, ) -> Response: """ Transforms a Playwright response into an internal `Response` object, encapsulating @@ -134,6 +135,7 @@ class ResponseFactory: "headers": first_response.all_headers(), "request_headers": first_response.request.all_headers(), "history": history, + "meta": meta, **parser_arguments, } ) @@ -220,6 +222,7 @@ class ResponseFactory: first_response: AsyncResponse, final_response: Optional[AsyncResponse], parser_arguments: Dict, + meta: Optional[Dict] = None, ) -> Response: """ Transforms a Playwright response into an internal `Response` object, encapsulating @@ -269,16 +272,18 @@ class ResponseFactory: "headers": await first_response.all_headers(), "request_headers": await first_response.request.all_headers(), "history": history, + "meta": meta, **parser_arguments, } ) @staticmethod - def from_http_request(response: CurlResponse, parser_arguments: Dict) -> Response: + def from_http_request(response: CurlResponse, parser_arguments: Dict, meta: Optional[Dict] = None) -> Response: """Takes `curl_cffi` response and generates `Response` object from it. :param response: `curl_cffi` response object :param parser_arguments: Additional arguments to be passed to the `Response` object constructor. + :param meta: Optional metadata dictionary to attach to the Response. :return: A `Response` object that is the same as `Selector` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers` """ return Response( @@ -293,6 +298,7 @@ class ResponseFactory: "request_headers": dict(response.request.headers) if response.request else {}, "method": response.request.method if response.request else "GET", "history": response.history, # https://github.com/lexiforest/curl_cffi/issues/82 + "meta": meta, **parser_arguments, } ) diff --git a/scrapling/engines/toolbelt/custom.py b/scrapling/engines/toolbelt/custom.py index f5b8f62..29b9e17 100644 --- a/scrapling/engines/toolbelt/custom.py +++ b/scrapling/engines/toolbelt/custom.py @@ -39,6 +39,7 @@ class Response(Selector): encoding: str = "utf-8", method: str = "GET", history: List | None = None, + meta: Dict[str, Any] | None = None, **selector_config: Any, ): adaptive_domain: str = cast(str, selector_config.pop("adaptive_domain", "")) @@ -57,7 +58,10 @@ class Response(Selector): # For easier debugging while working from a Python shell log.info(f"Fetched ({status}) <{method} {url}> (referer: {request_headers.get('referer')})") - self.meta: Dict[str, Any] = {} + if meta and not isinstance(meta, dict): + raise TypeError(f"Response meta should be dictionary but got {type(meta).__name__} instead!") + + self.meta: Dict[str, Any] = meta or {} self.request: Optional["Request"] = None # Will be set by crawler def follow( diff --git a/scrapling/spiders/engine.py b/scrapling/spiders/engine.py index c57a4b4..416911d 100644 --- a/scrapling/spiders/engine.py +++ b/scrapling/spiders/engine.py @@ -113,6 +113,9 @@ class CrawlerEngine: retry_request._retry_count += 1 retry_request.priority -= 1 # Don't retry immediately retry_request.dont_filter = True + retry_request._session_kwargs.pop("proxy", None) + retry_request._session_kwargs.pop("proxies", None) + new_request = await self.spider.retry_blocked_request(retry_request, response) self._normalize_request(new_request) await self.scheduler.enqueue(new_request) diff --git a/scrapling/spiders/session.py b/scrapling/spiders/session.py index f5b5a49..8f5e7af 100644 --- a/scrapling/spiders/session.py +++ b/scrapling/spiders/session.py @@ -124,7 +124,8 @@ class SessionManager: response = await session.fetch(url=request.url, **request._session_kwargs) response.request = request - response.meta = request.meta + # Merge request meta into response meta (response meta takes priority) + response.meta = {**request.meta, **response.meta} return response raise RuntimeError("No session found with the request session id")