From 724e3a648e219d820e72c2b780d43ec6e750d344 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Wed, 25 Jun 2025 03:53:54 +0300 Subject: [PATCH] fix(FetcherSession): Use Sentinel pattern to solve arguments precedence issue --- scrapling/engines/static.py | 291 ++++++++++++++++++++---------------- 1 file changed, 159 insertions(+), 132 deletions(-) diff --git a/scrapling/engines/static.py b/scrapling/engines/static.py index 8086bf5..d1940bd 100644 --- a/scrapling/engines/static.py +++ b/scrapling/engines/static.py @@ -33,6 +33,8 @@ from .toolbelt import ( __default_useragent__, ) +_UNSET = object() + class FetcherSession: """ @@ -104,35 +106,47 @@ class FetcherSession: """Merge request-specific arguments with default session arguments.""" url = kwargs.pop("url") request_args = {} - if kwargs.pop("http3", False) or self.default_http3: + + headers = self.get_with_precedence(kwargs, "headers", self.default_headers) + stealth = self.get_with_precedence(kwargs, "stealth", self.stealth) + impersonate = self.get_with_precedence( + kwargs, "impersonate", self.default_impersonate + ) + + if self.get_with_precedence(kwargs, "http3", self.default_http3): request_args["http_version"] = CurlHttpVersion.V3ONLY - if kwargs.get("impersonate"): + if impersonate: log.warning( "The argument `http3` might cause errors if used with `impersonate` argument, try switching it off if you encounter any curl errors." ) - impersonate = kwargs.pop("impersonate", self.default_impersonate) request_args.update( { "url": url, # Curl automatically generates the suitable browser headers when you use `impersonate` - "headers": self._headers_job( - url, kwargs.pop("headers"), kwargs.pop("stealth"), bool(impersonate) + "headers": self._headers_job(url, headers, stealth, bool(impersonate)), + "proxies": self.get_with_precedence( + kwargs, "proxies", self.default_proxies ), - "proxies": kwargs.pop("proxies", self.default_proxies), - "proxy": kwargs.pop("proxy", self.default_proxy), - "proxy_auth": kwargs.pop("proxy_auth", self.default_proxy_auth), - "timeout": kwargs.pop("timeout", self.default_timeout), - "allow_redirects": kwargs.pop( - "follow_redirects", self.default_follow_redirects + "proxy": self.get_with_precedence(kwargs, "proxy", self.default_proxy), + "proxy_auth": self.get_with_precedence( + kwargs, "proxy_auth", self.default_proxy_auth ), - "max_redirects": kwargs.pop( - "max_redirects", self.default_max_redirects + "timeout": self.get_with_precedence( + kwargs, "timeout", self.default_timeout ), - "verify": kwargs.pop("verify", self.default_verify), - "cert": kwargs.pop("cert", self.default_cert), + "allow_redirects": self.get_with_precedence( + kwargs, "allow_redirects", self.default_follow_redirects + ), + "max_redirects": self.get_with_precedence( + kwargs, "max_redirects", self.default_max_redirects + ), + "verify": self.get_with_precedence( + kwargs, "verify", self.default_verify + ), + "cert": self.get_with_precedence(kwargs, "cert", self.default_cert), "impersonate": impersonate, - **kwargs, + **kwargs, # Add any remaining parameters (after all known ones are popped) } ) return request_args @@ -152,9 +166,14 @@ class FetcherSession: :param impersonate_enabled: Whether the browser impersonation is enabled or not. :return: A dictionary of the new headers. """ - headers = {**self.default_headers, **(headers or {})} - headers_keys = set(map(str.lower, headers.keys())) + # Handle headers - if it was _UNSET, use default_headers + if headers is _UNSET: + headers = self.default_headers.copy() + else: + # Merge session headers with request headers, request takes precedence + headers = {**self.default_headers, **(headers or {})} + headers_keys = set(map(str.lower, headers.keys())) if stealth: if "referer" not in headers_keys: headers.update({"referer": generate_convincing_referer(url)}) @@ -307,6 +326,12 @@ class FetcherSession: raise RuntimeError("No active session available.") + @staticmethod + def get_with_precedence(kwargs, key, default_value): + """Get value with request-level priority over session-level""" + request_value = kwargs.pop(key, _UNSET) + return request_value if request_value is not _UNSET else default_value + def __prepare_and_dispatch( self, method: SUPPORTED_HTTP_METHODS, @@ -327,8 +352,10 @@ class FetcherSession: adaptor_arguments = ( kwargs.pop("adaptor_arguments", {}) or self.adaptor_arguments ) - max_retries = kwargs.pop("retries", self.default_retries) - retry_delay = kwargs.pop("retry_delay", self.default_retry_delay) + max_retries = self.get_with_precedence(kwargs, "retries", self.default_retries) + retry_delay = self.get_with_precedence( + kwargs, "retry_delay", self.default_retry_delay + ) request_args = self._merge_request_args(stealth=stealth, **kwargs) if self._curl_session: return self.__make_request( @@ -346,22 +373,22 @@ class FetcherSession: self, url: str, params: Optional[Union[Dict, List, Tuple]] = None, - headers: Optional[Mapping[str, Optional[str]]] = None, + headers: Optional[Mapping[str, Optional[str]]] = _UNSET, cookies: Optional[CookieTypes] = None, - timeout: Optional[Union[int, float]] = 30, - follow_redirects: Optional[bool] = True, - max_redirects: Optional[int] = 30, - retries: Optional[int] = 3, - retry_delay: Optional[int] = 1, - proxies: Optional[ProxySpec] = None, - proxy: Optional[str] = None, - proxy_auth: Optional[Tuple[str, str]] = None, + timeout: Optional[Union[int, float]] = _UNSET, + follow_redirects: Optional[bool] = _UNSET, + max_redirects: Optional[int] = _UNSET, + retries: Optional[int] = _UNSET, + retry_delay: Optional[int] = _UNSET, + proxies: Optional[ProxySpec] = _UNSET, + proxy: Optional[str] = _UNSET, + proxy_auth: Optional[Tuple[str, str]] = _UNSET, auth: Optional[Tuple[str, str]] = None, - verify: Optional[bool] = True, - cert: Optional[Union[str, Tuple[str, str]]] = None, - impersonate: Optional[BrowserTypeLiteral] = DEFAULT_CHROME, - http3: Optional[bool] = False, - stealthy_headers: Optional[bool] = True, + verify: Optional[bool] = _UNSET, + cert: Optional[Union[str, Tuple[str, str]]] = _UNSET, + impersonate: Optional[BrowserTypeLiteral] = _UNSET, + http3: Optional[bool] = _UNSET, + stealthy_headers: Optional[bool] = _UNSET, **kwargs, ) -> Union[Response, Awaitable[Response]]: """ @@ -418,23 +445,23 @@ class FetcherSession: url: str, data: Optional[Union[Dict, str]] = None, json: Optional[Union[Dict, List]] = None, - headers: Optional[Mapping[str, Optional[str]]] = None, + headers: Optional[Mapping[str, Optional[str]]] = _UNSET, params: Optional[Union[Dict, List, Tuple]] = None, cookies: Optional[CookieTypes] = None, - timeout: Optional[Union[int, float]] = 30, - follow_redirects: Optional[bool] = True, - max_redirects: Optional[int] = 30, - retries: Optional[int] = 3, - retry_delay: Optional[int] = 1, - proxies: Optional[ProxySpec] = None, - proxy: Optional[str] = None, - proxy_auth: Optional[Tuple[str, str]] = None, + timeout: Optional[Union[int, float]] = _UNSET, + follow_redirects: Optional[bool] = _UNSET, + max_redirects: Optional[int] = _UNSET, + retries: Optional[int] = _UNSET, + retry_delay: Optional[int] = _UNSET, + proxies: Optional[ProxySpec] = _UNSET, + proxy: Optional[str] = _UNSET, + proxy_auth: Optional[Tuple[str, str]] = _UNSET, auth: Optional[Tuple[str, str]] = None, - verify: Optional[bool] = True, - cert: Optional[Union[str, Tuple[str, str]]] = None, - impersonate: Optional[BrowserTypeLiteral] = DEFAULT_CHROME, - http3: Optional[bool] = False, - stealthy_headers: Optional[bool] = True, + verify: Optional[bool] = _UNSET, + cert: Optional[Union[str, Tuple[str, str]]] = _UNSET, + impersonate: Optional[BrowserTypeLiteral] = _UNSET, + http3: Optional[bool] = _UNSET, + stealthy_headers: Optional[bool] = _UNSET, **kwargs, ) -> Union[Response, Awaitable[Response]]: """ @@ -495,23 +522,23 @@ class FetcherSession: url: str, data: Optional[Union[Dict, str]] = None, json: Optional[Union[Dict, List]] = None, - headers: Optional[Mapping[str, Optional[str]]] = None, + headers: Optional[Mapping[str, Optional[str]]] = _UNSET, params: Optional[Union[Dict, List, Tuple]] = None, cookies: Optional[CookieTypes] = None, - timeout: Optional[Union[int, float]] = 30, - follow_redirects: Optional[bool] = True, - max_redirects: Optional[int] = 30, - retries: Optional[int] = 3, - retry_delay: Optional[int] = 1, - proxies: Optional[ProxySpec] = None, - proxy: Optional[str] = None, - proxy_auth: Optional[Tuple[str, str]] = None, + timeout: Optional[Union[int, float]] = _UNSET, + follow_redirects: Optional[bool] = _UNSET, + max_redirects: Optional[int] = _UNSET, + retries: Optional[int] = _UNSET, + retry_delay: Optional[int] = _UNSET, + proxies: Optional[ProxySpec] = _UNSET, + proxy: Optional[str] = _UNSET, + proxy_auth: Optional[Tuple[str, str]] = _UNSET, auth: Optional[Tuple[str, str]] = None, - verify: Optional[bool] = True, - cert: Optional[Union[str, Tuple[str, str]]] = None, - impersonate: Optional[BrowserTypeLiteral] = DEFAULT_CHROME, - http3: Optional[bool] = False, - stealthy_headers: Optional[bool] = True, + verify: Optional[bool] = _UNSET, + cert: Optional[Union[str, Tuple[str, str]]] = _UNSET, + impersonate: Optional[BrowserTypeLiteral] = _UNSET, + http3: Optional[bool] = _UNSET, + stealthy_headers: Optional[bool] = _UNSET, **kwargs, ) -> Union[Response, Awaitable[Response]]: """ @@ -572,23 +599,23 @@ class FetcherSession: url: str, data: Optional[Union[Dict, str]] = None, json: Optional[Union[Dict, List]] = None, - headers: Optional[Mapping[str, Optional[str]]] = None, + headers: Optional[Mapping[str, Optional[str]]] = _UNSET, params: Optional[Union[Dict, List, Tuple]] = None, cookies: Optional[CookieTypes] = None, - timeout: Optional[Union[int, float]] = 30, - follow_redirects: Optional[bool] = True, - max_redirects: Optional[int] = 30, - retries: Optional[int] = 3, - retry_delay: Optional[int] = 1, - proxies: Optional[ProxySpec] = None, - proxy: Optional[str] = None, - proxy_auth: Optional[Tuple[str, str]] = None, + timeout: Optional[Union[int, float]] = _UNSET, + follow_redirects: Optional[bool] = _UNSET, + max_redirects: Optional[int] = _UNSET, + retries: Optional[int] = _UNSET, + retry_delay: Optional[int] = _UNSET, + proxies: Optional[ProxySpec] = _UNSET, + proxy: Optional[str] = _UNSET, + proxy_auth: Optional[Tuple[str, str]] = _UNSET, auth: Optional[Tuple[str, str]] = None, - verify: Optional[bool] = True, - cert: Optional[Union[str, Tuple[str, str]]] = None, - impersonate: Optional[BrowserTypeLiteral] = DEFAULT_CHROME, - http3: Optional[bool] = False, - stealthy_headers: Optional[bool] = True, + verify: Optional[bool] = _UNSET, + cert: Optional[Union[str, Tuple[str, str]]] = _UNSET, + impersonate: Optional[BrowserTypeLiteral] = _UNSET, + http3: Optional[bool] = _UNSET, + stealthy_headers: Optional[bool] = _UNSET, **kwargs, ) -> Union[Response, Awaitable[Response]]: """ @@ -667,22 +694,22 @@ class AsyncFetcherClient: async def get( url: str, params: Optional[Union[Dict, List, Tuple]] = None, - headers: Optional[Mapping[str, Optional[str]]] = None, + headers: Optional[Mapping[str, Optional[str]]] = _UNSET, cookies: Optional[CookieTypes] = None, - timeout: Optional[Union[int, float]] = 30, - follow_redirects: Optional[bool] = True, - max_redirects: Optional[int] = 30, - retries: Optional[int] = 3, - retry_delay: Optional[int] = 1, - proxies: Optional[ProxySpec] = None, - proxy: Optional[str] = None, - proxy_auth: Optional[Tuple[str, str]] = None, + timeout: Optional[Union[int, float]] = _UNSET, + follow_redirects: Optional[bool] = _UNSET, + max_redirects: Optional[int] = _UNSET, + retries: Optional[int] = _UNSET, + retry_delay: Optional[int] = _UNSET, + proxies: Optional[ProxySpec] = _UNSET, + proxy: Optional[str] = _UNSET, + proxy_auth: Optional[Tuple[str, str]] = _UNSET, auth: Optional[Tuple[str, str]] = None, - verify: Optional[bool] = True, - cert: Optional[Union[str, Tuple[str, str]]] = None, - impersonate: Optional[BrowserTypeLiteral] = DEFAULT_CHROME, - stealthy_headers: Optional[bool] = True, - http3: Optional[bool] = False, + verify: Optional[bool] = _UNSET, + cert: Optional[Union[str, Tuple[str, str]]] = _UNSET, + impersonate: Optional[BrowserTypeLiteral] = _UNSET, + http3: Optional[bool] = _UNSET, + stealthy_headers: Optional[bool] = _UNSET, **kwargs, ) -> Response: """ @@ -739,23 +766,23 @@ class AsyncFetcherClient: url: str, data: Optional[Union[Dict, str]] = None, json: Optional[Union[Dict, List]] = None, - headers: Optional[Mapping[str, Optional[str]]] = None, + headers: Optional[Mapping[str, Optional[str]]] = _UNSET, params: Optional[Union[Dict, List, Tuple]] = None, cookies: Optional[CookieTypes] = None, - timeout: Optional[Union[int, float]] = 30, - follow_redirects: Optional[bool] = True, - max_redirects: Optional[int] = 30, - retries: Optional[int] = 3, - retry_delay: Optional[int] = 1, - proxies: Optional[ProxySpec] = None, - proxy: Optional[str] = None, - proxy_auth: Optional[Tuple[str, str]] = None, + timeout: Optional[Union[int, float]] = _UNSET, + follow_redirects: Optional[bool] = _UNSET, + max_redirects: Optional[int] = _UNSET, + retries: Optional[int] = _UNSET, + retry_delay: Optional[int] = _UNSET, + proxies: Optional[ProxySpec] = _UNSET, + proxy: Optional[str] = _UNSET, + proxy_auth: Optional[Tuple[str, str]] = _UNSET, auth: Optional[Tuple[str, str]] = None, - verify: Optional[bool] = True, - cert: Optional[Union[str, Tuple[str, str]]] = None, - impersonate: Optional[BrowserTypeLiteral] = DEFAULT_CHROME, - stealthy_headers: Optional[bool] = True, - http3: Optional[bool] = False, + verify: Optional[bool] = _UNSET, + cert: Optional[Union[str, Tuple[str, str]]] = _UNSET, + impersonate: Optional[BrowserTypeLiteral] = _UNSET, + http3: Optional[bool] = _UNSET, + stealthy_headers: Optional[bool] = _UNSET, **kwargs, ) -> Response: """ @@ -816,23 +843,23 @@ class AsyncFetcherClient: url: str, data: Optional[Union[Dict, str]] = None, json: Optional[Union[Dict, List]] = None, - headers: Optional[Mapping[str, Optional[str]]] = None, + headers: Optional[Mapping[str, Optional[str]]] = _UNSET, params: Optional[Union[Dict, List, Tuple]] = None, cookies: Optional[CookieTypes] = None, - timeout: Optional[Union[int, float]] = 30, - follow_redirects: Optional[bool] = True, - max_redirects: Optional[int] = 30, - retries: Optional[int] = 3, - retry_delay: Optional[int] = 1, - proxies: Optional[ProxySpec] = None, - proxy: Optional[str] = None, - proxy_auth: Optional[Tuple[str, str]] = None, + timeout: Optional[Union[int, float]] = _UNSET, + follow_redirects: Optional[bool] = _UNSET, + max_redirects: Optional[int] = _UNSET, + retries: Optional[int] = _UNSET, + retry_delay: Optional[int] = _UNSET, + proxies: Optional[ProxySpec] = _UNSET, + proxy: Optional[str] = _UNSET, + proxy_auth: Optional[Tuple[str, str]] = _UNSET, auth: Optional[Tuple[str, str]] = None, - verify: Optional[bool] = True, - cert: Optional[Union[str, Tuple[str, str]]] = None, - impersonate: Optional[BrowserTypeLiteral] = DEFAULT_CHROME, - stealthy_headers: Optional[bool] = True, - http3: Optional[bool] = False, + verify: Optional[bool] = _UNSET, + cert: Optional[Union[str, Tuple[str, str]]] = _UNSET, + impersonate: Optional[BrowserTypeLiteral] = _UNSET, + http3: Optional[bool] = _UNSET, + stealthy_headers: Optional[bool] = _UNSET, **kwargs, ) -> Response: """ @@ -893,23 +920,23 @@ class AsyncFetcherClient: url: str, data: Optional[Union[Dict, str]] = None, json: Optional[Union[Dict, List]] = None, - headers: Optional[Mapping[str, Optional[str]]] = None, + headers: Optional[Mapping[str, Optional[str]]] = _UNSET, params: Optional[Union[Dict, List, Tuple]] = None, cookies: Optional[CookieTypes] = None, - timeout: Optional[Union[int, float]] = 30, - follow_redirects: Optional[bool] = True, - max_redirects: Optional[int] = 30, - retries: Optional[int] = 3, - retry_delay: Optional[int] = 1, - proxies: Optional[ProxySpec] = None, - proxy: Optional[str] = None, - proxy_auth: Optional[Tuple[str, str]] = None, + timeout: Optional[Union[int, float]] = _UNSET, + follow_redirects: Optional[bool] = _UNSET, + max_redirects: Optional[int] = _UNSET, + retries: Optional[int] = _UNSET, + retry_delay: Optional[int] = _UNSET, + proxies: Optional[ProxySpec] = _UNSET, + proxy: Optional[str] = _UNSET, + proxy_auth: Optional[Tuple[str, str]] = _UNSET, auth: Optional[Tuple[str, str]] = None, - verify: Optional[bool] = True, - cert: Optional[Union[str, Tuple[str, str]]] = None, - impersonate: Optional[BrowserTypeLiteral] = DEFAULT_CHROME, - stealthy_headers: Optional[bool] = True, - http3: Optional[bool] = False, + verify: Optional[bool] = _UNSET, + cert: Optional[Union[str, Tuple[str, str]]] = _UNSET, + impersonate: Optional[BrowserTypeLiteral] = _UNSET, + http3: Optional[bool] = _UNSET, + stealthy_headers: Optional[bool] = _UNSET, **kwargs, ) -> Response: """