diff --git a/scrapling/engines/_browsers/_validators.py b/scrapling/engines/_browsers/_validators.py index 1c2dbbc..448f0de 100644 --- a/scrapling/engines/_browsers/_validators.py +++ b/scrapling/engines/_browsers/_validators.py @@ -13,7 +13,6 @@ from scrapling.core._types import ( Callable, Iterable, SelectorWaitStates, - cast, overload, ) from scrapling.engines.toolbelt.navigation import construct_proxy_dict @@ -215,7 +214,7 @@ def validate_fetch(params: List[Tuple], sentinel=None) -> _fetch_params: result[arg] = session_value if overrides: - overrides = cast(FetchConfig, validate(overrides, FetchConfig)).to_dict() + overrides = validate(overrides, FetchConfig).to_dict() overrides.update(result) return _fetch_params(**overrides) diff --git a/scrapling/engines/static.py b/scrapling/engines/static.py index d49b5da..6833982 100644 --- a/scrapling/engines/static.py +++ b/scrapling/engines/static.py @@ -1,9 +1,9 @@ +from abc import ABC from time import sleep as time_sleep from asyncio import sleep as asyncio_sleep from curl_cffi.curl import CurlError from curl_cffi import CurlHttpVersion -from curl_cffi.requests.impersonate import DEFAULT_CHROME from curl_cffi.requests import ( ProxySpec, CookieTypes, @@ -22,7 +22,6 @@ from scrapling.core._types import ( Awaitable, List, Any, - cast, ) from .toolbelt.custom import Response @@ -30,22 +29,14 @@ from .toolbelt.convertor import ResponseFactory from .toolbelt.fingerprints import generate_convincing_referer, generate_headers, __default_useragent__ _UNSET: Any = object() +_NO_SESSION: Any = object() -class FetcherSession: - """ - A context manager that provides configured Fetcher sessions. - - When this manager is used in a 'with' or 'async with' block, - it yields a new session configured with the manager's defaults. - A single instance of this manager should ideally be used for one active - session at a time (or sequentially). Re-entering a context with the - same manager instance while a session is already active is disallowed. - """ - +class _ConfigurationLogic(ABC): + # Core Logic Handler (Internal Engine) def __init__( self, - impersonate: Optional[BrowserTypeLiteral] = DEFAULT_CHROME, + impersonate: Optional[BrowserTypeLiteral] = "chrome", http3: Optional[bool] = False, stealthy_headers: Optional[bool] = True, proxies: Optional[Dict[str, str]] = None, @@ -79,185 +70,191 @@ class FetcherSession: :param cert: Tuple of (cert, key) filenames for the client certificate. :param selector_config: Arguments passed when creating the final Selector class. """ - self.default_impersonate = impersonate - self.stealth = stealthy_headers - self.default_proxies = proxies or {} - self.default_proxy = proxy or None - self.default_proxy_auth = proxy_auth or None - self.default_timeout = timeout - self.default_headers = headers or {} - self.default_retries = retries - self.default_retry_delay = retry_delay - self.default_follow_redirects = follow_redirects - self.default_max_redirects = max_redirects - self.default_verify = verify - self.default_cert = cert - self.default_http3 = http3 + self._default_impersonate = impersonate + self._stealth = stealthy_headers + self._default_proxies = proxies or {} + self._default_proxy = proxy or None + self._default_proxy_auth = proxy_auth or None + self._default_timeout = timeout + self._default_headers = headers or {} + self._default_retries = retries + self._default_retry_delay = retry_delay + self._default_follow_redirects = follow_redirects + self._default_max_redirects = max_redirects + self._default_verify = verify + self._default_cert = cert + self._default_http3 = http3 self.selector_config = selector_config or {} - self._curl_session: Optional[CurlSession] = None - self._async_curl_session: Optional[AsyncCurlSession] = None + @staticmethod + def _get_with_precedence(request_val: Any, default_val: Any) -> Any: + """Get value with request-level priority over session-level""" + return request_val if request_val is not _UNSET else default_val - def _merge_request_args(self, **kwargs) -> Dict[str, Any]: + def _merge_request_args(self, **method_kwargs) -> Dict[str, Any]: """Merge request-specific arguments with default session arguments.""" - url = kwargs.pop("url") - request_args = {} - - headers = self.get_with_precedence(kwargs, "headers", self.default_headers) - stealth = self.get_with_precedence(kwargs, "stealth", self.stealth) - impersonate = self.get_with_precedence(kwargs, "impersonate", self.default_impersonate) - - if self.get_with_precedence(kwargs, "http3", self.default_http3): # pragma: no cover - request_args["http_version"] = CurlHttpVersion.V3ONLY + url = method_kwargs.pop("url") + impersonate = self._get_with_precedence(method_kwargs.get("impersonate"), self._default_impersonate) + final_args = { + "url": url, + # Curl automatically generates the suitable browser headers when you use `impersonate` + "headers": self._headers_job( + url, + self._get_with_precedence(method_kwargs.pop("headers"), self._default_headers), + self._get_with_precedence(method_kwargs.pop("stealth"), self._stealth), + bool(impersonate), + ), + "proxies": self._get_with_precedence(method_kwargs.pop("proxies"), self._default_proxies), + "proxy": self._get_with_precedence(method_kwargs.pop("proxy"), self._default_proxy), + "proxy_auth": self._get_with_precedence(method_kwargs.pop("proxy_auth"), self._default_proxy_auth), + "timeout": self._get_with_precedence(method_kwargs.pop("timeout"), self._default_timeout), + "allow_redirects": self._get_with_precedence( + method_kwargs.pop("follow_redirects"), self._default_follow_redirects + ), + "max_redirects": self._get_with_precedence(method_kwargs.pop("max_redirects"), self._default_max_redirects), + "verify": self._get_with_precedence(method_kwargs.pop("verify"), self._default_verify), + "cert": self._get_with_precedence(method_kwargs.pop("cert"), self._default_cert), + "impersonate": impersonate, + **{ + k: v + for k, v in method_kwargs.items() + if v + not in ( + _UNSET, + None, + ) + }, # Add any remaining parameters (after all known ones are popped) + } + if self._get_with_precedence(method_kwargs.pop("http3"), self._default_http3): # pragma: no cover + final_args["http_version"] = CurlHttpVersion.V3ONLY if impersonate: log.warning( "The argument `http3` might cause errors if used with `impersonate` argument, try switching it off if you encounter any curl errors." ) - request_args.update( - { - "url": url, - # Curl automatically generates the suitable browser headers when you use `impersonate` - "headers": self._headers_job(url, headers, stealth, bool(impersonate)), - "proxies": self.get_with_precedence(kwargs, "proxies", self.default_proxies), - "proxy": self.get_with_precedence(kwargs, "proxy", self.default_proxy), - "proxy_auth": self.get_with_precedence(kwargs, "proxy_auth", self.default_proxy_auth), - "timeout": self.get_with_precedence(kwargs, "timeout", self.default_timeout), - "allow_redirects": self.get_with_precedence(kwargs, "allow_redirects", self.default_follow_redirects), - "max_redirects": self.get_with_precedence(kwargs, "max_redirects", self.default_max_redirects), - "verify": self.get_with_precedence(kwargs, "verify", self.default_verify), - "cert": self.get_with_precedence(kwargs, "cert", self.default_cert), - "impersonate": impersonate, - **{ - k: v - for k, v in kwargs.items() - if v - not in ( - _UNSET, - None, - ) - }, # Add any remaining parameters (after all known ones are popped) - } - ) - return request_args + return final_args - def _headers_job( - self, - url, - headers: Optional[Dict], - stealth: Optional[bool], - impersonate_enabled: bool, - ) -> Dict: - """Adds useragent to headers if it doesn't exist, generates real headers and append it to current headers, and - finally generates a referer header that looks like if this request came from Google's search of the current URL's domain. - - :param headers: Current headers in the request if the user passed any - :param stealth: Whether to enable the `stealthy_headers` argument to this request or not. If `None`, it defaults to the session default value. - :param impersonate_enabled: Whether the browser impersonation is enabled or not. - :return: A dictionary of the new headers. + def _headers_job(self, url, headers: Dict, stealth: bool, impersonate_enabled: bool) -> Dict: """ - # Handle headers - if it was _UNSET, use default_headers - if headers is _UNSET: - headers = self.default_headers.copy() - else: - # Merge session headers with request headers, request takes precedence - headers = {**self.default_headers, **(headers or {})} - - headers_keys = set(map(str.lower, headers.keys())) + 1. Adds a useragent to the headers if it doesn't have one + 2. Generates real headers and append them to current headers + 3. Generates a referer header that looks like as if this request came from a Google's search of the current URL's domain. + """ + # Merge session headers with request headers, request takes precedence (if it was set) + final_headers = {**self._default_headers, **(headers if headers is not _UNSET else {})} + headers_keys = {k.lower() for k in final_headers} if stealth: if "referer" not in headers_keys: - headers.update({"referer": generate_convincing_referer(url)}) + final_headers["referer"] = generate_convincing_referer(url) - if impersonate_enabled: # Curl will generate the suitable headers - return headers - - extra_headers = generate_headers(browser_mode=False) - # Don't overwrite user-supplied headers - extra_headers = {key: value for key, value in extra_headers.items() if key.lower() not in headers_keys} - headers.update(extra_headers) + if not impersonate_enabled: # Curl will generate the suitable headers + extra_headers = generate_headers(browser_mode=False) + final_headers.update( + {k: v for k, v in extra_headers.items() if k.lower() not in headers_keys} + ) # Don't overwrite user-supplied headers elif "user-agent" not in headers_keys and not impersonate_enabled: - headers["User-Agent"] = __default_useragent__ - log.debug(f"Can't find useragent in headers so '{headers['User-Agent']}' was used.") + final_headers["User-Agent"] = __default_useragent__ + log.debug(f"Can't find useragent in headers so '{final_headers['User-Agent']}' was used.") - return headers + return final_headers - def __enter__(self) -> "FetcherClient": + +class _SyncSessionLogic(_ConfigurationLogic): + def __init__( + self, + impersonate: Optional[BrowserTypeLiteral] = "chrome", + http3: Optional[bool] = False, + stealthy_headers: Optional[bool] = True, + proxies: Optional[Dict[str, str]] = None, + proxy: Optional[str] = None, + proxy_auth: Optional[Tuple[str, str]] = None, + timeout: Optional[int | float] = 30, + headers: Optional[Dict[str, str]] = None, + retries: Optional[int] = 3, + retry_delay: Optional[int] = 1, + follow_redirects: bool = True, + max_redirects: int = 30, + verify: bool = True, + cert: Optional[str | Tuple[str, str]] = None, + selector_config: Optional[Dict] = None, + ): + super().__init__( + impersonate, + http3, + stealthy_headers, + proxies, + proxy, + proxy_auth, + timeout, + headers, + retries, + retry_delay, + follow_redirects, + max_redirects, + verify, + cert, + selector_config, + ) + self._curl_session: Optional[CurlSession] = None + + def __enter__(self): """Creates and returns a new synchronous Fetcher Session""" if self._curl_session: - raise RuntimeError( - "This FetcherSession instance already has an active synchronous session. " - "Create a new FetcherSession instance for a new independent session, " - "or use the current instance sequentially after the previous context has exited." - ) - if self._async_curl_session: # Prevent mixing if async is active from this instance - raise RuntimeError( - "This FetcherSession instance has an active asynchronous session. " - "Cannot enter a synchronous context simultaneously with the same manager instance." - ) + raise RuntimeError("This FetcherSession instance already has an active synchronous session.") self._curl_session = CurlSession() - return cast("FetcherClient", self) + return self def __exit__(self, exc_type, exc_val, exc_tb): """Closes the active synchronous session managed by this instance, if any.""" + # For type checking (not accessed error) + _ = ( + exc_type, + exc_val, + exc_tb, + ) if self._curl_session: self._curl_session.close() self._curl_session = None - async def __aenter__(self) -> "AsyncFetcherClient": - """Creates and returns a new asynchronous Session.""" - if self._async_curl_session: - raise RuntimeError( - "This FetcherSession instance already has an active asynchronous session. " - "Create a new FetcherSession instance for a new independent session, " - "or use the current instance sequentially after the previous context has exited." - ) - if self._curl_session: # Prevent mixing if sync is active from this instance - raise RuntimeError( - "This FetcherSession instance has an active synchronous session. " - "Cannot enter an asynchronous context simultaneously with the same manager instance." - ) - - self._async_curl_session = AsyncCurlSession() - return cast("AsyncFetcherClient", self) - - async def __aexit__(self, exc_type, exc_val, exc_tb): - """Closes the active asynchronous session managed by this instance, if any.""" - if self._async_curl_session: - await self._async_curl_session.close() - self._async_curl_session = None - def __make_request( self, method: SUPPORTED_HTTP_METHODS, - request_args: Dict[str, Any], - max_retries: int, - retry_delay: int, - selector_config: Dict, + stealth: Optional[bool] = None, + **kwargs, ) -> Response: """ Perform an HTTP request using the configured session. :param method: HTTP method to be used, supported methods are ["GET", "POST", "PUT", "DELETE"] - :param request_args: Arguments to be passed to the session's `request()` method. - :param max_retries: Maximum number of retries for the request. - :param retry_delay: Number of seconds to wait between retries. - :param selector_config: Arguments passed when creating the final Selector class. - :return: A `Response` object for synchronous requests or an awaitable for asynchronous. + :param stealth: Whether to enable the `stealthy_headers` argument to this request or not. If `None`, it defaults to the session default value. + :param kwargs: Additional request-specific arguments. + :return: A `Response` object. """ + stealth = self._stealth if stealth is None else stealth + + selector_config = kwargs.pop("selector_config", {}) or self.selector_config + max_retries = self._get_with_precedence(kwargs.pop("retries"), self._default_retries) + retry_delay = self._get_with_precedence(kwargs.pop("retry_delay"), self._default_retry_delay) + request_args = self._merge_request_args(stealth=stealth, **kwargs) + session = self._curl_session - if session is True and not any((self.__enter__, self.__exit__, self.__aenter__, self.__aexit__)): + one_off_request = False + if session is _NO_SESSION and self.__enter__ is None: # For usage inside FetcherClient # It turns out `curl_cffi` caches impersonation state, so if you turned it off, then on then off, it won't be off on the last time. session = CurlSession() + one_off_request = True if session: for attempt in range(max_retries): try: response = session.request(method, **request_args) - # response.raise_for_status() # Retry responses with a status code between 200-400 - return ResponseFactory.from_http_request(response, selector_config) + result = ResponseFactory.from_http_request(response, selector_config) + if one_off_request: + session.close() + return result except CurlError as e: # pragma: no cover if attempt < max_retries - 1: log.error(f"Attempt {attempt + 1} failed: {e}. Retrying in {retry_delay} seconds...") @@ -265,86 +262,12 @@ class FetcherSession: else: log.error(f"Failed after {max_retries} attempts: {e}") raise # Raise the exception if all retries fail + finally: + if session and one_off_request: + session.close() raise RuntimeError("No active session available.") # pragma: no cover - async def __make_async_request( - self, - method: SUPPORTED_HTTP_METHODS, - request_args: Dict[str, Any], - max_retries: int, - retry_delay: int, - selector_config: Dict, - ) -> Response: - """ - Perform an HTTP request using the configured session. - - :param method: HTTP method to be used, supported methods are ["GET", "POST", "PUT", "DELETE"] - :param request_args: Arguments to be passed to the session's `request()` method. - :param max_retries: Maximum number of retries for the request. - :param retry_delay: Number of seconds to wait between retries. - :param selector_config: Arguments passed when creating the final Selector class. - :return: A `Response` object for synchronous requests or an awaitable for asynchronous. - """ - session = self._async_curl_session - if session is True and not any((self.__enter__, self.__exit__, self.__aenter__, self.__aexit__)): - # For usage inside the ` AsyncFetcherClient ` class, and that's for several reasons - # 1. It turns out `curl_cffi` caches impersonation state, so if you turned it off, then on then off, it won't be off on the last time. - # 2. `curl_cffi` doesn't support making async requests without sessions - # 3. Using a single session for many requests at the same time in async doesn't sit well with curl_cffi. - session = AsyncCurlSession() - - if session: - for attempt in range(max_retries): - try: - response = await session.request(method, **request_args) - # response.raise_for_status() # Retry responses with a status code between 200-400 - return ResponseFactory.from_http_request(response, selector_config) - except CurlError as e: # pragma: no cover - if attempt < max_retries - 1: - log.error(f"Attempt {attempt + 1} failed: {e}. Retrying in {retry_delay} seconds...") - await asyncio_sleep(retry_delay) - else: - log.error(f"Failed after {max_retries} attempts: {e}") - raise # Raise the exception if all retries fail - - raise RuntimeError("No active session available.") # pragma: no cover - - @staticmethod - def get_with_precedence(kwargs, key, default_value): - """Get value with request-level priority over session-level""" - request_value = kwargs.pop(key, _UNSET) - return request_value if request_value is not _UNSET else default_value - - def __prepare_and_dispatch( - self, - method: SUPPORTED_HTTP_METHODS, - stealth: Optional[bool] = None, - **kwargs, - ) -> Response | Awaitable[Response]: - """ - Internal dispatcher. Prepares arguments and calls sync or async request helper. - - :param method: HTTP method to be used, supported methods are ["GET", "POST", "PUT", "DELETE"] - :param stealth: Whether to enable the `stealthy_headers` argument to this request or not. If `None`, it defaults to the session default value. - :param url: Target URL for the request. - :param kwargs: Additional request-specific arguments. - :return: A `Response` object for synchronous requests or an awaitable for asynchronous. - """ - stealth = self.stealth if stealth is None else stealth - - selector_config = kwargs.pop("selector_config", {}) or self.selector_config - max_retries = self.get_with_precedence(kwargs, "retries", self.default_retries) - retry_delay = self.get_with_precedence(kwargs, "retry_delay", self.default_retry_delay) - request_args = self._merge_request_args(stealth=stealth, **kwargs) - if self._curl_session: - return self.__make_request(method, request_args, max_retries, retry_delay, selector_config) - elif self._async_curl_session: - # The returned value is a Coroutine - return self.__make_async_request(method, request_args, max_retries, retry_delay, selector_config) - - raise RuntimeError("No active session available.") - def get( self, url: str, @@ -366,7 +289,7 @@ class FetcherSession: http3: Optional[bool] = _UNSET, stealthy_headers: Optional[bool] = _UNSET, **kwargs, - ) -> Response | Awaitable[Response]: + ) -> Response: """ Perform a GET request. @@ -390,29 +313,31 @@ class FetcherSession: :param http3: Whether to use HTTP3. Defaults to False. It might be problematic if used it with `impersonate`. :param stealthy_headers: If enabled (default), it creates and adds real browser headers. It also sets the referer header as if this request came from a Google search of URL's domain. :param kwargs: Additional keyword arguments to pass to the [`curl_cffi.requests.Session().request()`, `curl_cffi.requests.AsyncSession().request()`] method. - :return: A `Response` object or an awaitable for async. + :return: A `Response` object. """ - request_args = { - "url": url, - "params": params, - "headers": headers, - "cookies": cookies, - "timeout": timeout, - "retry_delay": retry_delay, - "allow_redirects": follow_redirects, - "max_redirects": max_redirects, - "retries": retries, - "proxies": proxies, - "proxy": proxy, - "proxy_auth": proxy_auth, - "auth": auth, - "verify": verify, - "cert": cert, - "impersonate": impersonate, - "http3": http3, - **kwargs, - } - return self.__prepare_and_dispatch("GET", stealth=stealthy_headers, **request_args) + method_args = {k: v for k, v in locals().items() if k not in ("self", "stealthy_headers", "kwargs")} + method_args.update(kwargs) + # For type checking (not accessed error) + _ = ( + url, + params, + headers, + cookies, + timeout, + follow_redirects, + max_redirects, + retries, + retry_delay, + proxies, + proxy, + proxy_auth, + auth, + verify, + cert, + impersonate, + http3, + ) + return self.__make_request("GET", stealth=stealthy_headers, **method_args) def post( self, @@ -437,57 +362,59 @@ class FetcherSession: http3: Optional[bool] = _UNSET, stealthy_headers: Optional[bool] = _UNSET, **kwargs, - ) -> Response | Awaitable[Response]: + ) -> Response: """ Perform a POST request. :param url: Target URL for the request. :param data: Form data to include in the request body. :param json: A JSON serializable object to include in the body of the request. - :param headers: Headers to include in the request. :param params: Query string parameters for the request. + :param headers: Headers to include in the request. :param cookies: Cookies to use in the request. :param timeout: Number of seconds to wait before timing out. :param follow_redirects: Whether to follow redirects. Defaults to True. :param max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited. :param retries: Number of retry attempts. Defaults to 3. :param retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second. - :param proxies: Dict of proxies to use. Format: {"http": proxy_url, "https": proxy_url}. + :param proxies: Dict of proxies to use. :param proxy: Proxy URL to use. Format: "http://username:password@localhost:8030". Cannot be used together with the `proxies` parameter. :param proxy_auth: HTTP basic auth for proxy, tuple of (username, password). :param auth: HTTP basic auth tuple of (username, password). Only basic auth is supported. - :param verify: Whether to verify HTTPS certificates. Defaults to True. + :param verify: Whether to verify HTTPS certificates. :param cert: Tuple of (cert, key) filenames for the client certificate. :param impersonate: Browser version to impersonate. Automatically defaults to the latest available Chrome version. :param http3: Whether to use HTTP3. Defaults to False. It might be problematic if used it with `impersonate`. :param stealthy_headers: If enabled (default), it creates and adds real browser headers. It also sets the referer header as if this request came from a Google search of URL's domain. :param kwargs: Additional keyword arguments to pass to the [`curl_cffi.requests.Session().request()`, `curl_cffi.requests.AsyncSession().request()`] method. - :return: A `Response` object or an awaitable for async. + :return: A `Response` object. """ - request_args = { - "url": url, - "data": data, - "json": json, - "headers": headers, - "params": params, - "cookies": cookies, - "timeout": timeout, - "retry_delay": retry_delay, - "proxy": proxy, - "impersonate": impersonate, - "allow_redirects": follow_redirects, - "max_redirects": max_redirects, - "retries": retries, - "proxies": proxies, - "proxy_auth": proxy_auth, - "auth": auth, - "verify": verify, - "cert": cert, - "http3": http3, - **kwargs, - } - return self.__prepare_and_dispatch("POST", stealth=stealthy_headers, **request_args) + method_args = {k: v for k, v in locals().items() if k not in ("self", "stealthy_headers", "kwargs")} + method_args.update(kwargs) + # For type checking (not accessed error) + _ = ( + url, + params, + headers, + data, + json, + cookies, + timeout, + follow_redirects, + max_redirects, + retries, + retry_delay, + proxies, + proxy, + proxy_auth, + auth, + verify, + cert, + impersonate, + http3, + ) + return self.__make_request("POST", stealth=stealthy_headers, **method_args) def put( self, @@ -512,57 +439,59 @@ class FetcherSession: http3: Optional[bool] = _UNSET, stealthy_headers: Optional[bool] = _UNSET, **kwargs, - ) -> Response | Awaitable[Response]: + ) -> Response: """ Perform a PUT request. :param url: Target URL for the request. :param data: Form data to include in the request body. :param json: A JSON serializable object to include in the body of the request. - :param headers: Headers to include in the request. :param params: Query string parameters for the request. + :param headers: Headers to include in the request. :param cookies: Cookies to use in the request. :param timeout: Number of seconds to wait before timing out. :param follow_redirects: Whether to follow redirects. Defaults to True. :param max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited. :param retries: Number of retry attempts. Defaults to 3. :param retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second. - :param proxies: Dict of proxies to use. Format: {"http": proxy_url, "https": proxy_url}. + :param proxies: Dict of proxies to use. :param proxy: Proxy URL to use. Format: "http://username:password@localhost:8030". Cannot be used together with the `proxies` parameter. :param proxy_auth: HTTP basic auth for proxy, tuple of (username, password). :param auth: HTTP basic auth tuple of (username, password). Only basic auth is supported. - :param verify: Whether to verify HTTPS certificates. Defaults to True. + :param verify: Whether to verify HTTPS certificates. :param cert: Tuple of (cert, key) filenames for the client certificate. :param impersonate: Browser version to impersonate. Automatically defaults to the latest available Chrome version. :param http3: Whether to use HTTP3. Defaults to False. It might be problematic if used it with `impersonate`. :param stealthy_headers: If enabled (default), it creates and adds real browser headers. It also sets the referer header as if this request came from a Google search of URL's domain. :param kwargs: Additional keyword arguments to pass to the [`curl_cffi.requests.Session().request()`, `curl_cffi.requests.AsyncSession().request()`] method. - :return: A `Response` object or an awaitable for async. + :return: A `Response` object. """ - request_args = { - "url": url, - "data": data, - "json": json, - "headers": headers, - "params": params, - "cookies": cookies, - "timeout": timeout, - "retry_delay": retry_delay, - "proxy": proxy, - "impersonate": impersonate, - "allow_redirects": follow_redirects, - "max_redirects": max_redirects, - "retries": retries, - "proxies": proxies, - "proxy_auth": proxy_auth, - "auth": auth, - "verify": verify, - "cert": cert, - "http3": http3, - **kwargs, - } - return self.__prepare_and_dispatch("PUT", stealth=stealthy_headers, **request_args) + method_args = {k: v for k, v in locals().items() if k not in ("self", "stealthy_headers", "kwargs")} + method_args.update(kwargs) + # For type checking (not accessed error) + _ = ( + url, + params, + headers, + data, + json, + cookies, + timeout, + follow_redirects, + max_redirects, + retries, + retry_delay, + proxies, + proxy, + proxy_auth, + auth, + verify, + cert, + impersonate, + http3, + ) + return self.__make_request("PUT", stealth=stealthy_headers, **method_args) def delete( self, @@ -587,282 +516,173 @@ class FetcherSession: http3: Optional[bool] = _UNSET, stealthy_headers: Optional[bool] = _UNSET, **kwargs, - ) -> Response | Awaitable[Response]: + ) -> Response: """ Perform a DELETE request. :param url: Target URL for the request. :param data: Form data to include in the request body. :param json: A JSON serializable object to include in the body of the request. - :param headers: Headers to include in the request. :param params: Query string parameters for the request. + :param headers: Headers to include in the request. :param cookies: Cookies to use in the request. :param timeout: Number of seconds to wait before timing out. :param follow_redirects: Whether to follow redirects. Defaults to True. :param max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited. :param retries: Number of retry attempts. Defaults to 3. :param retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second. - :param proxies: Dict of proxies to use. Format: {"http": proxy_url, "https": proxy_url}. + :param proxies: Dict of proxies to use. :param proxy: Proxy URL to use. Format: "http://username:password@localhost:8030". Cannot be used together with the `proxies` parameter. :param proxy_auth: HTTP basic auth for proxy, tuple of (username, password). :param auth: HTTP basic auth tuple of (username, password). Only basic auth is supported. - :param verify: Whether to verify HTTPS certificates. Defaults to True. + :param verify: Whether to verify HTTPS certificates. :param cert: Tuple of (cert, key) filenames for the client certificate. :param impersonate: Browser version to impersonate. Automatically defaults to the latest available Chrome version. :param http3: Whether to use HTTP3. Defaults to False. It might be problematic if used it with `impersonate`. :param stealthy_headers: If enabled (default), it creates and adds real browser headers. It also sets the referer header as if this request came from a Google search of URL's domain. :param kwargs: Additional keyword arguments to pass to the [`curl_cffi.requests.Session().request()`, `curl_cffi.requests.AsyncSession().request()`] method. - :return: A `Response` object or an awaitable for async. + :return: A `Response` object. """ - request_args = { - "url": url, - # Careful of sending a body in a DELETE request, it might cause some websites to reject the request as per https://www.rfc-editor.org/rfc/rfc7231#section-4.3.5, - # But some websites accept it, it depends on the implementation used. - "data": data, - "json": json, - "headers": headers, - "params": params, - "cookies": cookies, - "timeout": timeout, - "retry_delay": retry_delay, - "proxy": proxy, - "impersonate": impersonate, - "allow_redirects": follow_redirects, - "max_redirects": max_redirects, - "retries": retries, - "proxies": proxies, - "proxy_auth": proxy_auth, - "auth": auth, - "verify": verify, - "cert": cert, - "http3": http3, - **kwargs, - } - return self.__prepare_and_dispatch("DELETE", stealth=stealthy_headers, **request_args) + # Careful of sending a body in a DELETE request, it might cause some websites to reject the request as per https://www.rfc-editor.org/rfc/rfc7231#section-4.3.5, + # But some websites accept it, it depends on the implementation used. + method_args = {k: v for k, v in locals().items() if k not in ("self", "stealthy_headers", "kwargs")} + method_args.update(kwargs) + # For type checking (not accessed error) + _ = ( + url, + params, + headers, + data, + json, + cookies, + timeout, + follow_redirects, + max_redirects, + retries, + retry_delay, + proxies, + proxy, + proxy_auth, + auth, + verify, + cert, + impersonate, + http3, + ) + return self.__make_request("DELETE", stealth=stealthy_headers, **method_args) -class FetcherClient(FetcherSession): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.__enter__: Any = None - self.__exit__: Any = None - self.__aenter__: Any = None - self.__aexit__: Any = None - self._curl_session: Any = True - - # Setting the correct return types for the type checking/autocompletion - def get( +class _ASyncSessionLogic(_ConfigurationLogic): + def __init__( self, - url: str, - params: Optional[Dict | List | Tuple] = None, - headers: Optional[Mapping[str, Optional[str]]] = _UNSET, - cookies: Optional[CookieTypes] = None, - timeout: Optional[int | float] = _UNSET, - follow_redirects: Optional[bool] = _UNSET, - max_redirects: Optional[int] = _UNSET, - retries: Optional[int] = _UNSET, - retry_delay: Optional[int] = _UNSET, - proxies: Optional[ProxySpec] = _UNSET, - proxy: Optional[str] = _UNSET, - proxy_auth: Optional[Tuple[str, str]] = _UNSET, - auth: Optional[Tuple[str, str]] = None, - verify: Optional[bool] = _UNSET, - cert: Optional[str | Tuple[str, str]] = _UNSET, - impersonate: Optional[BrowserTypeLiteral] = _UNSET, - http3: Optional[bool] = _UNSET, - stealthy_headers: Optional[bool] = _UNSET, + impersonate: Optional[BrowserTypeLiteral] = "chrome", + http3: Optional[bool] = False, + stealthy_headers: Optional[bool] = True, + proxies: Optional[Dict[str, str]] = None, + proxy: Optional[str] = None, + proxy_auth: Optional[Tuple[str, str]] = None, + timeout: Optional[int | float] = 30, + headers: Optional[Dict[str, str]] = None, + retries: Optional[int] = 3, + retry_delay: Optional[int] = 1, + follow_redirects: bool = True, + max_redirects: int = 30, + verify: bool = True, + cert: Optional[str | Tuple[str, str]] = None, + selector_config: Optional[Dict] = None, + ): + super().__init__( + impersonate, + http3, + stealthy_headers, + proxies, + proxy, + proxy_auth, + timeout, + headers, + retries, + retry_delay, + follow_redirects, + max_redirects, + verify, + cert, + selector_config, + ) + self._async_curl_session: Optional[AsyncCurlSession] = None + + async def __aenter__(self): + """Creates and returns a new asynchronous Session.""" + if self._async_curl_session: + raise RuntimeError("This FetcherSession instance already has an active asynchronous session.") + + self._async_curl_session = AsyncCurlSession() + return self + + async def __aexit__(self, exc_type, exc_val, exc_tb): + """Closes the active asynchronous session managed by this instance, if any.""" + # For type checking (not accessed error) + _ = ( + exc_type, + exc_val, + exc_tb, + ) + if self._async_curl_session: + await self._async_curl_session.close() + self._async_curl_session = None + + async def __make_request( + self, + method: SUPPORTED_HTTP_METHODS, + stealth: Optional[bool] = None, **kwargs, ) -> Response: - return cast( - Response, - super().get( - url, - params, - headers, - cookies, - timeout, - follow_redirects, - max_redirects, - retries, - retry_delay, - proxies, - proxy, - proxy_auth, - auth, - verify, - cert, - impersonate, - http3, - stealthy_headers, - **kwargs, - ), - ) + """ + Perform an HTTP request using the configured session. - def post( - self, - url: str, - data: Optional[Dict | str] = None, - json: Optional[Dict | List] = None, - headers: Optional[Mapping[str, Optional[str]]] = _UNSET, - params: Optional[Dict | List | Tuple] = None, - cookies: Optional[CookieTypes] = None, - timeout: Optional[int | float] = _UNSET, - follow_redirects: Optional[bool] = _UNSET, - max_redirects: Optional[int] = _UNSET, - retries: Optional[int] = _UNSET, - retry_delay: Optional[int] = _UNSET, - proxies: Optional[ProxySpec] = _UNSET, - proxy: Optional[str] = _UNSET, - proxy_auth: Optional[Tuple[str, str]] = _UNSET, - auth: Optional[Tuple[str, str]] = None, - verify: Optional[bool] = _UNSET, - cert: Optional[str | Tuple[str, str]] = _UNSET, - impersonate: Optional[BrowserTypeLiteral] = _UNSET, - http3: Optional[bool] = _UNSET, - stealthy_headers: Optional[bool] = _UNSET, - **kwargs, - ) -> Response: - return cast( - Response, - super().post( - url, - data, - json, - headers, - params, - cookies, - timeout, - follow_redirects, - max_redirects, - retries, - retry_delay, - proxies, - proxy, - proxy_auth, - auth, - verify, - cert, - impersonate, - http3, - stealthy_headers, - **kwargs, - ), - ) + :param method: HTTP method to be used, supported methods are ["GET", "POST", "PUT", "DELETE"] + :param stealth: Whether to enable the `stealthy_headers` argument to this request or not. If `None`, it defaults to the session default value. + :param kwargs: Additional request-specific arguments. + :return: A `Response` object. + """ + stealth = self._stealth if stealth is None else stealth - def put( - self, - url: str, - data: Optional[Dict | str] = None, - json: Optional[Dict | List] = None, - headers: Optional[Mapping[str, Optional[str]]] = _UNSET, - params: Optional[Dict | List | Tuple] = None, - cookies: Optional[CookieTypes] = None, - timeout: Optional[int | float] = _UNSET, - follow_redirects: Optional[bool] = _UNSET, - max_redirects: Optional[int] = _UNSET, - retries: Optional[int] = _UNSET, - retry_delay: Optional[int] = _UNSET, - proxies: Optional[ProxySpec] = _UNSET, - proxy: Optional[str] = _UNSET, - proxy_auth: Optional[Tuple[str, str]] = _UNSET, - auth: Optional[Tuple[str, str]] = None, - verify: Optional[bool] = _UNSET, - cert: Optional[str | Tuple[str, str]] = _UNSET, - impersonate: Optional[BrowserTypeLiteral] = _UNSET, - http3: Optional[bool] = _UNSET, - stealthy_headers: Optional[bool] = _UNSET, - **kwargs, - ) -> Response: - return cast( - Response, - super().put( - url, - data, - json, - headers, - params, - cookies, - timeout, - follow_redirects, - max_redirects, - retries, - retry_delay, - proxies, - proxy, - proxy_auth, - auth, - verify, - cert, - impersonate, - http3, - stealthy_headers, - **kwargs, - ), - ) + selector_config = kwargs.pop("selector_config", {}) or self.selector_config + max_retries = self._get_with_precedence(kwargs.pop("retries"), self._default_retries) + retry_delay = self._get_with_precedence(kwargs.pop("retry_delay"), self._default_retry_delay) + request_args = self._merge_request_args(stealth=stealth, **kwargs) - def delete( - self, - url: str, - data: Optional[Dict | str] = None, - json: Optional[Dict | List] = None, - headers: Optional[Mapping[str, Optional[str]]] = _UNSET, - params: Optional[Dict | List | Tuple] = None, - cookies: Optional[CookieTypes] = None, - timeout: Optional[int | float] = _UNSET, - follow_redirects: Optional[bool] = _UNSET, - max_redirects: Optional[int] = _UNSET, - retries: Optional[int] = _UNSET, - retry_delay: Optional[int] = _UNSET, - proxies: Optional[ProxySpec] = _UNSET, - proxy: Optional[str] = _UNSET, - proxy_auth: Optional[Tuple[str, str]] = _UNSET, - auth: Optional[Tuple[str, str]] = None, - verify: Optional[bool] = _UNSET, - cert: Optional[str | Tuple[str, str]] = _UNSET, - impersonate: Optional[BrowserTypeLiteral] = _UNSET, - http3: Optional[bool] = _UNSET, - stealthy_headers: Optional[bool] = _UNSET, - **kwargs, - ) -> Response: - return cast( - Response, - super().delete( - url, - data, - json, - headers, - params, - cookies, - timeout, - follow_redirects, - max_redirects, - retries, - retry_delay, - proxies, - proxy, - proxy_auth, - auth, - verify, - cert, - impersonate, - http3, - stealthy_headers, - **kwargs, - ), - ) + session = self._async_curl_session + one_off_request = False + if session is _NO_SESSION and self.__aenter__ is None: + # For usage inside the ` AsyncFetcherClient ` class, and that's for several reasons + # 1. It turns out `curl_cffi` caches impersonation state, so if you turned it off, then on then off, it won't be off on the last time. + # 2. `curl_cffi` doesn't support making async requests without sessions + # 3. Using a single session for many requests at the same time in async doesn't sit well with curl_cffi. + session = AsyncCurlSession() + one_off_request = True + if session: + for attempt in range(max_retries): + try: + response = await session.request(method, **request_args) + result = ResponseFactory.from_http_request(response, selector_config) + if one_off_request: + await session.close() + return result + except CurlError as e: # pragma: no cover + if attempt < max_retries - 1: + log.error(f"Attempt {attempt + 1} failed: {e}. Retrying in {retry_delay} seconds...") + await asyncio_sleep(retry_delay) + else: + log.error(f"Failed after {max_retries} attempts: {e}") + raise # Raise the exception if all retries fail + finally: + if session and one_off_request: + await session.close() -class AsyncFetcherClient(FetcherSession): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.__enter__: Any = None - self.__exit__: Any = None - self.__aenter__: Any = None - self.__aexit__: Any = None - self._async_curl_session: Any = True + raise RuntimeError("No active session available.") # pragma: no cover - # Setting the correct return types for the type checking/autocompletion def get( self, url: str, @@ -885,30 +705,54 @@ class AsyncFetcherClient(FetcherSession): stealthy_headers: Optional[bool] = _UNSET, **kwargs, ) -> Awaitable[Response]: - return cast( - Awaitable[Response], - super().get( - url, - params, - headers, - cookies, - timeout, - follow_redirects, - max_redirects, - retries, - retry_delay, - proxies, - proxy, - proxy_auth, - auth, - verify, - cert, - impersonate, - http3, - stealthy_headers, - **kwargs, - ), + """ + Perform a GET request. + + :param url: Target URL for the request. + :param params: Query string parameters for the request. + :param headers: Headers to include in the request. + :param cookies: Cookies to use in the request. + :param timeout: Number of seconds to wait before timing out. + :param follow_redirects: Whether to follow redirects. Defaults to True. + :param max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited. + :param retries: Number of retry attempts. Defaults to 3. + :param retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second. + :param proxies: Dict of proxies to use. + :param proxy: Proxy URL to use. Format: "http://username:password@localhost:8030". + Cannot be used together with the `proxies` parameter. + :param proxy_auth: HTTP basic auth for proxy, tuple of (username, password). + :param auth: HTTP basic auth tuple of (username, password). Only basic auth is supported. + :param verify: Whether to verify HTTPS certificates. + :param cert: Tuple of (cert, key) filenames for the client certificate. + :param impersonate: Browser version to impersonate. Automatically defaults to the latest available Chrome version. + :param http3: Whether to use HTTP3. Defaults to False. It might be problematic if used it with `impersonate`. + :param stealthy_headers: If enabled (default), it creates and adds real browser headers. It also sets the referer header as if this request came from a Google search of URL's domain. + :param kwargs: Additional keyword arguments to pass to the [`curl_cffi.requests.Session().request()`, `curl_cffi.requests.AsyncSession().request()`] method. + :return: A `Response` object. + """ + method_args = {k: v for k, v in locals().items() if k not in ("self", "stealthy_headers", "kwargs")} + method_args.update(kwargs) + # For type checking (not accessed error) + _ = ( + url, + params, + headers, + cookies, + timeout, + follow_redirects, + max_redirects, + retries, + retry_delay, + proxies, + proxy, + proxy_auth, + auth, + verify, + cert, + impersonate, + http3, ) + return self.__make_request("GET", stealth=stealthy_headers, **method_args) def post( self, @@ -934,32 +778,58 @@ class AsyncFetcherClient(FetcherSession): stealthy_headers: Optional[bool] = _UNSET, **kwargs, ) -> Awaitable[Response]: - return cast( - Awaitable[Response], - super().post( - url, - data, - json, - headers, - params, - cookies, - timeout, - follow_redirects, - max_redirects, - retries, - retry_delay, - proxies, - proxy, - proxy_auth, - auth, - verify, - cert, - impersonate, - http3, - stealthy_headers, - **kwargs, - ), + """ + Perform a POST request. + + :param url: Target URL for the request. + :param data: Form data to include in the request body. + :param json: A JSON serializable object to include in the body of the request. + :param params: Query string parameters for the request. + :param headers: Headers to include in the request. + :param cookies: Cookies to use in the request. + :param timeout: Number of seconds to wait before timing out. + :param follow_redirects: Whether to follow redirects. Defaults to True. + :param max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited. + :param retries: Number of retry attempts. Defaults to 3. + :param retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second. + :param proxies: Dict of proxies to use. + :param proxy: Proxy URL to use. Format: "http://username:password@localhost:8030". + Cannot be used together with the `proxies` parameter. + :param proxy_auth: HTTP basic auth for proxy, tuple of (username, password). + :param auth: HTTP basic auth tuple of (username, password). Only basic auth is supported. + :param verify: Whether to verify HTTPS certificates. + :param cert: Tuple of (cert, key) filenames for the client certificate. + :param impersonate: Browser version to impersonate. Automatically defaults to the latest available Chrome version. + :param http3: Whether to use HTTP3. Defaults to False. It might be problematic if used it with `impersonate`. + :param stealthy_headers: If enabled (default), it creates and adds real browser headers. It also sets the referer header as if this request came from a Google search of URL's domain. + :param kwargs: Additional keyword arguments to pass to the [`curl_cffi.requests.Session().request()`, `curl_cffi.requests.AsyncSession().request()`] method. + :return: A `Response` object. + """ + method_args = {k: v for k, v in locals().items() if k not in ("self", "stealthy_headers", "kwargs")} + method_args.update(kwargs) + # For type checking (not accessed error) + _ = ( + url, + params, + headers, + data, + json, + cookies, + timeout, + follow_redirects, + max_redirects, + retries, + retry_delay, + proxies, + proxy, + proxy_auth, + auth, + verify, + cert, + impersonate, + http3, ) + return self.__make_request("POST", stealth=stealthy_headers, **method_args) def put( self, @@ -985,32 +855,58 @@ class AsyncFetcherClient(FetcherSession): stealthy_headers: Optional[bool] = _UNSET, **kwargs, ) -> Awaitable[Response]: - return cast( - Awaitable[Response], - super().put( - url, - data, - json, - headers, - params, - cookies, - timeout, - follow_redirects, - max_redirects, - retries, - retry_delay, - proxies, - proxy, - proxy_auth, - auth, - verify, - cert, - impersonate, - http3, - stealthy_headers, - **kwargs, - ), + """ + Perform a PUT request. + + :param url: Target URL for the request. + :param data: Form data to include in the request body. + :param json: A JSON serializable object to include in the body of the request. + :param params: Query string parameters for the request. + :param headers: Headers to include in the request. + :param cookies: Cookies to use in the request. + :param timeout: Number of seconds to wait before timing out. + :param follow_redirects: Whether to follow redirects. Defaults to True. + :param max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited. + :param retries: Number of retry attempts. Defaults to 3. + :param retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second. + :param proxies: Dict of proxies to use. + :param proxy: Proxy URL to use. Format: "http://username:password@localhost:8030". + Cannot be used together with the `proxies` parameter. + :param proxy_auth: HTTP basic auth for proxy, tuple of (username, password). + :param auth: HTTP basic auth tuple of (username, password). Only basic auth is supported. + :param verify: Whether to verify HTTPS certificates. + :param cert: Tuple of (cert, key) filenames for the client certificate. + :param impersonate: Browser version to impersonate. Automatically defaults to the latest available Chrome version. + :param http3: Whether to use HTTP3. Defaults to False. It might be problematic if used it with `impersonate`. + :param stealthy_headers: If enabled (default), it creates and adds real browser headers. It also sets the referer header as if this request came from a Google search of URL's domain. + :param kwargs: Additional keyword arguments to pass to the [`curl_cffi.requests.Session().request()`, `curl_cffi.requests.AsyncSession().request()`] method. + :return: A `Response` object. + """ + method_args = {k: v for k, v in locals().items() if k not in ("self", "stealthy_headers", "kwargs")} + method_args.update(kwargs) + # For type checking (not accessed error) + _ = ( + url, + params, + headers, + data, + json, + cookies, + timeout, + follow_redirects, + max_redirects, + retries, + retry_delay, + proxies, + proxy, + proxy_auth, + auth, + verify, + cert, + impersonate, + http3, ) + return self.__make_request("PUT", stealth=stealthy_headers, **method_args) def delete( self, @@ -1036,29 +932,174 @@ class AsyncFetcherClient(FetcherSession): stealthy_headers: Optional[bool] = _UNSET, **kwargs, ) -> Awaitable[Response]: - return cast( - Awaitable[Response], - super().delete( - url, - data, - json, - headers, - params, - cookies, - timeout, - follow_redirects, - max_redirects, - retries, - retry_delay, - proxies, - proxy, - proxy_auth, - auth, - verify, - cert, - impersonate, - http3, - stealthy_headers, - **kwargs, - ), + """ + Perform a DELETE request. + + :param url: Target URL for the request. + :param data: Form data to include in the request body. + :param json: A JSON serializable object to include in the body of the request. + :param params: Query string parameters for the request. + :param headers: Headers to include in the request. + :param cookies: Cookies to use in the request. + :param timeout: Number of seconds to wait before timing out. + :param follow_redirects: Whether to follow redirects. Defaults to True. + :param max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited. + :param retries: Number of retry attempts. Defaults to 3. + :param retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second. + :param proxies: Dict of proxies to use. + :param proxy: Proxy URL to use. Format: "http://username:password@localhost:8030". + Cannot be used together with the `proxies` parameter. + :param proxy_auth: HTTP basic auth for proxy, tuple of (username, password). + :param auth: HTTP basic auth tuple of (username, password). Only basic auth is supported. + :param verify: Whether to verify HTTPS certificates. + :param cert: Tuple of (cert, key) filenames for the client certificate. + :param impersonate: Browser version to impersonate. Automatically defaults to the latest available Chrome version. + :param http3: Whether to use HTTP3. Defaults to False. It might be problematic if used it with `impersonate`. + :param stealthy_headers: If enabled (default), it creates and adds real browser headers. It also sets the referer header as if this request came from a Google search of URL's domain. + :param kwargs: Additional keyword arguments to pass to the [`curl_cffi.requests.Session().request()`, `curl_cffi.requests.AsyncSession().request()`] method. + :return: A `Response` object. + """ + # Careful of sending a body in a DELETE request, it might cause some websites to reject the request as per https://www.rfc-editor.org/rfc/rfc7231#section-4.3.5, + # But some websites accept it, it depends on the implementation used. + method_args = {k: v for k, v in locals().items() if k not in ("self", "stealthy_headers", "kwargs")} + method_args.update(kwargs) + # For type checking (not accessed error) + _ = ( + url, + params, + headers, + data, + json, + cookies, + timeout, + follow_redirects, + max_redirects, + retries, + retry_delay, + proxies, + proxy, + proxy_auth, + auth, + verify, + cert, + impersonate, + http3, ) + return self.__make_request("DELETE", stealth=stealthy_headers, **method_args) + + +class FetcherSession: + """ + A factory context manager that provides configured Fetcher sessions. + + When this manager is used in a 'with' or 'async with' block, + it yields a new session configured with the manager's defaults. + A single instance of this manager should ideally be used for one active + session at a time (or sequentially). Re-entering a context with the + same manager instance while a session is already active is disallowed. + """ + + def __init__( + self, + impersonate: Optional[BrowserTypeLiteral] = "chrome", + http3: Optional[bool] = False, + stealthy_headers: Optional[bool] = True, + proxies: Optional[Dict[str, str]] = None, + proxy: Optional[str] = None, + proxy_auth: Optional[Tuple[str, str]] = None, + timeout: Optional[int | float] = 30, + headers: Optional[Dict[str, str]] = None, + retries: Optional[int] = 3, + retry_delay: Optional[int] = 1, + follow_redirects: bool = True, + max_redirects: int = 30, + verify: bool = True, + cert: Optional[str | Tuple[str, str]] = None, + selector_config: Optional[Dict] = None, + ): + """ + :param impersonate: Browser version to impersonate. Automatically defaults to the latest available Chrome version. + :param http3: Whether to use HTTP3. Defaults to False. It might be problematic if used it with `impersonate`. + :param stealthy_headers: If enabled (default), it creates and adds real browser headers. It also sets the referer header as if this request came from a Google search of URL's domain. + :param proxies: Dict of proxies to use. Format: {"http": proxy_url, "https": proxy_url}. + :param proxy: Proxy URL to use. Format: "http://username:password@localhost:8030". + Cannot be used together with the `proxies` parameter. + :param proxy_auth: HTTP basic auth for proxy, tuple of (username, password). + :param timeout: Number of seconds to wait before timing out. + :param headers: Headers to include in the session with every request. + :param retries: Number of retry attempts. Defaults to 3. + :param retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second. + :param follow_redirects: Whether to follow redirects. Defaults to True. + :param max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited. + :param verify: Whether to verify HTTPS certificates. Defaults to True. + :param cert: Tuple of (cert, key) filenames for the client certificate. + :param selector_config: Arguments passed when creating the final Selector class. + """ + self._default_impersonate: Optional[BrowserTypeLiteral] = impersonate + self._stealth = stealthy_headers + self._default_proxies = proxies or {} + self._default_proxy = proxy or None + self._default_proxy_auth = proxy_auth or None + self._default_timeout = timeout + self._default_headers = headers or {} + self._default_retries = retries + self._default_retry_delay = retry_delay + self._default_follow_redirects = follow_redirects + self._default_max_redirects = max_redirects + self._default_verify = verify + self._default_cert = cert + self._default_http3 = http3 + self.selector_config = selector_config or {} + self._client: _SyncSessionLogic | _ASyncSessionLogic | None = None + + def __enter__(self) -> _SyncSessionLogic: + """Creates and returns a new synchronous Fetcher Session""" + if self._client is None: + # Use **vars(self) to avoid repeating all parameters + config = {k.replace("_default_", ""): v for k, v in vars(self).items() if k.startswith("_default")} + config["stealthy_headers"] = self._stealth + config["selector_config"] = self.selector_config + self._client = _SyncSessionLogic(**config) + return self._client.__enter__() + raise RuntimeError("This FetcherSession instance already has an active synchronous session.") + + def __exit__(self, exc_type, exc_val, exc_tb): + if self._client is not None and isinstance(self._client, _SyncSessionLogic): + self._client.__exit__(exc_type, exc_val, exc_tb) + self._client = None + return + raise RuntimeError("Cannot exit invalid session") + + async def __aenter__(self) -> _ASyncSessionLogic: + """Creates and returns a new asynchronous Session.""" + if self._client is None: + # Use **vars(self) to avoid repeating all parameters + config = {k.replace("_default_", ""): v for k, v in vars(self).items() if k.startswith("_default")} + config["stealthy_headers"] = self._stealth + config["selector_config"] = self.selector_config + self._client = _ASyncSessionLogic(**config) + return await self._client.__aenter__() + raise RuntimeError("This FetcherSession instance already has an active asynchronous session.") + + async def __aexit__(self, exc_type, exc_val, exc_tb): + if self._client is not None and isinstance(self._client, _ASyncSessionLogic): + await self._client.__aexit__(exc_type, exc_val, exc_tb) + self._client = None + return + raise RuntimeError("Cannot exit invalid session") + + +class FetcherClient(_SyncSessionLogic): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.__enter__: Any = None + self.__exit__: Any = None + self._curl_session: Any = _NO_SESSION + + +class AsyncFetcherClient(_ASyncSessionLogic): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.__aenter__: Any = None + self.__aexit__: Any = None + self._async_curl_session: Any = _NO_SESSION