From 6d7992723392465e98e0e748f2ecb5a461ed2cff Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Sun, 6 Jul 2025 02:21:15 +0300 Subject: [PATCH] refactor(Fetcher): Fix the issue of caching impersonation state + Less code duplication It now creates a session with each request, but at the same time, it's still faster than the fetcher in v0.2.99 by 20% with more features enabled. --- scrapling/engines/static.py | 347 ++++-------------------------------- scrapling/fetchers.py | 9 +- 2 files changed, 36 insertions(+), 320 deletions(-) diff --git a/scrapling/engines/static.py b/scrapling/engines/static.py index d1940bd..a81dd88 100644 --- a/scrapling/engines/static.py +++ b/scrapling/engines/static.py @@ -265,10 +265,18 @@ class FetcherSession: :param adaptor_arguments: Arguments passed when creating the final Adaptor class. :return: A `Response` object for synchronous requests or an awaitable for asynchronous. """ - if self._curl_session: + session = self._curl_session + if session is True and not any( + (self.__enter__, self.__exit__, self.__aenter__, self.__aexit__) + ): + # For usage inside FetcherClient + # It turns out `curl_cffi` caches impersonation state, so if you turned it off, then on then off, it won't be off on the last time. + session = CurlSession() + + if session: for attempt in range(max_retries): try: - response = self._curl_session.request(method, **request_args) + response = session.request(method, **request_args) # response.raise_for_status() # Retry responses with a status code between 200-400 return ResponseFactory.from_http_request( response, adaptor_arguments @@ -304,12 +312,20 @@ class FetcherSession: :param adaptor_arguments: Arguments passed when creating the final Adaptor class. :return: A `Response` object for synchronous requests or an awaitable for asynchronous. """ - if self._async_curl_session: + session = self._async_curl_session + if session is True and not any( + (self.__enter__, self.__exit__, self.__aenter__, self.__aexit__) + ): + # For usage inside the ` AsyncFetcherClient ` class, and that's for several reasons + # 1. It turns out `curl_cffi` caches impersonation state, so if you turned it off, then on then off, it won't be off on the last time. + # 2. `curl_cffi` doesn't support making async requests without sessions + # 3. Using a single session for many requests at the same time in async doesn't sit well with curl_cffi. + session = AsyncCurlSession() + + if session: for attempt in range(max_retries): try: - response = await self._async_curl_session.request( - method, **request_args - ) + response = await session.request(method, **request_args) # response.raise_for_status() # Retry responses with a status code between 200-400 return ResponseFactory.from_http_request( response, adaptor_arguments @@ -677,319 +693,18 @@ class FetcherSession: class FetcherClient(FetcherSession): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - # Using one session for all requests is faster than using stateless `curl_cffi.get` self.__enter__ = None self.__exit__ = None self.__aenter__ = None self.__aexit__ = None - self._curl_session = CurlSession() + self._curl_session = True -class AsyncFetcherClient: - # Since curl_cffi doesn't support making async requests without sessions - # And using a single session for many requests at the same time in async doesn't sit well with curl_cffi. - # We do this - - @staticmethod - async def get( - url: str, - params: Optional[Union[Dict, List, Tuple]] = None, - headers: Optional[Mapping[str, Optional[str]]] = _UNSET, - cookies: Optional[CookieTypes] = None, - timeout: Optional[Union[int, float]] = _UNSET, - follow_redirects: Optional[bool] = _UNSET, - max_redirects: Optional[int] = _UNSET, - retries: Optional[int] = _UNSET, - retry_delay: Optional[int] = _UNSET, - proxies: Optional[ProxySpec] = _UNSET, - proxy: Optional[str] = _UNSET, - proxy_auth: Optional[Tuple[str, str]] = _UNSET, - auth: Optional[Tuple[str, str]] = None, - verify: Optional[bool] = _UNSET, - cert: Optional[Union[str, Tuple[str, str]]] = _UNSET, - impersonate: Optional[BrowserTypeLiteral] = _UNSET, - http3: Optional[bool] = _UNSET, - stealthy_headers: Optional[bool] = _UNSET, - **kwargs, - ) -> Response: - """ - Perform a GET request. - - :param url: Target URL for the request. - :param params: Query string parameters for the request. - :param headers: Headers to include in the request. - :param cookies: Cookies to use in the request. - :param timeout: Number of seconds to wait before timing out. - :param follow_redirects: Whether to follow redirects. Defaults to True. - :param max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited. - :param retries: Number of retry attempts. Defaults to 3. - :param retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second. - :param proxies: Dict of proxies to use. - :param proxy: Proxy URL to use. Format: "http://username:password@localhost:8030". - Cannot be used together with the `proxies` parameter. - :param proxy_auth: HTTP basic auth for proxy, tuple of (username, password). - :param auth: HTTP basic auth tuple of (username, password). Only basic auth is supported. - :param verify: Whether to verify HTTPS certificates. - :param cert: Tuple of (cert, key) filenames for the client certificate. - :param impersonate: Browser version to impersonate. Automatically defaults to the latest available Chrome version. - :param http3: Whether to use HTTP3. Defaults to False. It might be problematic if used it with `impersonate`. - :param stealthy_headers: If enabled (default), it creates and adds real browser headers. It also sets the referer header as if this request came from a Google search of URL's domain. - :param kwargs: Additional keyword arguments to pass to the `curl_cffi.requests.AsyncSession().request()` method. - :return: An awaitable `Response` object. - """ - request_args = { - "url": url, - "params": params, - "headers": headers, - "cookies": cookies, - "timeout": timeout, - "retry_delay": retry_delay, - "allow_redirects": follow_redirects, - "max_redirects": max_redirects, - "retries": retries, - "proxies": proxies, - "proxy": proxy, - "proxy_auth": proxy_auth, - "auth": auth, - "verify": verify, - "cert": cert, - "impersonate": impersonate, - "http3": http3, - "stealthy_headers": stealthy_headers, - **kwargs, - } - async with FetcherSession() as client: - return await client.get(**request_args) - - @staticmethod - async def post( - url: str, - data: Optional[Union[Dict, str]] = None, - json: Optional[Union[Dict, List]] = None, - headers: Optional[Mapping[str, Optional[str]]] = _UNSET, - params: Optional[Union[Dict, List, Tuple]] = None, - cookies: Optional[CookieTypes] = None, - timeout: Optional[Union[int, float]] = _UNSET, - follow_redirects: Optional[bool] = _UNSET, - max_redirects: Optional[int] = _UNSET, - retries: Optional[int] = _UNSET, - retry_delay: Optional[int] = _UNSET, - proxies: Optional[ProxySpec] = _UNSET, - proxy: Optional[str] = _UNSET, - proxy_auth: Optional[Tuple[str, str]] = _UNSET, - auth: Optional[Tuple[str, str]] = None, - verify: Optional[bool] = _UNSET, - cert: Optional[Union[str, Tuple[str, str]]] = _UNSET, - impersonate: Optional[BrowserTypeLiteral] = _UNSET, - http3: Optional[bool] = _UNSET, - stealthy_headers: Optional[bool] = _UNSET, - **kwargs, - ) -> Response: - """ - Perform a POST request. - - :param url: Target URL for the request. - :param data: Form data to include in the request body. - :param json: A JSON serializable object to include in the body of the request. - :param headers: Headers to include in the request. - :param params: Query string parameters for the request. - :param cookies: Cookies to use in the request. - :param timeout: Number of seconds to wait before timing out. - :param follow_redirects: Whether to follow redirects. Defaults to True. - :param max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited. - :param retries: Number of retry attempts. Defaults to 3. - :param retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second. - :param proxies: Dict of proxies to use. Format: {"http": proxy_url, "https": proxy_url}. - :param proxy: Proxy URL to use. Format: "http://username:password@localhost:8030". - Cannot be used together with the `proxies` parameter. - :param proxy_auth: HTTP basic auth for proxy, tuple of (username, password). - :param auth: HTTP basic auth tuple of (username, password). Only basic auth is supported. - :param verify: Whether to verify HTTPS certificates. Defaults to True. - :param cert: Tuple of (cert, key) filenames for the client certificate. - :param impersonate: Browser version to impersonate. Automatically defaults to the latest available Chrome version. - :param http3: Whether to use HTTP3. Defaults to False. It might be problematic if used it with `impersonate`. - :param stealthy_headers: If enabled (default), it creates and adds real browser headers. It also sets the referer header as if this request came from a Google search of URL's domain. - :param kwargs: Additional keyword arguments to pass to the `curl_cffi.requests.AsyncSession().request()` method. - :return: An awaitable `Response` object. - """ - request_args = { - "url": url, - "data": data, - "json": json, - "headers": headers, - "params": params, - "cookies": cookies, - "timeout": timeout, - "retry_delay": retry_delay, - "proxy": proxy, - "impersonate": impersonate, - "allow_redirects": follow_redirects, - "max_redirects": max_redirects, - "retries": retries, - "proxies": proxies, - "proxy_auth": proxy_auth, - "auth": auth, - "verify": verify, - "cert": cert, - "http3": http3, - "stealthy_headers": stealthy_headers, - **kwargs, - } - async with FetcherSession() as client: - return await client.post(**request_args) - - @staticmethod - async def put( - url: str, - data: Optional[Union[Dict, str]] = None, - json: Optional[Union[Dict, List]] = None, - headers: Optional[Mapping[str, Optional[str]]] = _UNSET, - params: Optional[Union[Dict, List, Tuple]] = None, - cookies: Optional[CookieTypes] = None, - timeout: Optional[Union[int, float]] = _UNSET, - follow_redirects: Optional[bool] = _UNSET, - max_redirects: Optional[int] = _UNSET, - retries: Optional[int] = _UNSET, - retry_delay: Optional[int] = _UNSET, - proxies: Optional[ProxySpec] = _UNSET, - proxy: Optional[str] = _UNSET, - proxy_auth: Optional[Tuple[str, str]] = _UNSET, - auth: Optional[Tuple[str, str]] = None, - verify: Optional[bool] = _UNSET, - cert: Optional[Union[str, Tuple[str, str]]] = _UNSET, - impersonate: Optional[BrowserTypeLiteral] = _UNSET, - http3: Optional[bool] = _UNSET, - stealthy_headers: Optional[bool] = _UNSET, - **kwargs, - ) -> Response: - """ - Perform a PUT request. - - :param url: Target URL for the request. - :param data: Form data to include in the request body. - :param json: A JSON serializable object to include in the body of the request. - :param headers: Headers to include in the request. - :param params: Query string parameters for the request. - :param cookies: Cookies to use in the request. - :param timeout: Number of seconds to wait before timing out. - :param follow_redirects: Whether to follow redirects. Defaults to True. - :param max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited. - :param retries: Number of retry attempts. Defaults to 3. - :param retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second. - :param proxies: Dict of proxies to use. Format: {"http": proxy_url, "https": proxy_url}. - :param proxy: Proxy URL to use. Format: "http://username:password@localhost:8030". - Cannot be used together with the `proxies` parameter. - :param proxy_auth: HTTP basic auth for proxy, tuple of (username, password). - :param auth: HTTP basic auth tuple of (username, password). Only basic auth is supported. - :param verify: Whether to verify HTTPS certificates. Defaults to True. - :param cert: Tuple of (cert, key) filenames for the client certificate. - :param impersonate: Browser version to impersonate. Automatically defaults to the latest available Chrome version. - :param http3: Whether to use HTTP3. Defaults to False. It might be problematic if used it with `impersonate`. - :param stealthy_headers: If enabled (default), it creates and adds real browser headers. It also sets the referer header as if this request came from a Google search of URL's domain. - :param kwargs: Additional keyword arguments to pass to the `curl_cffi.requests.AsyncSession().request()` method. - :return: An awaitable `Response` object. - """ - request_args = { - "url": url, - "data": data, - "json": json, - "headers": headers, - "params": params, - "cookies": cookies, - "timeout": timeout, - "retry_delay": retry_delay, - "proxy": proxy, - "impersonate": impersonate, - "allow_redirects": follow_redirects, - "max_redirects": max_redirects, - "retries": retries, - "proxies": proxies, - "proxy_auth": proxy_auth, - "auth": auth, - "verify": verify, - "cert": cert, - "http3": http3, - "stealthy_headers": stealthy_headers, - **kwargs, - } - async with FetcherSession() as client: - return await client.put(**request_args) - - @staticmethod - async def delete( - url: str, - data: Optional[Union[Dict, str]] = None, - json: Optional[Union[Dict, List]] = None, - headers: Optional[Mapping[str, Optional[str]]] = _UNSET, - params: Optional[Union[Dict, List, Tuple]] = None, - cookies: Optional[CookieTypes] = None, - timeout: Optional[Union[int, float]] = _UNSET, - follow_redirects: Optional[bool] = _UNSET, - max_redirects: Optional[int] = _UNSET, - retries: Optional[int] = _UNSET, - retry_delay: Optional[int] = _UNSET, - proxies: Optional[ProxySpec] = _UNSET, - proxy: Optional[str] = _UNSET, - proxy_auth: Optional[Tuple[str, str]] = _UNSET, - auth: Optional[Tuple[str, str]] = None, - verify: Optional[bool] = _UNSET, - cert: Optional[Union[str, Tuple[str, str]]] = _UNSET, - impersonate: Optional[BrowserTypeLiteral] = _UNSET, - http3: Optional[bool] = _UNSET, - stealthy_headers: Optional[bool] = _UNSET, - **kwargs, - ) -> Response: - """ - Perform a DELETE request. - - :param url: Target URL for the request. - :param data: Form data to include in the request body. - :param json: A JSON serializable object to include in the body of the request. - :param headers: Headers to include in the request. - :param params: Query string parameters for the request. - :param cookies: Cookies to use in the request. - :param timeout: Number of seconds to wait before timing out. - :param follow_redirects: Whether to follow redirects. Defaults to True. - :param max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited. - :param retries: Number of retry attempts. Defaults to 3. - :param retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second. - :param proxies: Dict of proxies to use. Format: {"http": proxy_url, "https": proxy_url}. - :param proxy: Proxy URL to use. Format: "http://username:password@localhost:8030". - Cannot be used together with the `proxies` parameter. - :param proxy_auth: HTTP basic auth for proxy, tuple of (username, password). - :param auth: HTTP basic auth tuple of (username, password). Only basic auth is supported. - :param verify: Whether to verify HTTPS certificates. Defaults to True. - :param cert: Tuple of (cert, key) filenames for the client certificate. - :param impersonate: Browser version to impersonate. Automatically defaults to the latest available Chrome version. - :param http3: Whether to use HTTP3. Defaults to False. It might be problematic if used it with `impersonate`. - :param stealthy_headers: If enabled (default), it creates and adds real browser headers. It also sets the referer header as if this request came from a Google search of URL's domain. - :param kwargs: Additional keyword arguments to pass to the `curl_cffi.requests.AsyncSession().request()` method. - :return: An awaitable `Response` object. - """ - request_args = { - "url": url, - # Careful of sending a body in a DELETE request, it might cause some websites to reject the request as per https://www.rfc-editor.org/rfc/rfc7231#section-4.3.5, - # But some websites accept it, it depends on the implementation used. - "data": data, - "json": json, - "headers": headers, - "params": params, - "cookies": cookies, - "timeout": timeout, - "retry_delay": retry_delay, - "proxy": proxy, - "impersonate": impersonate, - "allow_redirects": follow_redirects, - "max_redirects": max_redirects, - "retries": retries, - "proxies": proxies, - "proxy_auth": proxy_auth, - "auth": auth, - "verify": verify, - "cert": cert, - "http3": http3, - "stealthy_headers": stealthy_headers, - **kwargs, - } - async with FetcherSession() as client: - return await client.delete(**request_args) +class AsyncFetcherClient(FetcherSession): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.__enter__ = None + self.__exit__ = None + self.__aenter__ = None + self.__aexit__ = None + self._async_curl_session = True diff --git a/scrapling/fetchers.py b/scrapling/fetchers.py index 6ba3611..31c3a69 100644 --- a/scrapling/fetchers.py +++ b/scrapling/fetchers.py @@ -20,6 +20,7 @@ from scrapling.engines import ( from scrapling.engines.toolbelt import BaseFetcher, Response __FetcherClientInstance__ = _FetcherClient() +__AsyncFetcherClientInstance__ = _AsyncFetcherClient() class Fetcher(BaseFetcher): @@ -34,10 +35,10 @@ class Fetcher(BaseFetcher): class AsyncFetcher(BaseFetcher): """A basic `Fetcher` class type that can only do basic GET, POST, PUT, and DELETE HTTP requests based on `curl_cffi`.""" - get = _AsyncFetcherClient.get - post = _AsyncFetcherClient.post - put = _AsyncFetcherClient.put - delete = _AsyncFetcherClient.delete + get = __AsyncFetcherClientInstance__.get + post = __AsyncFetcherClientInstance__.post + put = __AsyncFetcherClientInstance__.put + delete = __AsyncFetcherClientInstance__.delete class StealthyFetcher(BaseFetcher):