freedak/Scrapling
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat/fix(Fetcher): Adding http3 support + Fix stealthy_headers

Browse Source 
And some fixes here and there to the docs
This commit is contained in:
Karim shoair
2025-05-31 03:35:46 +03:00
parent af643e4c01
commit 178eed7533
2 changed files with 168 additions and 125 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scrapling/engines/static.py
+159 -116
View File
@@ -2,6 +2,8 @@ from time import sleep as time_sleep
from asyncio import sleep as asyncio_sleep from asyncio import sleep as asyncio_sleep
from curl_cffi.requests.session import CurlError from curl_cffi.requests.session import CurlError
from curl_cffi import CurlHttpVersion
from curl_cffi.requests.impersonate import DEFAULT_CHROME
from curl_cffi.requests import ( from curl_cffi.requests import (
ProxySpec, ProxySpec,
CookieTypes, CookieTypes,
@@ -46,7 +48,8 @@ class FetcherSession:
def __init__( def __init__(
self, self,
impersonate: Optional[str] = "chrome136", impersonate: Optional[BrowserTypeLiteral] = DEFAULT_CHROME,
http3: Optional[bool] = False,
stealthy_headers: Optional[bool] = True, stealthy_headers: Optional[bool] = True,
proxies: Optional[Dict[str, str]] = None, proxies: Optional[Dict[str, str]] = None,
proxy: Optional[str] = None, proxy: Optional[str] = None,
@@ -62,8 +65,9 @@ class FetcherSession:
adaptor_arguments: Optional[Dict] = None, adaptor_arguments: Optional[Dict] = None,
): ):
""" """
:param impersonate: Browser version to impersonate. Defaults to "chrome136". :param impersonate: Browser version to impersonate. Automatically defaults to the latest available Chrome version.
:param stealthy_headers: If enabled (default), it creates and adds real browser headers. It also referer header as if it is from a Google search of URL's domain. :param http3: Whether to use HTTP3. Defaults to False. It might be problematic if used it with `impersonate`.
:param stealthy_headers: If enabled (default), it creates and adds real browser headers. It also sets the referer header as if this request came from a Google search of URL's domain.
:param proxies: Dict of proxies to use. Format: {"http": proxy_url, "https": proxy_url}. :param proxies: Dict of proxies to use. Format: {"http": proxy_url, "https": proxy_url}.
:param proxy: Proxy URL to use. Format: "http://username:password@localhost:8030". :param proxy: Proxy URL to use. Format: "http://username:password@localhost:8030".
Cannot be used together with the `proxies` parameter. Cannot be used together with the `proxies` parameter.
@@ -91,6 +95,7 @@ class FetcherSession:
self.default_max_redirects = max_redirects self.default_max_redirects = max_redirects
self.default_verify = verify self.default_verify = verify
self.default_cert = cert self.default_cert = cert
self.default_http3 = http3
self.adaptor_arguments = adaptor_arguments or {} self.adaptor_arguments = adaptor_arguments or {}
self._curl_session: Optional[CurlSession] = None self._curl_session: Optional[CurlSession] = None
@@ -98,23 +103,37 @@ class FetcherSession:
def _merge_request_args(self, **kwargs) -> Dict[str, Any]: def _merge_request_args(self, **kwargs) -> Dict[str, Any]:
"""Merge request-specific arguments with default session arguments.""" """Merge request-specific arguments with default session arguments."""
request_args = { url = kwargs.pop("url")
"headers": self._headers_job( request_args = {}
kwargs["url"], kwargs.get("headers"), kwargs.pop("stealth") if kwargs.pop("http3", False) or self.default_http3:
), request_args["http_version"] = CurlHttpVersion.V3ONLY
"proxies": kwargs.get("proxies", self.default_proxies), if kwargs.get("impersonate"):
"proxy": kwargs.get("proxy", self.default_proxy), log.warning(
"proxy_auth": kwargs.get("proxy_auth", self.default_proxy_auth), "The argument `http3` might cause errors if used with `impersonate` argument, try switching it off if you encounter any curl errors."
"timeout": kwargs.get("timeout", self.default_timeout), )
"allow_redirects": kwargs.get(
"follow_redirects", self.default_follow_redirects request_args.update(
), {
"max_redirects": kwargs.get("max_redirects", self.default_max_redirects), "url": url,
"verify": kwargs.get("verify", self.default_verify), "headers": self._headers_job(
"cert": kwargs.get("cert", self.default_cert), url, kwargs.pop("headers"), kwargs.pop("stealth")
"impersonate": kwargs.get("impersonate", self.default_impersonate), ),
**kwargs, "proxies": kwargs.pop("proxies", self.default_proxies),
} "proxy": kwargs.pop("proxy", self.default_proxy),
"proxy_auth": kwargs.pop("proxy_auth", self.default_proxy_auth),
"timeout": kwargs.pop("timeout", self.default_timeout),
"allow_redirects": kwargs.pop(
"follow_redirects", self.default_follow_redirects
),
"max_redirects": kwargs.pop(
"max_redirects", self.default_max_redirects
),
"verify": kwargs.pop("verify", self.default_verify),
"cert": kwargs.pop("cert", self.default_cert),
"impersonate": kwargs.pop("impersonate", self.default_impersonate),
**kwargs,
}
)
return request_args return request_args
def _headers_job( def _headers_job(
@@ -316,21 +335,22 @@ class FetcherSession:
def get( def get(
self, self,
url: str, url: str,
params: Optional[Union[Dict, List, Tuple]] = None, # <-- params: Optional[Union[Dict, List, Tuple]] = None,
headers: Optional[Mapping[str, Optional[str]]] = None, headers: Optional[Mapping[str, Optional[str]]] = None,
cookies: Optional[CookieTypes] = None, # <-- cookies: Optional[CookieTypes] = None,
timeout: Optional[Union[int, float]] = 30, # <-- timeout: Optional[Union[int, float]] = 30,
follow_redirects: Optional[bool] = True, # <-- follow_redirects: Optional[bool] = True,
max_redirects: Optional[int] = 30, # <-- max_redirects: Optional[int] = 30,
retries: Optional[int] = 3, retries: Optional[int] = 3,
retry_delay: Optional[int] = 1, # <-- retry_delay: Optional[int] = 1,
proxies: Optional[ProxySpec] = None, # <-- proxies: Optional[ProxySpec] = None,
proxy: Optional[str] = None, # <-- proxy: Optional[str] = None,
proxy_auth: Optional[Tuple[str, str]] = None, proxy_auth: Optional[Tuple[str, str]] = None,
auth: Optional[Tuple[str, str]] = None, auth: Optional[Tuple[str, str]] = None,
verify: Optional[bool] = True, # <-- verify: Optional[bool] = True,
cert: Optional[Union[str, Tuple[str, str]]] = None, cert: Optional[Union[str, Tuple[str, str]]] = None,
impersonate: Optional[BrowserTypeLiteral] = "chrome136", # <-- impersonate: Optional[BrowserTypeLiteral] = DEFAULT_CHROME,
http3: Optional[bool] = False,
stealthy_headers: Optional[bool] = True, stealthy_headers: Optional[bool] = True,
**kwargs, **kwargs,
) -> Union[Response, Awaitable[Response]]: ) -> Union[Response, Awaitable[Response]]:
@@ -353,8 +373,9 @@ class FetcherSession:
:param auth: HTTP basic auth tuple of (username, password). Only basic auth is supported. :param auth: HTTP basic auth tuple of (username, password). Only basic auth is supported.
:param verify: Whether to verify HTTPS certificates. :param verify: Whether to verify HTTPS certificates.
:param cert: Tuple of (cert, key) filenames for the client certificate. :param cert: Tuple of (cert, key) filenames for the client certificate.
:param impersonate: Browser version to impersonate. Defaults to "chrome136". :param impersonate: Browser version to impersonate. Automatically defaults to the latest available Chrome version.
:param stealthy_headers: If enabled for this request (default), it creates and adds real browser headers. It also referer header as if it is from a Google search of URL's domain. :param http3: Whether to use HTTP3. Defaults to False. It might be problematic if used it with `impersonate`.
:param stealthy_headers: If enabled (default), it creates and adds real browser headers. It also sets the referer header as if this request came from a Google search of URL's domain.
:param kwargs: Additional keyword arguments to pass to the [`curl_cffi.requests.Session().request()`, `curl_cffi.requests.AsyncSession().request()`] method. :param kwargs: Additional keyword arguments to pass to the [`curl_cffi.requests.Session().request()`, `curl_cffi.requests.AsyncSession().request()`] method.
:return: A `Response` object or an awaitable for async. :return: A `Response` object or an awaitable for async.
""" """
@@ -375,6 +396,7 @@ class FetcherSession:
"verify": verify, "verify": verify,
"cert": cert, "cert": cert,
"impersonate": impersonate, "impersonate": impersonate,
"http3": http3,
**kwargs, **kwargs,
} }
return self.__prepare_and_dispatch( return self.__prepare_and_dispatch(
@@ -387,20 +409,21 @@ class FetcherSession:
data: Optional[Union[Dict, str]] = None, data: Optional[Union[Dict, str]] = None,
json: Optional[Union[Dict, List]] = None, json: Optional[Union[Dict, List]] = None,
headers: Optional[Mapping[str, Optional[str]]] = None, headers: Optional[Mapping[str, Optional[str]]] = None,
params: Optional[Union[Dict, List, Tuple]] = None, # <-- params: Optional[Union[Dict, List, Tuple]] = None,
cookies: Optional[CookieTypes] = None, # <-- cookies: Optional[CookieTypes] = None,
timeout: Optional[Union[int, float]] = 30, # <-- timeout: Optional[Union[int, float]] = 30,
follow_redirects: Optional[bool] = True, # <-- follow_redirects: Optional[bool] = True,
max_redirects: Optional[int] = 30, # <-- max_redirects: Optional[int] = 30,
retries: Optional[int] = 3, retries: Optional[int] = 3,
retry_delay: Optional[int] = 1, # <-- retry_delay: Optional[int] = 1,
proxies: Optional[ProxySpec] = None, # <-- proxies: Optional[ProxySpec] = None,
proxy: Optional[str] = None, # <-- proxy: Optional[str] = None,
proxy_auth: Optional[Tuple[str, str]] = None, proxy_auth: Optional[Tuple[str, str]] = None,
auth: Optional[Tuple[str, str]] = None, auth: Optional[Tuple[str, str]] = None,
verify: Optional[bool] = True, # <-- verify: Optional[bool] = True,
cert: Optional[Union[str, Tuple[str, str]]] = None, cert: Optional[Union[str, Tuple[str, str]]] = None,
impersonate: Optional[BrowserTypeLiteral] = "chrome136", # <-- impersonate: Optional[BrowserTypeLiteral] = DEFAULT_CHROME,
http3: Optional[bool] = False,
stealthy_headers: Optional[bool] = True, stealthy_headers: Optional[bool] = True,
**kwargs, **kwargs,
) -> Union[Response, Awaitable[Response]]: ) -> Union[Response, Awaitable[Response]]:
@@ -425,8 +448,9 @@ class FetcherSession:
:param auth: HTTP basic auth tuple of (username, password). Only basic auth is supported. :param auth: HTTP basic auth tuple of (username, password). Only basic auth is supported.
:param verify: Whether to verify HTTPS certificates. Defaults to True. :param verify: Whether to verify HTTPS certificates. Defaults to True.
:param cert: Tuple of (cert, key) filenames for the client certificate. :param cert: Tuple of (cert, key) filenames for the client certificate.
:param impersonate: Browser version to impersonate. Defaults to "chrome136". :param impersonate: Browser version to impersonate. Automatically defaults to the latest available Chrome version.
:param stealthy_headers: If enabled for this request (default), it creates and adds real browser headers. It also referer header as if it is from a Google search of URL's domain. :param http3: Whether to use HTTP3. Defaults to False. It might be problematic if used it with `impersonate`.
:param stealthy_headers: If enabled (default), it creates and adds real browser headers. It also sets the referer header as if this request came from a Google search of URL's domain.
:param kwargs: Additional keyword arguments to pass to the [`curl_cffi.requests.Session().request()`, `curl_cffi.requests.AsyncSession().request()`] method. :param kwargs: Additional keyword arguments to pass to the [`curl_cffi.requests.Session().request()`, `curl_cffi.requests.AsyncSession().request()`] method.
:return: A `Response` object or an awaitable for async. :return: A `Response` object or an awaitable for async.
""" """
@@ -449,6 +473,7 @@ class FetcherSession:
"auth": auth, "auth": auth,
"verify": verify, "verify": verify,
"cert": cert, "cert": cert,
"http3": http3,
**kwargs, **kwargs,
} }
return self.__prepare_and_dispatch( return self.__prepare_and_dispatch(
@@ -461,20 +486,21 @@ class FetcherSession:
data: Optional[Union[Dict, str]] = None, data: Optional[Union[Dict, str]] = None,
json: Optional[Union[Dict, List]] = None, json: Optional[Union[Dict, List]] = None,
headers: Optional[Mapping[str, Optional[str]]] = None, headers: Optional[Mapping[str, Optional[str]]] = None,
params: Optional[Union[Dict, List, Tuple]] = None, # <-- params: Optional[Union[Dict, List, Tuple]] = None,
cookies: Optional[CookieTypes] = None, # <-- cookies: Optional[CookieTypes] = None,
timeout: Optional[Union[int, float]] = 30, # <-- timeout: Optional[Union[int, float]] = 30,
follow_redirects: Optional[bool] = True, # <-- follow_redirects: Optional[bool] = True,
max_redirects: Optional[int] = 30, # <-- max_redirects: Optional[int] = 30,
retries: Optional[int] = 3, retries: Optional[int] = 3,
retry_delay: Optional[int] = 1, # <-- retry_delay: Optional[int] = 1,
proxies: Optional[ProxySpec] = None, # <-- proxies: Optional[ProxySpec] = None,
proxy: Optional[str] = None, # <-- proxy: Optional[str] = None,
proxy_auth: Optional[Tuple[str, str]] = None, proxy_auth: Optional[Tuple[str, str]] = None,
auth: Optional[Tuple[str, str]] = None, auth: Optional[Tuple[str, str]] = None,
verify: Optional[bool] = True, # <-- verify: Optional[bool] = True,
cert: Optional[Union[str, Tuple[str, str]]] = None, cert: Optional[Union[str, Tuple[str, str]]] = None,
impersonate: Optional[BrowserTypeLiteral] = "chrome136", # <-- impersonate: Optional[BrowserTypeLiteral] = DEFAULT_CHROME,
http3: Optional[bool] = False,
stealthy_headers: Optional[bool] = True, stealthy_headers: Optional[bool] = True,
**kwargs, **kwargs,
) -> Union[Response, Awaitable[Response]]: ) -> Union[Response, Awaitable[Response]]:
@@ -499,8 +525,9 @@ class FetcherSession:
:param auth: HTTP basic auth tuple of (username, password). Only basic auth is supported. :param auth: HTTP basic auth tuple of (username, password). Only basic auth is supported.
:param verify: Whether to verify HTTPS certificates. Defaults to True. :param verify: Whether to verify HTTPS certificates. Defaults to True.
:param cert: Tuple of (cert, key) filenames for the client certificate. :param cert: Tuple of (cert, key) filenames for the client certificate.
:param impersonate: Browser version to impersonate. Defaults to "chrome136". :param impersonate: Browser version to impersonate. Automatically defaults to the latest available Chrome version.
:param stealthy_headers: If enabled for this request (default), it creates and adds real browser headers. It also referer header as if it is from a Google search of URL's domain. :param http3: Whether to use HTTP3. Defaults to False. It might be problematic if used it with `impersonate`.
:param stealthy_headers: If enabled (default), it creates and adds real browser headers. It also sets the referer header as if this request came from a Google search of URL's domain.
:param kwargs: Additional keyword arguments to pass to the [`curl_cffi.requests.Session().request()`, `curl_cffi.requests.AsyncSession().request()`] method. :param kwargs: Additional keyword arguments to pass to the [`curl_cffi.requests.Session().request()`, `curl_cffi.requests.AsyncSession().request()`] method.
:return: A `Response` object or an awaitable for async. :return: A `Response` object or an awaitable for async.
""" """
@@ -523,6 +550,7 @@ class FetcherSession:
"auth": auth, "auth": auth,
"verify": verify, "verify": verify,
"cert": cert, "cert": cert,
"http3": http3,
**kwargs, **kwargs,
} }
return self.__prepare_and_dispatch( return self.__prepare_and_dispatch(
@@ -535,20 +563,21 @@ class FetcherSession:
data: Optional[Union[Dict, str]] = None, data: Optional[Union[Dict, str]] = None,
json: Optional[Union[Dict, List]] = None, json: Optional[Union[Dict, List]] = None,
headers: Optional[Mapping[str, Optional[str]]] = None, headers: Optional[Mapping[str, Optional[str]]] = None,
params: Optional[Union[Dict, List, Tuple]] = None, # <-- params: Optional[Union[Dict, List, Tuple]] = None,
cookies: Optional[CookieTypes] = None, # <-- cookies: Optional[CookieTypes] = None,
timeout: Optional[Union[int, float]] = 30, # <-- timeout: Optional[Union[int, float]] = 30,
follow_redirects: Optional[bool] = True, # <-- follow_redirects: Optional[bool] = True,
max_redirects: Optional[int] = 30, # <-- max_redirects: Optional[int] = 30,
retries: Optional[int] = 3, retries: Optional[int] = 3,
retry_delay: Optional[int] = 1, # <-- retry_delay: Optional[int] = 1,
proxies: Optional[ProxySpec] = None, # <-- proxies: Optional[ProxySpec] = None,
proxy: Optional[str] = None, # <-- proxy: Optional[str] = None,
proxy_auth: Optional[Tuple[str, str]] = None, proxy_auth: Optional[Tuple[str, str]] = None,
auth: Optional[Tuple[str, str]] = None, auth: Optional[Tuple[str, str]] = None,
verify: Optional[bool] = True, # <-- verify: Optional[bool] = True,
cert: Optional[Union[str, Tuple[str, str]]] = None, cert: Optional[Union[str, Tuple[str, str]]] = None,
impersonate: Optional[BrowserTypeLiteral] = "chrome136", # <-- impersonate: Optional[BrowserTypeLiteral] = DEFAULT_CHROME,
http3: Optional[bool] = False,
stealthy_headers: Optional[bool] = True, stealthy_headers: Optional[bool] = True,
**kwargs, **kwargs,
) -> Union[Response, Awaitable[Response]]: ) -> Union[Response, Awaitable[Response]]:
@@ -573,8 +602,9 @@ class FetcherSession:
:param auth: HTTP basic auth tuple of (username, password). Only basic auth is supported. :param auth: HTTP basic auth tuple of (username, password). Only basic auth is supported.
:param verify: Whether to verify HTTPS certificates. Defaults to True. :param verify: Whether to verify HTTPS certificates. Defaults to True.
:param cert: Tuple of (cert, key) filenames for the client certificate. :param cert: Tuple of (cert, key) filenames for the client certificate.
:param impersonate: Browser version to impersonate. Defaults to "chrome136". :param impersonate: Browser version to impersonate. Automatically defaults to the latest available Chrome version.
:param stealthy_headers: If enabled for this request (default), it creates and adds real browser headers. It also referer header as if it is from a Google search of URL's domain. :param http3: Whether to use HTTP3. Defaults to False. It might be problematic if used it with `impersonate`.
:param stealthy_headers: If enabled (default), it creates and adds real browser headers. It also sets the referer header as if this request came from a Google search of URL's domain.
:param kwargs: Additional keyword arguments to pass to the [`curl_cffi.requests.Session().request()`, `curl_cffi.requests.AsyncSession().request()`] method. :param kwargs: Additional keyword arguments to pass to the [`curl_cffi.requests.Session().request()`, `curl_cffi.requests.AsyncSession().request()`] method.
:return: A `Response` object or an awaitable for async. :return: A `Response` object or an awaitable for async.
""" """
@@ -599,6 +629,7 @@ class FetcherSession:
"auth": auth, "auth": auth,
"verify": verify, "verify": verify,
"cert": cert, "cert": cert,
"http3": http3,
**kwargs, **kwargs,
} }
return self.__prepare_and_dispatch( return self.__prepare_and_dispatch(
@@ -625,22 +656,23 @@ class AsyncFetcherClient:
@staticmethod @staticmethod
async def get( async def get(
url: str, url: str,
params: Optional[Union[Dict, List, Tuple]] = None, # <-- params: Optional[Union[Dict, List, Tuple]] = None,
headers: Optional[Mapping[str, Optional[str]]] = None, headers: Optional[Mapping[str, Optional[str]]] = None,
cookies: Optional[CookieTypes] = None, # <-- cookies: Optional[CookieTypes] = None,
timeout: Optional[Union[int, float]] = 30, # <-- timeout: Optional[Union[int, float]] = 30,
follow_redirects: Optional[bool] = True, # <-- follow_redirects: Optional[bool] = True,
max_redirects: Optional[int] = 30, # <-- max_redirects: Optional[int] = 30,
retries: Optional[int] = 3, retries: Optional[int] = 3,
retry_delay: Optional[int] = 1, # <-- retry_delay: Optional[int] = 1,
proxies: Optional[ProxySpec] = None, # <-- proxies: Optional[ProxySpec] = None,
proxy: Optional[str] = None, # <-- proxy: Optional[str] = None,
proxy_auth: Optional[Tuple[str, str]] = None, proxy_auth: Optional[Tuple[str, str]] = None,
auth: Optional[Tuple[str, str]] = None, auth: Optional[Tuple[str, str]] = None,
verify: Optional[bool] = True, # <-- verify: Optional[bool] = True,
cert: Optional[Union[str, Tuple[str, str]]] = None, cert: Optional[Union[str, Tuple[str, str]]] = None,
impersonate: Optional[BrowserTypeLiteral] = "chrome136", # <-- impersonate: Optional[BrowserTypeLiteral] = DEFAULT_CHROME,
stealthy_headers: Optional[bool] = True, stealthy_headers: Optional[bool] = True,
http3: Optional[bool] = False,
**kwargs, **kwargs,
) -> Response: ) -> Response:
""" """
@@ -662,8 +694,9 @@ class AsyncFetcherClient:
:param auth: HTTP basic auth tuple of (username, password). Only basic auth is supported. :param auth: HTTP basic auth tuple of (username, password). Only basic auth is supported.
:param verify: Whether to verify HTTPS certificates. :param verify: Whether to verify HTTPS certificates.
:param cert: Tuple of (cert, key) filenames for the client certificate. :param cert: Tuple of (cert, key) filenames for the client certificate.
:param impersonate: Browser version to impersonate. Defaults to "chrome136". :param impersonate: Browser version to impersonate. Automatically defaults to the latest available Chrome version.
:param stealthy_headers: If enabled for this request (default), it creates and adds real browser headers. It also referer header as if it is from a Google search of URL's domain. :param http3: Whether to use HTTP3. Defaults to False. It might be problematic if used it with `impersonate`.
:param stealthy_headers: If enabled (default), it creates and adds real browser headers. It also sets the referer header as if this request came from a Google search of URL's domain.
:param kwargs: Additional keyword arguments to pass to the `curl_cffi.requests.AsyncSession().request()` method. :param kwargs: Additional keyword arguments to pass to the `curl_cffi.requests.AsyncSession().request()` method.
:return: An awaitable `Response` object. :return: An awaitable `Response` object.
""" """
@@ -684,6 +717,7 @@ class AsyncFetcherClient:
"verify": verify, "verify": verify,
"cert": cert, "cert": cert,
"impersonate": impersonate, "impersonate": impersonate,
"http3": http3,
**kwargs, **kwargs,
} }
async with FetcherSession(stealthy_headers=stealthy_headers) as client: async with FetcherSession(stealthy_headers=stealthy_headers) as client:
@@ -695,21 +729,22 @@ class AsyncFetcherClient:
data: Optional[Union[Dict, str]] = None, data: Optional[Union[Dict, str]] = None,
json: Optional[Union[Dict, List]] = None, json: Optional[Union[Dict, List]] = None,
headers: Optional[Mapping[str, Optional[str]]] = None, headers: Optional[Mapping[str, Optional[str]]] = None,
params: Optional[Union[Dict, List, Tuple]] = None, # <-- params: Optional[Union[Dict, List, Tuple]] = None,
cookies: Optional[CookieTypes] = None, # <-- cookies: Optional[CookieTypes] = None,
timeout: Optional[Union[int, float]] = 30, # <-- timeout: Optional[Union[int, float]] = 30,
follow_redirects: Optional[bool] = True, # <-- follow_redirects: Optional[bool] = True,
max_redirects: Optional[int] = 30, # <-- max_redirects: Optional[int] = 30,
retries: Optional[int] = 3, retries: Optional[int] = 3,
retry_delay: Optional[int] = 1, # <-- retry_delay: Optional[int] = 1,
proxies: Optional[ProxySpec] = None, # <-- proxies: Optional[ProxySpec] = None,
proxy: Optional[str] = None, # <-- proxy: Optional[str] = None,
proxy_auth: Optional[Tuple[str, str]] = None, proxy_auth: Optional[Tuple[str, str]] = None,
auth: Optional[Tuple[str, str]] = None, auth: Optional[Tuple[str, str]] = None,
verify: Optional[bool] = True, # <-- verify: Optional[bool] = True,
cert: Optional[Union[str, Tuple[str, str]]] = None, cert: Optional[Union[str, Tuple[str, str]]] = None,
impersonate: Optional[BrowserTypeLiteral] = "chrome136", # <-- impersonate: Optional[BrowserTypeLiteral] = DEFAULT_CHROME,
stealthy_headers: Optional[bool] = True, stealthy_headers: Optional[bool] = True,
http3: Optional[bool] = False,
**kwargs, **kwargs,
) -> Response: ) -> Response:
""" """
@@ -733,8 +768,9 @@ class AsyncFetcherClient:
:param auth: HTTP basic auth tuple of (username, password). Only basic auth is supported. :param auth: HTTP basic auth tuple of (username, password). Only basic auth is supported.
:param verify: Whether to verify HTTPS certificates. Defaults to True. :param verify: Whether to verify HTTPS certificates. Defaults to True.
:param cert: Tuple of (cert, key) filenames for the client certificate. :param cert: Tuple of (cert, key) filenames for the client certificate.
:param impersonate: Browser version to impersonate. Defaults to "chrome136". :param impersonate: Browser version to impersonate. Automatically defaults to the latest available Chrome version.
:param stealthy_headers: If enabled for this request (default), it creates and adds real browser headers. It also referer header as if it is from a Google search of URL's domain. :param http3: Whether to use HTTP3. Defaults to False. It might be problematic if used it with `impersonate`.
:param stealthy_headers: If enabled (default), it creates and adds real browser headers. It also sets the referer header as if this request came from a Google search of URL's domain.
:param kwargs: Additional keyword arguments to pass to the `curl_cffi.requests.AsyncSession().request()` method. :param kwargs: Additional keyword arguments to pass to the `curl_cffi.requests.AsyncSession().request()` method.
:return: An awaitable `Response` object. :return: An awaitable `Response` object.
""" """
@@ -757,6 +793,7 @@ class AsyncFetcherClient:
"auth": auth, "auth": auth,
"verify": verify, "verify": verify,
"cert": cert, "cert": cert,
"http3": http3,
**kwargs, **kwargs,
} }
async with FetcherSession(stealthy_headers=stealthy_headers) as client: async with FetcherSession(stealthy_headers=stealthy_headers) as client:
@@ -768,21 +805,22 @@ class AsyncFetcherClient:
data: Optional[Union[Dict, str]] = None, data: Optional[Union[Dict, str]] = None,
json: Optional[Union[Dict, List]] = None, json: Optional[Union[Dict, List]] = None,
headers: Optional[Mapping[str, Optional[str]]] = None, headers: Optional[Mapping[str, Optional[str]]] = None,
params: Optional[Union[Dict, List, Tuple]] = None, # <-- params: Optional[Union[Dict, List, Tuple]] = None,
cookies: Optional[CookieTypes] = None, # <-- cookies: Optional[CookieTypes] = None,
timeout: Optional[Union[int, float]] = 30, # <-- timeout: Optional[Union[int, float]] = 30,
follow_redirects: Optional[bool] = True, # <-- follow_redirects: Optional[bool] = True,
max_redirects: Optional[int] = 30, # <-- max_redirects: Optional[int] = 30,
retries: Optional[int] = 3, retries: Optional[int] = 3,
retry_delay: Optional[int] = 1, # <-- retry_delay: Optional[int] = 1,
proxies: Optional[ProxySpec] = None, # <-- proxies: Optional[ProxySpec] = None,
proxy: Optional[str] = None, # <-- proxy: Optional[str] = None,
proxy_auth: Optional[Tuple[str, str]] = None, proxy_auth: Optional[Tuple[str, str]] = None,
auth: Optional[Tuple[str, str]] = None, auth: Optional[Tuple[str, str]] = None,
verify: Optional[bool] = True, # <-- verify: Optional[bool] = True,
cert: Optional[Union[str, Tuple[str, str]]] = None, cert: Optional[Union[str, Tuple[str, str]]] = None,
impersonate: Optional[BrowserTypeLiteral] = "chrome136", # <-- impersonate: Optional[BrowserTypeLiteral] = DEFAULT_CHROME,
stealthy_headers: Optional[bool] = True, stealthy_headers: Optional[bool] = True,
http3: Optional[bool] = False,
**kwargs, **kwargs,
) -> Response: ) -> Response:
""" """
@@ -806,8 +844,9 @@ class AsyncFetcherClient:
:param auth: HTTP basic auth tuple of (username, password). Only basic auth is supported. :param auth: HTTP basic auth tuple of (username, password). Only basic auth is supported.
:param verify: Whether to verify HTTPS certificates. Defaults to True. :param verify: Whether to verify HTTPS certificates. Defaults to True.
:param cert: Tuple of (cert, key) filenames for the client certificate. :param cert: Tuple of (cert, key) filenames for the client certificate.
:param impersonate: Browser version to impersonate. Defaults to "chrome136". :param impersonate: Browser version to impersonate. Automatically defaults to the latest available Chrome version.
:param stealthy_headers: If enabled for this request (default), it creates and adds real browser headers. It also referer header as if it is from a Google search of URL's domain. :param http3: Whether to use HTTP3. Defaults to False. It might be problematic if used it with `impersonate`.
:param stealthy_headers: If enabled (default), it creates and adds real browser headers. It also sets the referer header as if this request came from a Google search of URL's domain.
:param kwargs: Additional keyword arguments to pass to the `curl_cffi.requests.AsyncSession().request()` method. :param kwargs: Additional keyword arguments to pass to the `curl_cffi.requests.AsyncSession().request()` method.
:return: An awaitable `Response` object. :return: An awaitable `Response` object.
""" """
@@ -830,6 +869,7 @@ class AsyncFetcherClient:
"auth": auth, "auth": auth,
"verify": verify, "verify": verify,
"cert": cert, "cert": cert,
"http3": http3,
**kwargs, **kwargs,
} }
async with FetcherSession(stealthy_headers=stealthy_headers) as client: async with FetcherSession(stealthy_headers=stealthy_headers) as client:
@@ -841,21 +881,22 @@ class AsyncFetcherClient:
data: Optional[Union[Dict, str]] = None, data: Optional[Union[Dict, str]] = None,
json: Optional[Union[Dict, List]] = None, json: Optional[Union[Dict, List]] = None,
headers: Optional[Mapping[str, Optional[str]]] = None, headers: Optional[Mapping[str, Optional[str]]] = None,
params: Optional[Union[Dict, List, Tuple]] = None, # <-- params: Optional[Union[Dict, List, Tuple]] = None,
cookies: Optional[CookieTypes] = None, # <-- cookies: Optional[CookieTypes] = None,
timeout: Optional[Union[int, float]] = 30, # <-- timeout: Optional[Union[int, float]] = 30,
follow_redirects: Optional[bool] = True, # <-- follow_redirects: Optional[bool] = True,
max_redirects: Optional[int] = 30, # <-- max_redirects: Optional[int] = 30,
retries: Optional[int] = 3, retries: Optional[int] = 3,
retry_delay: Optional[int] = 1, # <-- retry_delay: Optional[int] = 1,
proxies: Optional[ProxySpec] = None, # <-- proxies: Optional[ProxySpec] = None,
proxy: Optional[str] = None, # <-- proxy: Optional[str] = None,
proxy_auth: Optional[Tuple[str, str]] = None, proxy_auth: Optional[Tuple[str, str]] = None,
auth: Optional[Tuple[str, str]] = None, auth: Optional[Tuple[str, str]] = None,
verify: Optional[bool] = True, # <-- verify: Optional[bool] = True,
cert: Optional[Union[str, Tuple[str, str]]] = None, cert: Optional[Union[str, Tuple[str, str]]] = None,
impersonate: Optional[BrowserTypeLiteral] = "chrome136", # <-- impersonate: Optional[BrowserTypeLiteral] = DEFAULT_CHROME,
stealthy_headers: Optional[bool] = True, stealthy_headers: Optional[bool] = True,
http3: Optional[bool] = False,
**kwargs, **kwargs,
) -> Response: ) -> Response:
""" """
@@ -879,8 +920,9 @@ class AsyncFetcherClient:
:param auth: HTTP basic auth tuple of (username, password). Only basic auth is supported. :param auth: HTTP basic auth tuple of (username, password). Only basic auth is supported.
:param verify: Whether to verify HTTPS certificates. Defaults to True. :param verify: Whether to verify HTTPS certificates. Defaults to True.
:param cert: Tuple of (cert, key) filenames for the client certificate. :param cert: Tuple of (cert, key) filenames for the client certificate.
:param impersonate: Browser version to impersonate. Defaults to "chrome136". :param impersonate: Browser version to impersonate. Automatically defaults to the latest available Chrome version.
:param stealthy_headers: If enabled for this request (default), it creates and adds real browser headers. It also referer header as if it is from a Google search of URL's domain. :param http3: Whether to use HTTP3. Defaults to False. It might be problematic if used it with `impersonate`.
:param stealthy_headers: If enabled (default), it creates and adds real browser headers. It also sets the referer header as if this request came from a Google search of URL's domain.
:param kwargs: Additional keyword arguments to pass to the `curl_cffi.requests.AsyncSession().request()` method. :param kwargs: Additional keyword arguments to pass to the `curl_cffi.requests.AsyncSession().request()` method.
:return: An awaitable `Response` object. :return: An awaitable `Response` object.
""" """
@@ -905,6 +947,7 @@ class AsyncFetcherClient:
"auth": auth, "auth": auth,
"verify": verify, "verify": verify,
"cert": cert, "cert": cert,
"http3": http3,
**kwargs, **kwargs,
} }
async with FetcherSession(stealthy_headers=stealthy_headers) as client: async with FetcherSession(stealthy_headers=stealthy_headers) as client:
scrapling/engines/toolbelt/fingerprints.py
+9 -9
View File
@@ -14,7 +14,7 @@ from scrapling.core.utils import lru_cache
@lru_cache(10, typed=True) @lru_cache(10, typed=True)
def generate_convincing_referer(url: str) -> str: def generate_convincing_referer(url: str) -> str:
"""Takes the domain from the URL without the subdomain/suffix and make it look like you were searching google for this website """Takes the domain from the URL without the subdomain/suffix and make it look like you were searching Google for this website
>>> generate_convincing_referer('https://www.somewebsite.com/blah') >>> generate_convincing_referer('https://www.somewebsite.com/blah')
'https://www.google.com/search?q=somewebsite' 'https://www.google.com/search?q=somewebsite'
@@ -38,13 +38,13 @@ def get_os_name() -> Union[str, None]:
"Linux": "linux", "Linux": "linux",
"Darwin": "macos", "Darwin": "macos",
"Windows": "windows", "Windows": "windows",
# For the future? because why not # For the future? because why not?
"iOS": "ios", "iOS": "ios",
}.get(os_name) }.get(os_name)
def generate_suitable_fingerprint() -> Fingerprint: def generate_suitable_fingerprint() -> Fingerprint:
"""Generates a browserforge's fingerprint that matches current OS, desktop device, and Chrome with version 128 at least. """Generates a browserforge's fingerprint that matches the current OS, desktop device, and Chrome with version 128 at least.
This function was originally created to test Browserforge's injector. This function was originally created to test Browserforge's injector.
:return: `Fingerprint` object :return: `Fingerprint` object
@@ -59,11 +59,11 @@ def generate_suitable_fingerprint() -> Fingerprint:
def generate_headers(browser_mode: bool = False) -> Dict: def generate_headers(browser_mode: bool = False) -> Dict:
"""Generate real browser-like headers using browserforge's generator """Generate real browser-like headers using browserforge's generator
:param browser_mode: If enabled, the headers created are used for playwright so it have to match everything :param browser_mode: If enabled, the headers created are used for playwright, so it has to match everything
:return: A dictionary of the generated headers :return: A dictionary of the generated headers
""" """
if browser_mode: if browser_mode:
# In this mode we don't care about anything other than matching the OS and the browser type with the browser we are using # In this mode we don't care about anything other than matching the OS and the browser type with the browser we are using,
# So we don't raise any inconsistency red flags while websites fingerprinting us # So we don't raise any inconsistency red flags while websites fingerprinting us
os_name = get_os_name() os_name = get_os_name()
return HeaderGenerator( return HeaderGenerator(
@@ -72,10 +72,10 @@ def generate_headers(browser_mode: bool = False) -> Dict:
device="desktop", device="desktop",
).generate() ).generate()
else: else:
# Here it's used for normal requests that aren't done through browsers so we can take it lightly # Here it's used for normal requests that aren't done through browsers
browsers = [ browsers = [
Browser(name="chrome", min_version=120), Browser(name="chrome", min_version=130),
Browser(name="firefox", min_version=120), Browser(name="firefox", min_version=130),
Browser(name="edge", min_version=120), Browser(name="edge", min_version=130),
] ]
return HeaderGenerator(browser=browsers, device="desktop").generate() return HeaderGenerator(browser=browsers, device="desktop").generate()