freedak/Scrapling
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat/fix(Fetcher): Adding http3 support + Fix stealthy_headers

Browse Source 
And some fixes here and there to the docs
This commit is contained in:
Karim shoair
2025-05-31 03:35:46 +03:00
parent af643e4c01
commit 178eed7533
2 changed files with 168 additions and 125 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scrapling/engines/static.py
+159 -116
View File
@@ -2,6 +2,8 @@ from time import sleep as time_sleep
from asyncio import sleep as asyncio_sleep
from curl_cffi.requests.session import CurlError
from curl_cffi import CurlHttpVersion
from curl_cffi.requests.impersonate import DEFAULT_CHROME
from curl_cffi.requests import (
ProxySpec,
CookieTypes,
@@ -46,7 +48,8 @@ class FetcherSession:
def __init__(
self,
impersonate: Optional[str] = "chrome136",
impersonate: Optional[BrowserTypeLiteral] = DEFAULT_CHROME,
http3: Optional[bool] = False,
stealthy_headers: Optional[bool] = True,
proxies: Optional[Dict[str, str]] = None,
proxy: Optional[str] = None,
@@ -62,8 +65,9 @@ class FetcherSession:
adaptor_arguments: Optional[Dict] = None,
):
"""
:param impersonate: Browser version to impersonate. Defaults to "chrome136".
:param stealthy_headers: If enabled (default), it creates and adds real browser headers. It also referer header as if it is from a Google search of URL's domain.
:param impersonate: Browser version to impersonate. Automatically defaults to the latest available Chrome version.
:param http3: Whether to use HTTP3. Defaults to False. It might be problematic if used it with `impersonate`.
:param stealthy_headers: If enabled (default), it creates and adds real browser headers. It also sets the referer header as if this request came from a Google search of URL's domain.
:param proxies: Dict of proxies to use. Format: {"http": proxy_url, "https": proxy_url}.
:param proxy: Proxy URL to use. Format: "http://username:password@localhost:8030".
Cannot be used together with the `proxies` parameter.
@@ -91,6 +95,7 @@ class FetcherSession:
self.default_max_redirects = max_redirects
self.default_verify = verify
self.default_cert = cert
self.default_http3 = http3
self.adaptor_arguments = adaptor_arguments or {}
self._curl_session: Optional[CurlSession] = None
@@ -98,23 +103,37 @@ class FetcherSession:
def _merge_request_args(self, **kwargs) -> Dict[str, Any]:
"""Merge request-specific arguments with default session arguments."""
request_args = {
"headers": self._headers_job(
kwargs["url"], kwargs.get("headers"), kwargs.pop("stealth")
),
"proxies": kwargs.get("proxies", self.default_proxies),
"proxy": kwargs.get("proxy", self.default_proxy),
"proxy_auth": kwargs.get("proxy_auth", self.default_proxy_auth),
"timeout": kwargs.get("timeout", self.default_timeout),
"allow_redirects": kwargs.get(
"follow_redirects", self.default_follow_redirects
),
"max_redirects": kwargs.get("max_redirects", self.default_max_redirects),
"verify": kwargs.get("verify", self.default_verify),
"cert": kwargs.get("cert", self.default_cert),
"impersonate": kwargs.get("impersonate", self.default_impersonate),
**kwargs,
}
url = kwargs.pop("url")
request_args = {}
if kwargs.pop("http3", False) or self.default_http3:
request_args["http_version"] = CurlHttpVersion.V3ONLY
if kwargs.get("impersonate"):
log.warning(
"The argument `http3` might cause errors if used with `impersonate` argument, try switching it off if you encounter any curl errors."
)
request_args.update(
{
"url": url,
"headers": self._headers_job(
url, kwargs.pop("headers"), kwargs.pop("stealth")
),
"proxies": kwargs.pop("proxies", self.default_proxies),
"proxy": kwargs.pop("proxy", self.default_proxy),
"proxy_auth": kwargs.pop("proxy_auth", self.default_proxy_auth),
"timeout": kwargs.pop("timeout", self.default_timeout),
"allow_redirects": kwargs.pop(
"follow_redirects", self.default_follow_redirects
),
"max_redirects": kwargs.pop(
"max_redirects", self.default_max_redirects
),
"verify": kwargs.pop("verify", self.default_verify),
"cert": kwargs.pop("cert", self.default_cert),
"impersonate": kwargs.pop("impersonate", self.default_impersonate),
**kwargs,
}
)
return request_args
def _headers_job(
@@ -316,21 +335,22 @@ class FetcherSession:
def get(
self,
url: str,
params: Optional[Union[Dict, List, Tuple]] = None, # <--
params: Optional[Union[Dict, List, Tuple]] = None,
headers: Optional[Mapping[str, Optional[str]]] = None,
cookies: Optional[CookieTypes] = None, # <--
timeout: Optional[Union[int, float]] = 30, # <--
follow_redirects: Optional[bool] = True, # <--
max_redirects: Optional[int] = 30, # <--
cookies: Optional[CookieTypes] = None,
timeout: Optional[Union[int, float]] = 30,
follow_redirects: Optional[bool] = True,
max_redirects: Optional[int] = 30,
retries: Optional[int] = 3,
retry_delay: Optional[int] = 1, # <--
proxies: Optional[ProxySpec] = None, # <--
proxy: Optional[str] = None, # <--
retry_delay: Optional[int] = 1,
proxies: Optional[ProxySpec] = None,
proxy: Optional[str] = None,
proxy_auth: Optional[Tuple[str, str]] = None,
auth: Optional[Tuple[str, str]] = None,
verify: Optional[bool] = True, # <--
verify: Optional[bool] = True,
cert: Optional[Union[str, Tuple[str, str]]] = None,
impersonate: Optional[BrowserTypeLiteral] = "chrome136", # <--
impersonate: Optional[BrowserTypeLiteral] = DEFAULT_CHROME,
http3: Optional[bool] = False,
stealthy_headers: Optional[bool] = True,
**kwargs,
) -> Union[Response, Awaitable[Response]]:
@@ -353,8 +373,9 @@ class FetcherSession:
:param auth: HTTP basic auth tuple of (username, password). Only basic auth is supported.
:param verify: Whether to verify HTTPS certificates.
:param cert: Tuple of (cert, key) filenames for the client certificate.
:param impersonate: Browser version to impersonate. Defaults to "chrome136".
:param stealthy_headers: If enabled for this request (default), it creates and adds real browser headers. It also referer header as if it is from a Google search of URL's domain.
:param impersonate: Browser version to impersonate. Automatically defaults to the latest available Chrome version.
:param http3: Whether to use HTTP3. Defaults to False. It might be problematic if used it with `impersonate`.
:param stealthy_headers: If enabled (default), it creates and adds real browser headers. It also sets the referer header as if this request came from a Google search of URL's domain.
:param kwargs: Additional keyword arguments to pass to the [`curl_cffi.requests.Session().request()`, `curl_cffi.requests.AsyncSession().request()`] method.
:return: A `Response` object or an awaitable for async.
"""
@@ -375,6 +396,7 @@ class FetcherSession:
"verify": verify,
"cert": cert,
"impersonate": impersonate,
"http3": http3,
**kwargs,
}
return self.__prepare_and_dispatch(
@@ -387,20 +409,21 @@ class FetcherSession:
data: Optional[Union[Dict, str]] = None,
json: Optional[Union[Dict, List]] = None,
headers: Optional[Mapping[str, Optional[str]]] = None,
params: Optional[Union[Dict, List, Tuple]] = None, # <--
cookies: Optional[CookieTypes] = None, # <--
timeout: Optional[Union[int, float]] = 30, # <--
follow_redirects: Optional[bool] = True, # <--
max_redirects: Optional[int] = 30, # <--
params: Optional[Union[Dict, List, Tuple]] = None,
cookies: Optional[CookieTypes] = None,
timeout: Optional[Union[int, float]] = 30,
follow_redirects: Optional[bool] = True,
max_redirects: Optional[int] = 30,
retries: Optional[int] = 3,
retry_delay: Optional[int] = 1, # <--
proxies: Optional[ProxySpec] = None, # <--
proxy: Optional[str] = None, # <--
retry_delay: Optional[int] = 1,
proxies: Optional[ProxySpec] = None,
proxy: Optional[str] = None,
proxy_auth: Optional[Tuple[str, str]] = None,
auth: Optional[Tuple[str, str]] = None,
verify: Optional[bool] = True, # <--
verify: Optional[bool] = True,
cert: Optional[Union[str, Tuple[str, str]]] = None,
impersonate: Optional[BrowserTypeLiteral] = "chrome136", # <--
impersonate: Optional[BrowserTypeLiteral] = DEFAULT_CHROME,
http3: Optional[bool] = False,
stealthy_headers: Optional[bool] = True,
**kwargs,
) -> Union[Response, Awaitable[Response]]:
@@ -425,8 +448,9 @@ class FetcherSession:
:param auth: HTTP basic auth tuple of (username, password). Only basic auth is supported.
:param verify: Whether to verify HTTPS certificates. Defaults to True.
:param cert: Tuple of (cert, key) filenames for the client certificate.
:param impersonate: Browser version to impersonate. Defaults to "chrome136".
:param stealthy_headers: If enabled for this request (default), it creates and adds real browser headers. It also referer header as if it is from a Google search of URL's domain.
:param impersonate: Browser version to impersonate. Automatically defaults to the latest available Chrome version.
:param http3: Whether to use HTTP3. Defaults to False. It might be problematic if used it with `impersonate`.
:param stealthy_headers: If enabled (default), it creates and adds real browser headers. It also sets the referer header as if this request came from a Google search of URL's domain.
:param kwargs: Additional keyword arguments to pass to the [`curl_cffi.requests.Session().request()`, `curl_cffi.requests.AsyncSession().request()`] method.
:return: A `Response` object or an awaitable for async.
"""
@@ -449,6 +473,7 @@ class FetcherSession:
"auth": auth,
"verify": verify,
"cert": cert,
"http3": http3,
**kwargs,
}
return self.__prepare_and_dispatch(
@@ -461,20 +486,21 @@ class FetcherSession:
data: Optional[Union[Dict, str]] = None,
json: Optional[Union[Dict, List]] = None,
headers: Optional[Mapping[str, Optional[str]]] = None,
params: Optional[Union[Dict, List, Tuple]] = None, # <--
cookies: Optional[CookieTypes] = None, # <--
timeout: Optional[Union[int, float]] = 30, # <--
follow_redirects: Optional[bool] = True, # <--
max_redirects: Optional[int] = 30, # <--
params: Optional[Union[Dict, List, Tuple]] = None,
cookies: Optional[CookieTypes] = None,
timeout: Optional[Union[int, float]] = 30,
follow_redirects: Optional[bool] = True,
max_redirects: Optional[int] = 30,
retries: Optional[int] = 3,
retry_delay: Optional[int] = 1, # <--
proxies: Optional[ProxySpec] = None, # <--
proxy: Optional[str] = None, # <--
retry_delay: Optional[int] = 1,
proxies: Optional[ProxySpec] = None,
proxy: Optional[str] = None,
proxy_auth: Optional[Tuple[str, str]] = None,
auth: Optional[Tuple[str, str]] = None,
verify: Optional[bool] = True, # <--
verify: Optional[bool] = True,
cert: Optional[Union[str, Tuple[str, str]]] = None,
impersonate: Optional[BrowserTypeLiteral] = "chrome136", # <--
impersonate: Optional[BrowserTypeLiteral] = DEFAULT_CHROME,
http3: Optional[bool] = False,
stealthy_headers: Optional[bool] = True,
**kwargs,
) -> Union[Response, Awaitable[Response]]:
@@ -499,8 +525,9 @@ class FetcherSession:
:param auth: HTTP basic auth tuple of (username, password). Only basic auth is supported.
:param verify: Whether to verify HTTPS certificates. Defaults to True.
:param cert: Tuple of (cert, key) filenames for the client certificate.
:param impersonate: Browser version to impersonate. Defaults to "chrome136".
:param stealthy_headers: If enabled for this request (default), it creates and adds real browser headers. It also referer header as if it is from a Google search of URL's domain.
:param impersonate: Browser version to impersonate. Automatically defaults to the latest available Chrome version.
:param http3: Whether to use HTTP3. Defaults to False. It might be problematic if used it with `impersonate`.
:param stealthy_headers: If enabled (default), it creates and adds real browser headers. It also sets the referer header as if this request came from a Google search of URL's domain.
:param kwargs: Additional keyword arguments to pass to the [`curl_cffi.requests.Session().request()`, `curl_cffi.requests.AsyncSession().request()`] method.
:return: A `Response` object or an awaitable for async.
"""
@@ -523,6 +550,7 @@ class FetcherSession:
"auth": auth,
"verify": verify,
"cert": cert,
"http3": http3,
**kwargs,
}
return self.__prepare_and_dispatch(
@@ -535,20 +563,21 @@ class FetcherSession:
data: Optional[Union[Dict, str]] = None,
json: Optional[Union[Dict, List]] = None,
headers: Optional[Mapping[str, Optional[str]]] = None,
params: Optional[Union[Dict, List, Tuple]] = None, # <--
cookies: Optional[CookieTypes] = None, # <--
timeout: Optional[Union[int, float]] = 30, # <--
follow_redirects: Optional[bool] = True, # <--
max_redirects: Optional[int] = 30, # <--
params: Optional[Union[Dict, List, Tuple]] = None,
cookies: Optional[CookieTypes] = None,
timeout: Optional[Union[int, float]] = 30,
follow_redirects: Optional[bool] = True,
max_redirects: Optional[int] = 30,
retries: Optional[int] = 3,
retry_delay: Optional[int] = 1, # <--
proxies: Optional[ProxySpec] = None, # <--
proxy: Optional[str] = None, # <--
retry_delay: Optional[int] = 1,
proxies: Optional[ProxySpec] = None,
proxy: Optional[str] = None,
proxy_auth: Optional[Tuple[str, str]] = None,
auth: Optional[Tuple[str, str]] = None,
verify: Optional[bool] = True, # <--
verify: Optional[bool] = True,
cert: Optional[Union[str, Tuple[str, str]]] = None,
impersonate: Optional[BrowserTypeLiteral] = "chrome136", # <--
impersonate: Optional[BrowserTypeLiteral] = DEFAULT_CHROME,
http3: Optional[bool] = False,
stealthy_headers: Optional[bool] = True,
**kwargs,
) -> Union[Response, Awaitable[Response]]:
@@ -573,8 +602,9 @@ class FetcherSession:
:param auth: HTTP basic auth tuple of (username, password). Only basic auth is supported.
:param verify: Whether to verify HTTPS certificates. Defaults to True.
:param cert: Tuple of (cert, key) filenames for the client certificate.
:param impersonate: Browser version to impersonate. Defaults to "chrome136".
:param stealthy_headers: If enabled for this request (default), it creates and adds real browser headers. It also referer header as if it is from a Google search of URL's domain.
:param impersonate: Browser version to impersonate. Automatically defaults to the latest available Chrome version.
:param http3: Whether to use HTTP3. Defaults to False. It might be problematic if used it with `impersonate`.
:param stealthy_headers: If enabled (default), it creates and adds real browser headers. It also sets the referer header as if this request came from a Google search of URL's domain.
:param kwargs: Additional keyword arguments to pass to the [`curl_cffi.requests.Session().request()`, `curl_cffi.requests.AsyncSession().request()`] method.
:return: A `Response` object or an awaitable for async.
"""
@@ -599,6 +629,7 @@ class FetcherSession:
"auth": auth,
"verify": verify,
"cert": cert,
"http3": http3,
**kwargs,
}
return self.__prepare_and_dispatch(
@@ -625,22 +656,23 @@ class AsyncFetcherClient:
@staticmethod
async def get(
url: str,
params: Optional[Union[Dict, List, Tuple]] = None, # <--
params: Optional[Union[Dict, List, Tuple]] = None,
headers: Optional[Mapping[str, Optional[str]]] = None,
cookies: Optional[CookieTypes] = None, # <--
timeout: Optional[Union[int, float]] = 30, # <--
follow_redirects: Optional[bool] = True, # <--
max_redirects: Optional[int] = 30, # <--
cookies: Optional[CookieTypes] = None,
timeout: Optional[Union[int, float]] = 30,
follow_redirects: Optional[bool] = True,
max_redirects: Optional[int] = 30,
retries: Optional[int] = 3,
retry_delay: Optional[int] = 1, # <--
proxies: Optional[ProxySpec] = None, # <--
proxy: Optional[str] = None, # <--
retry_delay: Optional[int] = 1,
proxies: Optional[ProxySpec] = None,
proxy: Optional[str] = None,
proxy_auth: Optional[Tuple[str, str]] = None,
auth: Optional[Tuple[str, str]] = None,
verify: Optional[bool] = True, # <--
verify: Optional[bool] = True,
cert: Optional[Union[str, Tuple[str, str]]] = None,
impersonate: Optional[BrowserTypeLiteral] = "chrome136", # <--
impersonate: Optional[BrowserTypeLiteral] = DEFAULT_CHROME,
stealthy_headers: Optional[bool] = True,
http3: Optional[bool] = False,
**kwargs,
) -> Response:
"""
@@ -662,8 +694,9 @@ class AsyncFetcherClient:
:param auth: HTTP basic auth tuple of (username, password). Only basic auth is supported.
:param verify: Whether to verify HTTPS certificates.
:param cert: Tuple of (cert, key) filenames for the client certificate.
:param impersonate: Browser version to impersonate. Defaults to "chrome136".
:param stealthy_headers: If enabled for this request (default), it creates and adds real browser headers. It also referer header as if it is from a Google search of URL's domain.
:param impersonate: Browser version to impersonate. Automatically defaults to the latest available Chrome version.
:param http3: Whether to use HTTP3. Defaults to False. It might be problematic if used it with `impersonate`.
:param stealthy_headers: If enabled (default), it creates and adds real browser headers. It also sets the referer header as if this request came from a Google search of URL's domain.
:param kwargs: Additional keyword arguments to pass to the `curl_cffi.requests.AsyncSession().request()` method.
:return: An awaitable `Response` object.
"""
@@ -684,6 +717,7 @@ class AsyncFetcherClient:
"verify": verify,
"cert": cert,
"impersonate": impersonate,
"http3": http3,
**kwargs,
}
async with FetcherSession(stealthy_headers=stealthy_headers) as client:
@@ -695,21 +729,22 @@ class AsyncFetcherClient:
data: Optional[Union[Dict, str]] = None,
json: Optional[Union[Dict, List]] = None,
headers: Optional[Mapping[str, Optional[str]]] = None,
params: Optional[Union[Dict, List, Tuple]] = None, # <--
cookies: Optional[CookieTypes] = None, # <--
timeout: Optional[Union[int, float]] = 30, # <--
follow_redirects: Optional[bool] = True, # <--
max_redirects: Optional[int] = 30, # <--
params: Optional[Union[Dict, List, Tuple]] = None,
cookies: Optional[CookieTypes] = None,
timeout: Optional[Union[int, float]] = 30,
follow_redirects: Optional[bool] = True,
max_redirects: Optional[int] = 30,
retries: Optional[int] = 3,
retry_delay: Optional[int] = 1, # <--
proxies: Optional[ProxySpec] = None, # <--
proxy: Optional[str] = None, # <--
retry_delay: Optional[int] = 1,
proxies: Optional[ProxySpec] = None,
proxy: Optional[str] = None,
proxy_auth: Optional[Tuple[str, str]] = None,
auth: Optional[Tuple[str, str]] = None,
verify: Optional[bool] = True, # <--
verify: Optional[bool] = True,
cert: Optional[Union[str, Tuple[str, str]]] = None,
impersonate: Optional[BrowserTypeLiteral] = "chrome136", # <--
impersonate: Optional[BrowserTypeLiteral] = DEFAULT_CHROME,
stealthy_headers: Optional[bool] = True,
http3: Optional[bool] = False,
**kwargs,
) -> Response:
"""
@@ -733,8 +768,9 @@ class AsyncFetcherClient:
:param auth: HTTP basic auth tuple of (username, password). Only basic auth is supported.
:param verify: Whether to verify HTTPS certificates. Defaults to True.
:param cert: Tuple of (cert, key) filenames for the client certificate.
:param impersonate: Browser version to impersonate. Defaults to "chrome136".
:param stealthy_headers: If enabled for this request (default), it creates and adds real browser headers. It also referer header as if it is from a Google search of URL's domain.
:param impersonate: Browser version to impersonate. Automatically defaults to the latest available Chrome version.
:param http3: Whether to use HTTP3. Defaults to False. It might be problematic if used it with `impersonate`.
:param stealthy_headers: If enabled (default), it creates and adds real browser headers. It also sets the referer header as if this request came from a Google search of URL's domain.
:param kwargs: Additional keyword arguments to pass to the `curl_cffi.requests.AsyncSession().request()` method.
:return: An awaitable `Response` object.
"""
@@ -757,6 +793,7 @@ class AsyncFetcherClient:
"auth": auth,
"verify": verify,
"cert": cert,
"http3": http3,
**kwargs,
}
async with FetcherSession(stealthy_headers=stealthy_headers) as client:
@@ -768,21 +805,22 @@ class AsyncFetcherClient:
data: Optional[Union[Dict, str]] = None,
json: Optional[Union[Dict, List]] = None,
headers: Optional[Mapping[str, Optional[str]]] = None,
params: Optional[Union[Dict, List, Tuple]] = None, # <--
cookies: Optional[CookieTypes] = None, # <--
timeout: Optional[Union[int, float]] = 30, # <--
follow_redirects: Optional[bool] = True, # <--
max_redirects: Optional[int] = 30, # <--
params: Optional[Union[Dict, List, Tuple]] = None,
cookies: Optional[CookieTypes] = None,
timeout: Optional[Union[int, float]] = 30,
follow_redirects: Optional[bool] = True,
max_redirects: Optional[int] = 30,
retries: Optional[int] = 3,
retry_delay: Optional[int] = 1, # <--
proxies: Optional[ProxySpec] = None, # <--
proxy: Optional[str] = None, # <--
retry_delay: Optional[int] = 1,
proxies: Optional[ProxySpec] = None,
proxy: Optional[str] = None,
proxy_auth: Optional[Tuple[str, str]] = None,
auth: Optional[Tuple[str, str]] = None,
verify: Optional[bool] = True, # <--
verify: Optional[bool] = True,
cert: Optional[Union[str, Tuple[str, str]]] = None,
impersonate: Optional[BrowserTypeLiteral] = "chrome136", # <--
impersonate: Optional[BrowserTypeLiteral] = DEFAULT_CHROME,
stealthy_headers: Optional[bool] = True,
http3: Optional[bool] = False,
**kwargs,
) -> Response:
"""
@@ -806,8 +844,9 @@ class AsyncFetcherClient:
:param auth: HTTP basic auth tuple of (username, password). Only basic auth is supported.
:param verify: Whether to verify HTTPS certificates. Defaults to True.
:param cert: Tuple of (cert, key) filenames for the client certificate.
:param impersonate: Browser version to impersonate. Defaults to "chrome136".
:param stealthy_headers: If enabled for this request (default), it creates and adds real browser headers. It also referer header as if it is from a Google search of URL's domain.
:param impersonate: Browser version to impersonate. Automatically defaults to the latest available Chrome version.
:param http3: Whether to use HTTP3. Defaults to False. It might be problematic if used it with `impersonate`.
:param stealthy_headers: If enabled (default), it creates and adds real browser headers. It also sets the referer header as if this request came from a Google search of URL's domain.
:param kwargs: Additional keyword arguments to pass to the `curl_cffi.requests.AsyncSession().request()` method.
:return: An awaitable `Response` object.
"""
@@ -830,6 +869,7 @@ class AsyncFetcherClient:
"auth": auth,
"verify": verify,
"cert": cert,
"http3": http3,
**kwargs,
}
async with FetcherSession(stealthy_headers=stealthy_headers) as client:
@@ -841,21 +881,22 @@ class AsyncFetcherClient:
data: Optional[Union[Dict, str]] = None,
json: Optional[Union[Dict, List]] = None,
headers: Optional[Mapping[str, Optional[str]]] = None,
params: Optional[Union[Dict, List, Tuple]] = None, # <--
cookies: Optional[CookieTypes] = None, # <--
timeout: Optional[Union[int, float]] = 30, # <--
follow_redirects: Optional[bool] = True, # <--
max_redirects: Optional[int] = 30, # <--
params: Optional[Union[Dict, List, Tuple]] = None,
cookies: Optional[CookieTypes] = None,
timeout: Optional[Union[int, float]] = 30,
follow_redirects: Optional[bool] = True,
max_redirects: Optional[int] = 30,
retries: Optional[int] = 3,
retry_delay: Optional[int] = 1, # <--
proxies: Optional[ProxySpec] = None, # <--
proxy: Optional[str] = None, # <--
retry_delay: Optional[int] = 1,
proxies: Optional[ProxySpec] = None,
proxy: Optional[str] = None,
proxy_auth: Optional[Tuple[str, str]] = None,
auth: Optional[Tuple[str, str]] = None,
verify: Optional[bool] = True, # <--
verify: Optional[bool] = True,
cert: Optional[Union[str, Tuple[str, str]]] = None,
impersonate: Optional[BrowserTypeLiteral] = "chrome136", # <--
impersonate: Optional[BrowserTypeLiteral] = DEFAULT_CHROME,
stealthy_headers: Optional[bool] = True,
http3: Optional[bool] = False,
**kwargs,
) -> Response:
"""
@@ -879,8 +920,9 @@ class AsyncFetcherClient:
:param auth: HTTP basic auth tuple of (username, password). Only basic auth is supported.
:param verify: Whether to verify HTTPS certificates. Defaults to True.
:param cert: Tuple of (cert, key) filenames for the client certificate.
:param impersonate: Browser version to impersonate. Defaults to "chrome136".
:param stealthy_headers: If enabled for this request (default), it creates and adds real browser headers. It also referer header as if it is from a Google search of URL's domain.
:param impersonate: Browser version to impersonate. Automatically defaults to the latest available Chrome version.
:param http3: Whether to use HTTP3. Defaults to False. It might be problematic if used it with `impersonate`.
:param stealthy_headers: If enabled (default), it creates and adds real browser headers. It also sets the referer header as if this request came from a Google search of URL's domain.
:param kwargs: Additional keyword arguments to pass to the `curl_cffi.requests.AsyncSession().request()` method.
:return: An awaitable `Response` object.
"""
@@ -905,6 +947,7 @@ class AsyncFetcherClient:
"auth": auth,
"verify": verify,
"cert": cert,
"http3": http3,
**kwargs,
}
async with FetcherSession(stealthy_headers=stealthy_headers) as client:
scrapling/engines/toolbelt/fingerprints.py
+9 -9
View File
@@ -14,7 +14,7 @@ from scrapling.core.utils import lru_cache
@lru_cache(10, typed=True)
def generate_convincing_referer(url: str) -> str:
"""Takes the domain from the URL without the subdomain/suffix and make it look like you were searching google for this website
"""Takes the domain from the URL without the subdomain/suffix and make it look like you were searching Google for this website
>>> generate_convincing_referer('https://www.somewebsite.com/blah')
'https://www.google.com/search?q=somewebsite'
@@ -38,13 +38,13 @@ def get_os_name() -> Union[str, None]:
"Linux": "linux",
"Darwin": "macos",
"Windows": "windows",
# For the future? because why not
# For the future? because why not?
"iOS": "ios",
}.get(os_name)
def generate_suitable_fingerprint() -> Fingerprint:
"""Generates a browserforge's fingerprint that matches current OS, desktop device, and Chrome with version 128 at least.
"""Generates a browserforge's fingerprint that matches the current OS, desktop device, and Chrome with version 128 at least.
This function was originally created to test Browserforge's injector.
:return: `Fingerprint` object
@@ -59,11 +59,11 @@ def generate_suitable_fingerprint() -> Fingerprint:
def generate_headers(browser_mode: bool = False) -> Dict:
"""Generate real browser-like headers using browserforge's generator
:param browser_mode: If enabled, the headers created are used for playwright so it have to match everything
:param browser_mode: If enabled, the headers created are used for playwright, so it has to match everything
:return: A dictionary of the generated headers
"""
if browser_mode:
# In this mode we don't care about anything other than matching the OS and the browser type with the browser we are using
# In this mode we don't care about anything other than matching the OS and the browser type with the browser we are using,
# So we don't raise any inconsistency red flags while websites fingerprinting us
os_name = get_os_name()
return HeaderGenerator(
@@ -72,10 +72,10 @@ def generate_headers(browser_mode: bool = False) -> Dict:
device="desktop",
).generate()
else:
# Here it's used for normal requests that aren't done through browsers so we can take it lightly
# Here it's used for normal requests that aren't done through browsers
browsers = [
Browser(name="chrome", min_version=120),
Browser(name="firefox", min_version=120),
Browser(name="edge", min_version=120),
Browser(name="chrome", min_version=130),
Browser(name="firefox", min_version=130),
Browser(name="edge", min_version=130),
]
return HeaderGenerator(browser=browsers, device="desktop").generate()