From 3565f9d5009bc5e963f16baefa883e23b484c739 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Sun, 16 Nov 2025 16:33:58 +0200 Subject: [PATCH] feat(fetcher): Make `impersonate` able to randomize fingerprint --- scrapling/cli.py | 28 +++++++++++++++++---- scrapling/core/ai.py | 8 +++--- scrapling/engines/static.py | 50 ++++++++++++++++++++++++++----------- 3 files changed, 61 insertions(+), 25 deletions(-) diff --git a/scrapling/cli.py b/scrapling/cli.py index 25fdce3..22f2dd1 100644 --- a/scrapling/cli.py +++ b/scrapling/cli.py @@ -2,6 +2,8 @@ from pathlib import Path from subprocess import check_output from sys import executable as python_executable +from curl_cffi.requests import impersonate + from scrapling.core.utils import log from scrapling.engines.toolbelt.custom import Response from scrapling.core.utils._shell import _CookieParser, _ParseHeaders @@ -84,7 +86,7 @@ def __BuildRequest(headers: List[str], cookies: str, params: str, json: Optional # Parse parameters parsed_headers, parsed_cookies, parsed_params, parsed_json = __ParseExtractArguments(headers, cookies, params, json) # Build request arguments - request_kwargs = { + request_kwargs: Dict[str, Any] = { "headers": parsed_headers if parsed_headers else None, "cookies": parsed_cookies if parsed_cookies else None, } @@ -95,6 +97,10 @@ def __BuildRequest(headers: List[str], cookies: str, params: str, json: Optional if "proxy" in kwargs: request_kwargs["proxy"] = kwargs.pop("proxy") + # Parse impersonate parameter if it contains commas (for random selection) + if "impersonate" in kwargs and "," in (kwargs.get("impersonate") or ""): + kwargs["impersonate"] = [browser.strip() for browser in kwargs["impersonate"].split(",")] + return {**request_kwargs, **kwargs} @@ -225,7 +231,10 @@ def extract(): default=True, help="Whether to verify SSL certificates (default: True)", ) -@option("--impersonate", help="Browser to impersonate (e.g., chrome, firefox).") +@option( + "--impersonate", + help="Browser to impersonate. Can be a single browser (e.g., chrome) or comma-separated list for random selection (e.g., chrome,firefox,safari).", +) @option( "--stealthy-headers/--no-stealthy-headers", default=True, @@ -318,7 +327,10 @@ def get( default=True, help="Whether to verify SSL certificates (default: True)", ) -@option("--impersonate", help="Browser to impersonate (e.g., chrome, firefox).") +@option( + "--impersonate", + help="Browser to impersonate. Can be a single browser (e.g., chrome) or comma-separated list for random selection (e.g., chrome,firefox,safari).", +) @option( "--stealthy-headers/--no-stealthy-headers", default=True, @@ -412,7 +424,10 @@ def post( default=True, help="Whether to verify SSL certificates (default: True)", ) -@option("--impersonate", help="Browser to impersonate (e.g., chrome, firefox).") +@option( + "--impersonate", + help="Browser to impersonate. Can be a single browser (e.g., chrome) or comma-separated list for random selection (e.g., chrome,firefox,safari).", +) @option( "--stealthy-headers/--no-stealthy-headers", default=True, @@ -504,7 +519,10 @@ def put( default=True, help="Whether to verify SSL certificates (default: True)", ) -@option("--impersonate", help="Browser to impersonate (e.g., chrome, firefox).") +@option( + "--impersonate", + help="Browser to impersonate. Can be a single browser (e.g., chrome) or comma-separated list for random selection (e.g., chrome,firefox,safari).", +) @option( "--stealthy-headers/--no-stealthy-headers", default=True, diff --git a/scrapling/core/ai.py b/scrapling/core/ai.py index d497e57..4d5277a 100644 --- a/scrapling/core/ai.py +++ b/scrapling/core/ai.py @@ -5,6 +5,7 @@ from pydantic import BaseModel, Field from scrapling.core.shell import Convertor from scrapling.engines.toolbelt.custom import Response as _ScraplingResponse +from scrapling.engines.static import ImpersonateType from scrapling.fetchers import ( Fetcher, FetcherSession, @@ -24,9 +25,6 @@ from scrapling.core._types import ( SelectorWaitStates, Generator, ) -from curl_cffi.requests import ( - BrowserTypeLiteral, -) class ResponseModel(BaseModel): @@ -46,7 +44,7 @@ class ScraplingMCPServer: @staticmethod def get( url: str, - impersonate: Optional[BrowserTypeLiteral] = "chrome", + impersonate: ImpersonateType = "chrome", extraction_type: extraction_types = "markdown", css_selector: Optional[str] = None, main_content_only: bool = True, @@ -124,7 +122,7 @@ class ScraplingMCPServer: @staticmethod async def bulk_get( urls: Tuple[str, ...], - impersonate: Optional[BrowserTypeLiteral] = "chrome", + impersonate: ImpersonateType = "chrome", extraction_type: extraction_types = "markdown", css_selector: Optional[str] = None, main_content_only: bool = True, diff --git a/scrapling/engines/static.py b/scrapling/engines/static.py index dd54168..6f68737 100644 --- a/scrapling/engines/static.py +++ b/scrapling/engines/static.py @@ -1,4 +1,5 @@ from abc import ABC +from random import choice from time import sleep as time_sleep from asyncio import sleep as asyncio_sleep @@ -31,12 +32,29 @@ from .toolbelt.fingerprints import generate_convincing_referer, generate_headers _UNSET: Any = object() _NO_SESSION: Any = object() +# Type alias for `impersonate` parameter - accepts a single browser or list of browsers +ImpersonateType = BrowserTypeLiteral | List[BrowserTypeLiteral] | None + + +def _select_random_browser(impersonate: ImpersonateType) -> Optional[BrowserTypeLiteral]: + """ + Handle browser selection logic for the ` impersonate ` parameter. + + If impersonate is a list, randomly select one browser from it. + If it's a string or None, return as is. + """ + if isinstance(impersonate, list): + if not impersonate: + return None + return choice(impersonate) + return impersonate + class _ConfigurationLogic(ABC): # Core Logic Handler (Internal Engine) def __init__( self, - impersonate: Optional[BrowserTypeLiteral] = "chrome", + impersonate: ImpersonateType = "chrome", http3: Optional[bool] = False, stealthy_headers: Optional[bool] = True, proxies: Optional[Dict[str, str]] = None, @@ -76,7 +94,9 @@ class _ConfigurationLogic(ABC): def _merge_request_args(self, **method_kwargs) -> Dict[str, Any]: """Merge request-specific arguments with default session arguments.""" url = method_kwargs.pop("url") - impersonate = self._get_with_precedence(method_kwargs.pop("impersonate"), self._default_impersonate) + impersonate = _select_random_browser( + self._get_with_precedence(method_kwargs.pop("impersonate"), self._default_impersonate) + ) http3_enabled = self._get_with_precedence(method_kwargs.pop("http3"), self._default_http3) final_args = { "url": url, @@ -146,7 +166,7 @@ class _ConfigurationLogic(ABC): class _SyncSessionLogic(_ConfigurationLogic): def __init__( self, - impersonate: Optional[BrowserTypeLiteral] = "chrome", + impersonate: ImpersonateType = "chrome", http3: Optional[bool] = False, stealthy_headers: Optional[bool] = True, proxies: Optional[Dict[str, str]] = None, @@ -261,7 +281,7 @@ class _SyncSessionLogic(_ConfigurationLogic): auth: Optional[Tuple[str, str]] = None, verify: Optional[bool] = _UNSET, cert: Optional[str | Tuple[str, str]] = _UNSET, - impersonate: Optional[BrowserTypeLiteral] = _UNSET, + impersonate: ImpersonateType = _UNSET, http3: Optional[bool] = _UNSET, stealthy_headers: Optional[bool] = _UNSET, **kwargs, @@ -334,7 +354,7 @@ class _SyncSessionLogic(_ConfigurationLogic): auth: Optional[Tuple[str, str]] = None, verify: Optional[bool] = _UNSET, cert: Optional[str | Tuple[str, str]] = _UNSET, - impersonate: Optional[BrowserTypeLiteral] = _UNSET, + impersonate: ImpersonateType = _UNSET, http3: Optional[bool] = _UNSET, stealthy_headers: Optional[bool] = _UNSET, **kwargs, @@ -411,7 +431,7 @@ class _SyncSessionLogic(_ConfigurationLogic): auth: Optional[Tuple[str, str]] = None, verify: Optional[bool] = _UNSET, cert: Optional[str | Tuple[str, str]] = _UNSET, - impersonate: Optional[BrowserTypeLiteral] = _UNSET, + impersonate: ImpersonateType = _UNSET, http3: Optional[bool] = _UNSET, stealthy_headers: Optional[bool] = _UNSET, **kwargs, @@ -488,7 +508,7 @@ class _SyncSessionLogic(_ConfigurationLogic): auth: Optional[Tuple[str, str]] = None, verify: Optional[bool] = _UNSET, cert: Optional[str | Tuple[str, str]] = _UNSET, - impersonate: Optional[BrowserTypeLiteral] = _UNSET, + impersonate: ImpersonateType = _UNSET, http3: Optional[bool] = _UNSET, stealthy_headers: Optional[bool] = _UNSET, **kwargs, @@ -552,7 +572,7 @@ class _SyncSessionLogic(_ConfigurationLogic): class _ASyncSessionLogic(_ConfigurationLogic): def __init__( self, - impersonate: Optional[BrowserTypeLiteral] = "chrome", + impersonate: ImpersonateType = "chrome", http3: Optional[bool] = False, stealthy_headers: Optional[bool] = True, proxies: Optional[Dict[str, str]] = None, @@ -669,7 +689,7 @@ class _ASyncSessionLogic(_ConfigurationLogic): auth: Optional[Tuple[str, str]] = None, verify: Optional[bool] = _UNSET, cert: Optional[str | Tuple[str, str]] = _UNSET, - impersonate: Optional[BrowserTypeLiteral] = _UNSET, + impersonate: ImpersonateType = _UNSET, http3: Optional[bool] = _UNSET, stealthy_headers: Optional[bool] = _UNSET, **kwargs, @@ -742,7 +762,7 @@ class _ASyncSessionLogic(_ConfigurationLogic): auth: Optional[Tuple[str, str]] = None, verify: Optional[bool] = _UNSET, cert: Optional[str | Tuple[str, str]] = _UNSET, - impersonate: Optional[BrowserTypeLiteral] = _UNSET, + impersonate: ImpersonateType = _UNSET, http3: Optional[bool] = _UNSET, stealthy_headers: Optional[bool] = _UNSET, **kwargs, @@ -819,7 +839,7 @@ class _ASyncSessionLogic(_ConfigurationLogic): auth: Optional[Tuple[str, str]] = None, verify: Optional[bool] = _UNSET, cert: Optional[str | Tuple[str, str]] = _UNSET, - impersonate: Optional[BrowserTypeLiteral] = _UNSET, + impersonate: ImpersonateType = _UNSET, http3: Optional[bool] = _UNSET, stealthy_headers: Optional[bool] = _UNSET, **kwargs, @@ -896,7 +916,7 @@ class _ASyncSessionLogic(_ConfigurationLogic): auth: Optional[Tuple[str, str]] = None, verify: Optional[bool] = _UNSET, cert: Optional[str | Tuple[str, str]] = _UNSET, - impersonate: Optional[BrowserTypeLiteral] = _UNSET, + impersonate: ImpersonateType = _UNSET, http3: Optional[bool] = _UNSET, stealthy_headers: Optional[bool] = _UNSET, **kwargs, @@ -970,7 +990,7 @@ class FetcherSession: def __init__( self, - impersonate: Optional[BrowserTypeLiteral] = "chrome", + impersonate: ImpersonateType = "chrome", http3: Optional[bool] = False, stealthy_headers: Optional[bool] = True, proxies: Optional[Dict[str, str]] = None, @@ -987,7 +1007,7 @@ class FetcherSession: selector_config: Optional[Dict] = None, ): """ - :param impersonate: Browser version to impersonate. Automatically defaults to the latest available Chrome version. + :param impersonate: Browser version to impersonate. Can be a single browser string or a list of browser strings for random selection. (Default: latest available Chrome version) :param http3: Whether to use HTTP3. Defaults to False. It might be problematic if used it with `impersonate`. :param stealthy_headers: If enabled (default), it creates and adds real browser headers. It also sets the referer header as if this request came from a Google search of URL's domain. :param proxies: Dict of proxies to use. Format: {"http": proxy_url, "https": proxy_url}. @@ -1004,7 +1024,7 @@ class FetcherSession: :param cert: Tuple of (cert, key) filenames for the client certificate. :param selector_config: Arguments passed when creating the final Selector class. """ - self._default_impersonate: Optional[BrowserTypeLiteral] = impersonate + self._default_impersonate: ImpersonateType = impersonate self._stealth = stealthy_headers self._default_proxies = proxies or {} self._default_proxy = proxy or None