freedak/Scrapling
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(fetcher): Make impersonate able to randomize fingerprint

Browse Source
This commit is contained in:
Karim shoair
2025-11-16 16:33:58 +02:00
parent aa245159d0
commit 3565f9d500
3 changed files with 61 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scrapling/cli.py
+23 -5
View File
@@ -2,6 +2,8 @@ from pathlib import Path
from subprocess import check_output
from sys import executable as python_executable
from curl_cffi.requests import impersonate
from scrapling.core.utils import log
from scrapling.engines.toolbelt.custom import Response
from scrapling.core.utils._shell import _CookieParser, _ParseHeaders
@@ -84,7 +86,7 @@ def __BuildRequest(headers: List[str], cookies: str, params: str, json: Optional
# Parse parameters
parsed_headers, parsed_cookies, parsed_params, parsed_json = __ParseExtractArguments(headers, cookies, params, json)
# Build request arguments
request_kwargs = {
request_kwargs: Dict[str, Any] = {
"headers": parsed_headers if parsed_headers else None,
"cookies": parsed_cookies if parsed_cookies else None,
}
@@ -95,6 +97,10 @@ def __BuildRequest(headers: List[str], cookies: str, params: str, json: Optional
if "proxy" in kwargs:
request_kwargs["proxy"] = kwargs.pop("proxy")
# Parse impersonate parameter if it contains commas (for random selection)
if "impersonate" in kwargs and "," in (kwargs.get("impersonate") or ""):
kwargs["impersonate"] = [browser.strip() for browser in kwargs["impersonate"].split(",")]
return {**request_kwargs, **kwargs}
@@ -225,7 +231,10 @@ def extract():
default=True,
help="Whether to verify SSL certificates (default: True)",
)
@option("--impersonate", help="Browser to impersonate (e.g., chrome, firefox).")
@option(
"--impersonate",
help="Browser to impersonate. Can be a single browser (e.g., chrome) or comma-separated list for random selection (e.g., chrome,firefox,safari).",
)
@option(
"--stealthy-headers/--no-stealthy-headers",
default=True,
@@ -318,7 +327,10 @@ def get(
default=True,
help="Whether to verify SSL certificates (default: True)",
)
@option("--impersonate", help="Browser to impersonate (e.g., chrome, firefox).")
@option(
"--impersonate",
help="Browser to impersonate. Can be a single browser (e.g., chrome) or comma-separated list for random selection (e.g., chrome,firefox,safari).",
)
@option(
"--stealthy-headers/--no-stealthy-headers",
default=True,
@@ -412,7 +424,10 @@ def post(
default=True,
help="Whether to verify SSL certificates (default: True)",
)
@option("--impersonate", help="Browser to impersonate (e.g., chrome, firefox).")
@option(
"--impersonate",
help="Browser to impersonate. Can be a single browser (e.g., chrome) or comma-separated list for random selection (e.g., chrome,firefox,safari).",
)
@option(
"--stealthy-headers/--no-stealthy-headers",
default=True,
@@ -504,7 +519,10 @@ def put(
default=True,
help="Whether to verify SSL certificates (default: True)",
)
@option("--impersonate", help="Browser to impersonate (e.g., chrome, firefox).")
@option(
"--impersonate",
help="Browser to impersonate. Can be a single browser (e.g., chrome) or comma-separated list for random selection (e.g., chrome,firefox,safari).",
)
@option(
"--stealthy-headers/--no-stealthy-headers",
default=True,
scrapling/core/ai.py
+3 -5
View File
@@ -5,6 +5,7 @@ from pydantic import BaseModel, Field
from scrapling.core.shell import Convertor
from scrapling.engines.toolbelt.custom import Response as _ScraplingResponse
from scrapling.engines.static import ImpersonateType
from scrapling.fetchers import (
Fetcher,
FetcherSession,
@@ -24,9 +25,6 @@ from scrapling.core._types import (
SelectorWaitStates,
Generator,
)
from curl_cffi.requests import (
BrowserTypeLiteral,
)
class ResponseModel(BaseModel):
@@ -46,7 +44,7 @@ class ScraplingMCPServer:
@staticmethod
def get(
url: str,
impersonate: Optional[BrowserTypeLiteral] = "chrome",
impersonate: ImpersonateType = "chrome",
extraction_type: extraction_types = "markdown",
css_selector: Optional[str] = None,
main_content_only: bool = True,
@@ -124,7 +122,7 @@ class ScraplingMCPServer:
@staticmethod
async def bulk_get(
urls: Tuple[str, ...],
impersonate: Optional[BrowserTypeLiteral] = "chrome",
impersonate: ImpersonateType = "chrome",
extraction_type: extraction_types = "markdown",
css_selector: Optional[str] = None,
main_content_only: bool = True,
scrapling/engines/static.py
+35 -15
View File
@@ -1,4 +1,5 @@
from abc import ABC
from random import choice
from time import sleep as time_sleep
from asyncio import sleep as asyncio_sleep
@@ -31,12 +32,29 @@ from .toolbelt.fingerprints import generate_convincing_referer, generate_headers
_UNSET: Any = object()
_NO_SESSION: Any = object()
# Type alias for `impersonate` parameter - accepts a single browser or list of browsers
ImpersonateType = BrowserTypeLiteral | List[BrowserTypeLiteral] | None
def _select_random_browser(impersonate: ImpersonateType) -> Optional[BrowserTypeLiteral]:
"""
Handle browser selection logic for the ` impersonate ` parameter.
If impersonate is a list, randomly select one browser from it.
If it's a string or None, return as is.
"""
if isinstance(impersonate, list):
if not impersonate:
return None
return choice(impersonate)
return impersonate
class _ConfigurationLogic(ABC):
# Core Logic Handler (Internal Engine)
def __init__(
self,
impersonate: Optional[BrowserTypeLiteral] = "chrome",
impersonate: ImpersonateType = "chrome",
http3: Optional[bool] = False,
stealthy_headers: Optional[bool] = True,
proxies: Optional[Dict[str, str]] = None,
@@ -76,7 +94,9 @@ class _ConfigurationLogic(ABC):
def _merge_request_args(self, **method_kwargs) -> Dict[str, Any]:
"""Merge request-specific arguments with default session arguments."""
url = method_kwargs.pop("url")
impersonate = self._get_with_precedence(method_kwargs.pop("impersonate"), self._default_impersonate)
impersonate = _select_random_browser(
self._get_with_precedence(method_kwargs.pop("impersonate"), self._default_impersonate)
)
http3_enabled = self._get_with_precedence(method_kwargs.pop("http3"), self._default_http3)
final_args = {
"url": url,
@@ -146,7 +166,7 @@ class _ConfigurationLogic(ABC):
class _SyncSessionLogic(_ConfigurationLogic):
def __init__(
self,
impersonate: Optional[BrowserTypeLiteral] = "chrome",
impersonate: ImpersonateType = "chrome",
http3: Optional[bool] = False,
stealthy_headers: Optional[bool] = True,
proxies: Optional[Dict[str, str]] = None,
@@ -261,7 +281,7 @@ class _SyncSessionLogic(_ConfigurationLogic):
auth: Optional[Tuple[str, str]] = None,
verify: Optional[bool] = _UNSET,
cert: Optional[str | Tuple[str, str]] = _UNSET,
impersonate: Optional[BrowserTypeLiteral] = _UNSET,
impersonate: ImpersonateType = _UNSET,
http3: Optional[bool] = _UNSET,
stealthy_headers: Optional[bool] = _UNSET,
**kwargs,
@@ -334,7 +354,7 @@ class _SyncSessionLogic(_ConfigurationLogic):
auth: Optional[Tuple[str, str]] = None,
verify: Optional[bool] = _UNSET,
cert: Optional[str | Tuple[str, str]] = _UNSET,
impersonate: Optional[BrowserTypeLiteral] = _UNSET,
impersonate: ImpersonateType = _UNSET,
http3: Optional[bool] = _UNSET,
stealthy_headers: Optional[bool] = _UNSET,
**kwargs,
@@ -411,7 +431,7 @@ class _SyncSessionLogic(_ConfigurationLogic):
auth: Optional[Tuple[str, str]] = None,
verify: Optional[bool] = _UNSET,
cert: Optional[str | Tuple[str, str]] = _UNSET,
impersonate: Optional[BrowserTypeLiteral] = _UNSET,
impersonate: ImpersonateType = _UNSET,
http3: Optional[bool] = _UNSET,
stealthy_headers: Optional[bool] = _UNSET,
**kwargs,
@@ -488,7 +508,7 @@ class _SyncSessionLogic(_ConfigurationLogic):
auth: Optional[Tuple[str, str]] = None,
verify: Optional[bool] = _UNSET,
cert: Optional[str | Tuple[str, str]] = _UNSET,
impersonate: Optional[BrowserTypeLiteral] = _UNSET,
impersonate: ImpersonateType = _UNSET,
http3: Optional[bool] = _UNSET,
stealthy_headers: Optional[bool] = _UNSET,
**kwargs,
@@ -552,7 +572,7 @@ class _SyncSessionLogic(_ConfigurationLogic):
class _ASyncSessionLogic(_ConfigurationLogic):
def __init__(
self,
impersonate: Optional[BrowserTypeLiteral] = "chrome",
impersonate: ImpersonateType = "chrome",
http3: Optional[bool] = False,
stealthy_headers: Optional[bool] = True,
proxies: Optional[Dict[str, str]] = None,
@@ -669,7 +689,7 @@ class _ASyncSessionLogic(_ConfigurationLogic):
auth: Optional[Tuple[str, str]] = None,
verify: Optional[bool] = _UNSET,
cert: Optional[str | Tuple[str, str]] = _UNSET,
impersonate: Optional[BrowserTypeLiteral] = _UNSET,
impersonate: ImpersonateType = _UNSET,
http3: Optional[bool] = _UNSET,
stealthy_headers: Optional[bool] = _UNSET,
**kwargs,
@@ -742,7 +762,7 @@ class _ASyncSessionLogic(_ConfigurationLogic):
auth: Optional[Tuple[str, str]] = None,
verify: Optional[bool] = _UNSET,
cert: Optional[str | Tuple[str, str]] = _UNSET,
impersonate: Optional[BrowserTypeLiteral] = _UNSET,
impersonate: ImpersonateType = _UNSET,
http3: Optional[bool] = _UNSET,
stealthy_headers: Optional[bool] = _UNSET,
**kwargs,
@@ -819,7 +839,7 @@ class _ASyncSessionLogic(_ConfigurationLogic):
auth: Optional[Tuple[str, str]] = None,
verify: Optional[bool] = _UNSET,
cert: Optional[str | Tuple[str, str]] = _UNSET,
impersonate: Optional[BrowserTypeLiteral] = _UNSET,
impersonate: ImpersonateType = _UNSET,
http3: Optional[bool] = _UNSET,
stealthy_headers: Optional[bool] = _UNSET,
**kwargs,
@@ -896,7 +916,7 @@ class _ASyncSessionLogic(_ConfigurationLogic):
auth: Optional[Tuple[str, str]] = None,
verify: Optional[bool] = _UNSET,
cert: Optional[str | Tuple[str, str]] = _UNSET,
impersonate: Optional[BrowserTypeLiteral] = _UNSET,
impersonate: ImpersonateType = _UNSET,
http3: Optional[bool] = _UNSET,
stealthy_headers: Optional[bool] = _UNSET,
**kwargs,
@@ -970,7 +990,7 @@ class FetcherSession:
def __init__(
self,
impersonate: Optional[BrowserTypeLiteral] = "chrome",
impersonate: ImpersonateType = "chrome",
http3: Optional[bool] = False,
stealthy_headers: Optional[bool] = True,
proxies: Optional[Dict[str, str]] = None,
@@ -987,7 +1007,7 @@ class FetcherSession:
selector_config: Optional[Dict] = None,
):
"""
:param impersonate: Browser version to impersonate. Automatically defaults to the latest available Chrome version.
:param impersonate: Browser version to impersonate. Can be a single browser string or a list of browser strings for random selection. (Default: latest available Chrome version)
:param http3: Whether to use HTTP3. Defaults to False. It might be problematic if used it with `impersonate`.
:param stealthy_headers: If enabled (default), it creates and adds real browser headers. It also sets the referer header as if this request came from a Google search of URL's domain.
:param proxies: Dict of proxies to use. Format: {"http": proxy_url, "https": proxy_url}.
@@ -1004,7 +1024,7 @@ class FetcherSession:
:param cert: Tuple of (cert, key) filenames for the client certificate.
:param selector_config: Arguments passed when creating the final Selector class.
"""
self._default_impersonate: Optional[BrowserTypeLiteral] = impersonate
self._default_impersonate: ImpersonateType = impersonate
self._stealth = stealthy_headers
self._default_proxies = proxies or {}
self._default_proxy = proxy or None