From f9f34e6ee9c0a1c9aabe15c3a373996714ed16f4 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Wed, 24 Dec 2025 23:08:30 +0200 Subject: [PATCH] fix: improve the handling of useragent in chrome fetcher --- scrapling/engines/_browsers/_config_tools.py | 8 +++++++- scrapling/engines/toolbelt/fingerprints.py | 8 ++++++-- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/scrapling/engines/_browsers/_config_tools.py b/scrapling/engines/_browsers/_config_tools.py index eaf28d6..f1f4171 100644 --- a/scrapling/engines/_browsers/_config_tools.py +++ b/scrapling/engines/_browsers/_config_tools.py @@ -10,6 +10,7 @@ from scrapling.engines.toolbelt.navigation import js_bypass_path from scrapling.engines.toolbelt.fingerprints import generate_headers __default_useragent__ = generate_headers(browser_mode=True).get("User-Agent") +__default_chrome_useragent__ = generate_headers(browser_mode="chrome").get("User-Agent") @lru_cache(1) @@ -89,8 +90,13 @@ def _launch_kwargs( "ignore_default_args": HARMFUL_DEFAULT_ARGS, "channel": "chrome" if real_chrome else "chromium", "extra_http_headers": extra_headers or tuple(), - "user_agent": useragent or __default_useragent__, } + # The default useragent in the headful is always correct now in the current versions of Playwright + if useragent: + launch_kwargs["user_agent"] = useragent + elif not useragent and headless: + launch_kwargs["user_agent"] = __default_chrome_useragent__ if real_chrome else __default_useragent__ + if stealth: stealth_args = base_args + _set_flags(hide_canvas, disable_webgl) launch_kwargs.update( diff --git a/scrapling/engines/toolbelt/fingerprints.py b/scrapling/engines/toolbelt/fingerprints.py index 7b551d8..4e9263e 100644 --- a/scrapling/engines/toolbelt/fingerprints.py +++ b/scrapling/engines/toolbelt/fingerprints.py @@ -13,6 +13,9 @@ from scrapling.core._types import Dict, Literal, Tuple __OS_NAME__ = platform_system() OSName = Literal["linux", "macos", "windows"] +# Current versions hardcoded for now (Playwright doesn't allow to know the version of a browser without launching it) +chromium_version = 141 +chrome_version = 143 @lru_cache(10, typed=True) @@ -46,7 +49,7 @@ def get_os_name() -> OSName | Tuple: return SUPPORTED_OPERATING_SYSTEMS -def generate_headers(browser_mode: bool = False) -> Dict: +def generate_headers(browser_mode: bool | str = False) -> Dict: """Generate real browser-like headers using browserforge's generator :param browser_mode: If enabled, the headers created are used for playwright, so it has to match everything @@ -55,7 +58,8 @@ def generate_headers(browser_mode: bool = False) -> Dict: # In the browser mode, we don't care about anything other than matching the OS and the browser type with the browser we are using, # So we don't raise any inconsistency red flags while websites fingerprinting us os_name = get_os_name() - browsers = [Browser(name="chrome", min_version=141, max_version=141)] + ver = chrome_version if browser_mode and browser_mode == "chrome" else chromium_version + browsers = [Browser(name="chrome", min_version=ver, max_version=ver)] if not browser_mode: os_name = ("windows", "macos", "linux") browsers.extend(