refactor(fetchers)!: Replace Camoufox with patchright and many optimizations

- DynamicFetcher became 20% faster - StealthyFetcher became 99% faster - Scrapling size decreased - Code became ~400 lines shorter - Most importantly, scrapling is more stable and reliable now. - Less confusing for new users. - More...
2025-12-26 03:21:13 +02:00
parent f9f34e6ee9
commit cfc667f7dc
12 changed files with 469 additions and 707 deletions
@@ -58,88 +58,3 @@ def _set_flags(hide_canvas, disable_webgl):  # pragma: no cover
        )

    return flags
-
-
-@lru_cache(2, typed=True)
-def _launch_kwargs(
-    headless,
-    proxy: Tuple,
-    locale,
-    extra_headers,
-    useragent,
-    real_chrome,
-    stealth,
-    hide_canvas,
-    disable_webgl,
-    timezone_id,
-    extra_flags: Tuple,
-) -> Tuple:
-    """Creates the arguments we will use while launching playwright's browser"""
-    base_args = DEFAULT_FLAGS
-    if extra_flags:
-        base_args = base_args + extra_flags
-
-    launch_kwargs = {
-        "locale": locale,
-        "timezone_id": timezone_id or None,
-        "headless": headless,
-        "args": base_args,
-        "color_scheme": "dark",  # Bypasses the 'prefersLightColor' check in creepjs
-        "proxy": proxy or tuple(),
-        "device_scale_factor": 2,
-        "ignore_default_args": HARMFUL_DEFAULT_ARGS,
-        "channel": "chrome" if real_chrome else "chromium",
-        "extra_http_headers": extra_headers or tuple(),
-    }
-    # The default useragent in the headful is always correct now in the current versions of Playwright
-    if useragent:
-        launch_kwargs["user_agent"] = useragent
-    elif not useragent and headless:
-        launch_kwargs["user_agent"] = __default_chrome_useragent__ if real_chrome else __default_useragent__
-
-    if stealth:
-        stealth_args = base_args + _set_flags(hide_canvas, disable_webgl)
-        launch_kwargs.update(
-            {
-                "args": stealth_args,
-                "chromium_sandbox": True,
-                "is_mobile": False,
-                "has_touch": False,
-                # I'm thinking about disabling it to rest from all Service Workers' headache, but let's keep it as it is for now
-                "service_workers": "allow",
-                "ignore_https_errors": True,
-                "screen": {"width": 1920, "height": 1080},
-                "viewport": {"width": 1920, "height": 1080},
-                "permissions": ["geolocation", "notifications"],
-            }
-        )
-
-    return tuple(launch_kwargs.items())
-
-
-@lru_cache(2, typed=True)
-def _context_kwargs(proxy, locale, extra_headers, useragent, stealth) -> Tuple:
-    """Creates the arguments for the browser context"""
-    context_kwargs = {
-        "proxy": proxy or tuple(),
-        "locale": locale,
-        "color_scheme": "dark",  # Bypasses the 'prefersLightColor' check in creepjs
-        "device_scale_factor": 2,
-        "extra_http_headers": extra_headers or tuple(),
-        "user_agent": useragent or __default_useragent__,
-    }
-    if stealth:
-        context_kwargs.update(
-            {
-                "is_mobile": False,
-                "has_touch": False,
-                # I'm thinking about disabling it to rest from all Service Workers' headache, but let's keep it as it is for now
-                "service_workers": "allow",
-                "ignore_https_errors": True,
-                "screen": {"width": 1920, "height": 1080},
-                "viewport": {"width": 1920, "height": 1080},
-                "permissions": ["geolocation", "notifications"],
-            }
-        )
-
-    return tuple(context_kwargs.items())