feat(fetchers): Adding the foundation of the new browser-based fetchers logic

2025-06-19 04:06:52 +03:00
parent 23b883f05a
commit 145b710960
7 changed files with 906 additions and 0 deletions
@@ -0,0 +1,99 @@
+from functools import lru_cache
+
+from scrapling.core._types import Tuple
+from scrapling.engines.constants import DEFAULT_STEALTH_FLAGS, HARMFUL_DEFAULT_ARGS
+from scrapling.engines.toolbelt import js_bypass_path, generate_headers
+
+__default_useragent__ = generate_headers(browser_mode=True).get("User-Agent")
+
+
+@lru_cache(1)
+def _compiled_stealth_scripts():
+    """Pre-read and compile stealth scripts"""
+    # Basic bypasses nothing fancy as I'm still working on it
+    # But with adding these bypasses to the above config, it bypasses many online tests like
+    # https://bot.sannysoft.com/
+    # https://kaliiiiiiiiii.github.io/brotector/
+    # https://pixelscan.net/
+    # https://iphey.com/
+    # https://www.browserscan.net/bot-detection <== this one also checks for the CDP runtime fingerprint
+    # https://arh.antoinevastel.com/bots/areyouheadless/
+    # https://prescience-data.github.io/execution-monitor.html
+    stealth_scripts_paths = tuple(
+        js_bypass_path(script)
+        for script in (
+            # Order is important
+            "webdriver_fully.js",
+            "window_chrome.js",
+            "navigator_plugins.js",
+            "pdf_viewer.js",
+            "notification_permission.js",
+            "screen_props.js",
+            "playwright_fingerprint.js",
+        )
+    )
+    scripts = []
+    for script_path in stealth_scripts_paths:
+        with open(script_path, "r") as f:
+            scripts.append(f.read())
+    return tuple(scripts)
+
+
+@lru_cache(2, typed=True)
+def _set_flags(hide_canvas, disable_webgl):
+    """Returns the flags that will be used while launching the browser if stealth mode is enabled"""
+    flags = DEFAULT_STEALTH_FLAGS
+    if hide_canvas:
+        flags += ("--fingerprinting-canvas-image-data-noise",)
+    if disable_webgl:
+        flags += (
+            "--disable-webgl",
+            "--disable-webgl-image-chromium",
+            "--disable-webgl2",
+        )
+
+    return flags
+
+
+@lru_cache(2, typed=True)
+def _launch_kwargs(headless, real_chrome, stealth, hide_canvas, disable_webgl) -> Tuple:
+    """Creates the arguments we will use while launching playwright's browser"""
+    launch_kwargs = {
+        "headless": headless,
+        "ignore_default_args": HARMFUL_DEFAULT_ARGS,
+        "channel": "chrome" if real_chrome else "chromium",
+    }
+    if stealth:
+        launch_kwargs.update(
+            {"args": _set_flags(hide_canvas, disable_webgl), "chromium_sandbox": True}
+        )
+
+    return tuple(launch_kwargs.items())
+
+
+@lru_cache(2, typed=True)
+def _context_kwargs(proxy, locale, extra_headers, useragent, stealth) -> Tuple:
+    """Creates the arguments for the browser context"""
+    context_kwargs = {
+        "proxy": proxy or tuple(),
+        "locale": locale,
+        "color_scheme": "dark",  # Bypasses the 'prefersLightColor' check in creepjs
+        "device_scale_factor": 2,
+        "extra_http_headers": extra_headers or tuple(),
+        "user_agent": useragent or __default_useragent__,
+    }
+    if stealth:
+        context_kwargs.update(
+            {
+                "is_mobile": False,
+                "has_touch": False,
+                # I'm thinking about disabling it to rest from all Service Workers' headache, but let's keep it as it is for now
+                "service_workers": "allow",
+                "ignore_https_errors": True,
+                "screen": {"width": 1920, "height": 1080},
+                "viewport": {"width": 1920, "height": 1080},
+                "permissions": ["geolocation", "notifications"],
+            }
+        )
+
+    return tuple(context_kwargs.items())