From 6eb580a6d0bb8bbdbf29fe854b2fde24341e67cf Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Sun, 31 Aug 2025 02:56:34 +0300 Subject: [PATCH] feat(DynamicSession): Improve normal fetches speed --- scrapling/engines/__init__.py | 2 +- scrapling/engines/_browsers/_config_tools.py | 17 ++++++++++----- scrapling/engines/constants.py | 23 +++++++++++++------- 3 files changed, 27 insertions(+), 15 deletions(-) diff --git a/scrapling/engines/__init__.py b/scrapling/engines/__init__.py index 7d29a16..477bb21 100644 --- a/scrapling/engines/__init__.py +++ b/scrapling/engines/__init__.py @@ -1,4 +1,4 @@ -from .constants import DEFAULT_DISABLED_RESOURCES, DEFAULT_STEALTH_FLAGS +from .constants import DEFAULT_DISABLED_RESOURCES, DEFAULT_STEALTH_FLAGS, DEFAULT_FLAGS from .static import FetcherSession, FetcherClient, AsyncFetcherClient from ._browsers import ( DynamicSession, diff --git a/scrapling/engines/_browsers/_config_tools.py b/scrapling/engines/_browsers/_config_tools.py index 46d2ad6..7f60efc 100644 --- a/scrapling/engines/_browsers/_config_tools.py +++ b/scrapling/engines/_browsers/_config_tools.py @@ -1,7 +1,11 @@ from functools import lru_cache from scrapling.core._types import Tuple -from scrapling.engines.constants import DEFAULT_STEALTH_FLAGS, HARMFUL_DEFAULT_ARGS +from scrapling.engines.constants import ( + DEFAULT_STEALTH_FLAGS, + HARMFUL_DEFAULT_ARGS, + DEFAULT_FLAGS, +) from scrapling.engines.toolbelt import js_bypass_path, generate_headers __default_useragent__ = generate_headers(browser_mode=True).get("User-Agent") @@ -69,20 +73,21 @@ def _launch_kwargs( ) -> Tuple: """Creates the arguments we will use while launching playwright's browser""" launch_kwargs = { + "locale": locale, "headless": headless, + "args": DEFAULT_FLAGS, + "color_scheme": "dark", # Bypasses the 'prefersLightColor' check in creepjs + "proxy": proxy or tuple(), + "device_scale_factor": 2, "ignore_default_args": HARMFUL_DEFAULT_ARGS, "channel": "chrome" if real_chrome else "chromium", - "proxy": proxy or tuple(), - "locale": locale, - "color_scheme": "dark", # Bypasses the 'prefersLightColor' check in creepjs - "device_scale_factor": 2, "extra_http_headers": extra_headers or tuple(), "user_agent": useragent or __default_useragent__, } if stealth: launch_kwargs.update( { - "args": _set_flags(hide_canvas, disable_webgl), + "args": DEFAULT_FLAGS + _set_flags(hide_canvas, disable_webgl), "chromium_sandbox": True, "is_mobile": False, "has_touch": False, diff --git a/scrapling/engines/constants.py b/scrapling/engines/constants.py index 2a84a2d..c7bdb1b 100644 --- a/scrapling/engines/constants.py +++ b/scrapling/engines/constants.py @@ -21,22 +21,32 @@ HARMFUL_DEFAULT_ARGS = ( # '--disable-extensions', ) +DEFAULT_FLAGS = ( + # Speed up chromium browsers by default + "--no-pings", + "--no-first-run", + "--disable-infobars", + "--disable-breakpad", + "--no-service-autorun", + "--homepage=about:blank", + "--password-store=basic", + "--no-default-browser-check", + "--disable-session-crashed-bubble", + "--disable-search-engine-choice-screen", +) + DEFAULT_STEALTH_FLAGS = ( # Explanation: https://peter.sh/experiments/chromium-command-line-switches/ # Generally this will make the browser faster and less detectable - "--no-pings", "--incognito", "--test-type", "--lang=en-US", "--mute-audio", - "--no-first-run", "--disable-sync", "--hide-scrollbars", "--disable-logging", "--start-maximized", # For headless check bypass "--enable-async-dns", - "--disable-breakpad", - "--disable-infobars", "--accept-lang=en-US", "--use-mock-keychain", "--disable-translate", @@ -48,7 +58,6 @@ DEFAULT_STEALTH_FLAGS = ( "--enable-tcp-fast-open", "--enable-web-bluetooth", "--disable-hang-monitor", - "--password-store=basic", "--disable-cloud-import", "--disable-default-apps", "--disable-print-preview", @@ -62,17 +71,15 @@ DEFAULT_STEALTH_FLAGS = ( "--disable-prompt-on-repost", "--force-color-profile=srgb", "--font-render-hinting=none", - "--no-default-browser-check", "--aggressive-cache-discard", "--disable-component-update", "--disable-cookie-encryption", "--disable-domain-reliability", "--disable-threaded-animation", "--disable-threaded-scrolling", - # '--disable-reading-from-canvas', # For Firefox + # '--disable-reading-from-canvas', # For Firefox "--enable-simple-cache-backend", "--disable-background-networking", - "--disable-session-crashed-bubble", "--enable-surface-synchronization", "--disable-image-animation-resync", "--disable-renderer-backgrounding",