From 13b9a4668cc2983466195f1e0d2689c62b9060dc Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Sun, 26 Oct 2025 16:53:27 +0300 Subject: [PATCH] feat(DynamicSession): New option to add extra browser flags --- docs/fetching/dynamic.md | 1 + scrapling/engines/_browsers/_base.py | 2 ++ scrapling/engines/_browsers/_config_tools.py | 10 ++++++++-- scrapling/engines/_browsers/_controllers.py | 6 ++++++ scrapling/engines/_browsers/_validators.py | 3 +++ scrapling/fetchers/chrome.py | 6 ++++++ 6 files changed, 26 insertions(+), 2 deletions(-) diff --git a/docs/fetching/dynamic.md b/docs/fetching/dynamic.md index 18c39d7..1054d54 100644 --- a/docs/fetching/dynamic.md +++ b/docs/fetching/dynamic.md @@ -89,6 +89,7 @@ Scrapling provides many options with this fetcher and its session classes. To ma | locale | Set the locale for the browser if wanted. The default value is `en-US`. | ✔️ | | cdp_url | Instead of launching a new browser instance, connect to this CDP URL to control real browsers through CDP. | ✔️ | | user_data_dir | Path to a User Data Directory, which stores browser session data like cookies and local storage. The default is to create a temporary directory. **Only Works with sessions** | ✔️ | +| extra_flags | A list of additional browser flags to pass to the browser on launch. | ✔️ | | additional_args | Additional arguments to be passed to Playwright's context as additional settings, and they take higher priority than Scrapling's settings. | ✔️ | | selector_config | A dictionary of custom parsing arguments to be used when creating the final `Selector`/`Response` class. | ✔️ | diff --git a/scrapling/engines/_browsers/_base.py b/scrapling/engines/_browsers/_base.py index 4157494..f670e4c 100644 --- a/scrapling/engines/_browsers/_base.py +++ b/scrapling/engines/_browsers/_base.py @@ -276,6 +276,7 @@ class DynamicSessionMixin: self.wait_selector = config.wait_selector self.init_script = config.init_script self.wait_selector_state = config.wait_selector_state + self.extra_flags = config.extra_flags self.selector_config = config.selector_config self.additional_args = config.additional_args self.page_action = config.page_action @@ -300,6 +301,7 @@ class DynamicSessionMixin: self.stealth, self.hide_canvas, self.disable_webgl, + tuple(self.extra_flags) if self.extra_flags else tuple(), ) ) self.launch_options["extra_http_headers"] = dict(self.launch_options["extra_http_headers"]) diff --git a/scrapling/engines/_browsers/_config_tools.py b/scrapling/engines/_browsers/_config_tools.py index 322d7ad..57e2f17 100644 --- a/scrapling/engines/_browsers/_config_tools.py +++ b/scrapling/engines/_browsers/_config_tools.py @@ -70,12 +70,17 @@ def _launch_kwargs( stealth, hide_canvas, disable_webgl, + extra_flags: Tuple, ) -> Tuple: """Creates the arguments we will use while launching playwright's browser""" + base_args = DEFAULT_FLAGS + if extra_flags: + base_args = base_args + extra_flags + launch_kwargs = { "locale": locale, "headless": headless, - "args": DEFAULT_FLAGS, + "args": base_args, "color_scheme": "dark", # Bypasses the 'prefersLightColor' check in creepjs "proxy": proxy or tuple(), "device_scale_factor": 2, @@ -85,9 +90,10 @@ def _launch_kwargs( "user_agent": useragent or __default_useragent__, } if stealth: + stealth_args = base_args + _set_flags(hide_canvas, disable_webgl) launch_kwargs.update( { - "args": DEFAULT_FLAGS + _set_flags(hide_canvas, disable_webgl), + "args": stealth_args, "chromium_sandbox": True, "is_mobile": False, "has_touch": False, diff --git a/scrapling/engines/_browsers/_controllers.py b/scrapling/engines/_browsers/_controllers.py index 155789a..d8f841b 100644 --- a/scrapling/engines/_browsers/_controllers.py +++ b/scrapling/engines/_browsers/_controllers.py @@ -95,6 +95,7 @@ class DynamicSession(DynamicSessionMixin, SyncSession): load_dom: bool = True, wait_selector_state: SelectorWaitStates = "attached", user_data_dir: str = "", + extra_flags: Optional[List[str]] = None, selector_config: Optional[Dict] = None, additional_args: Optional[Dict] = None, ): @@ -124,6 +125,7 @@ class DynamicSession(DynamicSessionMixin, SyncSession): :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only. :param user_data_dir: Path to a User Data Directory, which stores browser session data like cookies and local storage. The default is to create a temporary directory. + :param extra_flags: A list of additional browser flags to pass to the browser on launch. :param selector_config: The arguments that will be passed in the end while creating the final Selector's class. :param additional_args: Additional arguments to be passed to Playwright's context as additional settings, and it takes higher priority than Scrapling's settings. """ @@ -149,6 +151,7 @@ class DynamicSession(DynamicSessionMixin, SyncSession): extra_headers=extra_headers, wait_selector=wait_selector, disable_webgl=disable_webgl, + extra_flags=extra_flags, selector_config=selector_config, additional_args=additional_args, disable_resources=disable_resources, @@ -306,6 +309,7 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession): load_dom: bool = True, wait_selector_state: SelectorWaitStates = "attached", user_data_dir: str = "", + extra_flags: Optional[List[str]] = None, selector_config: Optional[Dict] = None, additional_args: Optional[Dict] = None, ): @@ -336,6 +340,7 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession): :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only. :param max_pages: The maximum number of tabs to be opened at the same time. It will be used in rotation through a PagePool. :param user_data_dir: Path to a User Data Directory, which stores browser session data like cookies and local storage. The default is to create a temporary directory. + :param extra_flags: A list of additional browser flags to pass to the browser on launch. :param selector_config: The arguments that will be passed in the end while creating the final Selector's class. :param additional_args: Additional arguments to be passed to Playwright's context as additional settings, and it takes higher priority than Scrapling's settings. """ @@ -362,6 +367,7 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession): extra_headers=extra_headers, wait_selector=wait_selector, disable_webgl=disable_webgl, + extra_flags=extra_flags, selector_config=selector_config, additional_args=additional_args, disable_resources=disable_resources, diff --git a/scrapling/engines/_browsers/_validators.py b/scrapling/engines/_browsers/_validators.py index 0d7d236..831aacb 100644 --- a/scrapling/engines/_browsers/_validators.py +++ b/scrapling/engines/_browsers/_validators.py @@ -88,6 +88,7 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False): load_dom: bool = True wait_selector_state: SelectorWaitStates = "attached" user_data_dir: str = "" + extra_flags: Optional[List[str]] = None selector_config: Optional[Dict] = {} additional_args: Optional[Dict] = {} @@ -102,6 +103,8 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False): if not self.cookies: self.cookies = [] + if not self.extra_flags: + self.extra_flags = [] if not self.selector_config: self.selector_config = {} if not self.additional_args: diff --git a/scrapling/fetchers/chrome.py b/scrapling/fetchers/chrome.py index 9cc980d..0c2ab84 100644 --- a/scrapling/fetchers/chrome.py +++ b/scrapling/fetchers/chrome.py @@ -50,6 +50,7 @@ class DynamicFetcher(BaseFetcher): network_idle: bool = False, load_dom: bool = True, wait_selector_state: SelectorWaitStates = "attached", + extra_flags: Optional[List[str]] = None, additional_args: Optional[Dict] = None, custom_config: Optional[Dict] = None, ) -> Response: @@ -79,6 +80,7 @@ class DynamicFetcher(BaseFetcher): :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name. :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only. + :param extra_flags: A list of additional browser flags to pass to the browser on launch. :param custom_config: A dictionary of custom parser arguments to use with this request. Any argument passed will override any class parameters values. :param additional_args: Additional arguments to be passed to Playwright's context as additional settings, and it takes higher priority than Scrapling's settings. :return: A `Response` object. @@ -108,6 +110,7 @@ class DynamicFetcher(BaseFetcher): extra_headers=extra_headers, wait_selector=wait_selector, disable_webgl=disable_webgl, + extra_flags=extra_flags, additional_args=additional_args, disable_resources=disable_resources, wait_selector_state=wait_selector_state, @@ -140,6 +143,7 @@ class DynamicFetcher(BaseFetcher): network_idle: bool = False, load_dom: bool = True, wait_selector_state: SelectorWaitStates = "attached", + extra_flags: Optional[List[str]] = None, additional_args: Optional[Dict] = None, custom_config: Optional[Dict] = None, ) -> Response: @@ -169,6 +173,7 @@ class DynamicFetcher(BaseFetcher): :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name. :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only. + :param extra_flags: A list of additional browser flags to pass to the browser on launch. :param custom_config: A dictionary of custom parser arguments to use with this request. Any argument passed will override any class parameters values. :param additional_args: Additional arguments to be passed to Playwright's context as additional settings, and it takes higher priority than Scrapling's settings. :return: A `Response` object. @@ -199,6 +204,7 @@ class DynamicFetcher(BaseFetcher): extra_headers=extra_headers, wait_selector=wait_selector, disable_webgl=disable_webgl, + extra_flags=extra_flags, additional_args=additional_args, disable_resources=disable_resources, wait_selector_state=wait_selector_state,