From a38b92cc878f3061c8aac5e5289c9bd26a6eba80 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Sun, 12 Oct 2025 05:07:22 +0300 Subject: [PATCH] feat(Dynamic fetcher): Add `additional_args` option to customize browser as in StealthyFetcher --- docs/fetching/dynamic.md | 1 + scrapling/engines/_browsers/_base.py | 3 +++ scrapling/engines/_browsers/_controllers.py | 10 ++++++++-- scrapling/engines/_browsers/_validators.py | 3 +++ scrapling/fetchers/chrome.py | 6 ++++++ 5 files changed, 21 insertions(+), 2 deletions(-) diff --git a/docs/fetching/dynamic.md b/docs/fetching/dynamic.md index f8d20c8..62e6ad9 100644 --- a/docs/fetching/dynamic.md +++ b/docs/fetching/dynamic.md @@ -89,6 +89,7 @@ Scrapling provides many options with this fetcher and its session classes. To ma | locale | Set the locale for the browser if wanted. The default value is `en-US`. | ✔️ | | cdp_url | Instead of launching a new browser instance, connect to this CDP URL to control real browsers through CDP. | ✔️ | | user_data_dir | Path to a User Data Directory, which stores browser session data like cookies and local storage. The default is to create a temporary directory. **Only Works with sessions** | ✔️ | +| additional_args | Additional arguments to be passed to Playwright's context as additional settings, and it takes higher priority than Scrapling's settings. | ✔️ | | selector_config | A dictionary of custom parsing arguments to be used when creating the final `Selector`/`Response` class. | ✔️ | In session classes, all these arguments can be set globally for the session. Still, you can configure each request individually by passing some of the arguments here that can be configured on the browser tab level like: `google_search`, `timeout`, `wait`, `page_action`, `extra_headers`, `disable_resources`, `wait_selector`, `wait_selector_state`, `network_idle`, `load_dom`, and `selector_config`. diff --git a/scrapling/engines/_browsers/_base.py b/scrapling/engines/_browsers/_base.py index ce82dbb..9a6dfae 100644 --- a/scrapling/engines/_browsers/_base.py +++ b/scrapling/engines/_browsers/_base.py @@ -148,6 +148,7 @@ class DynamicSessionMixin: self.init_script = config.init_script self.wait_selector_state = config.wait_selector_state self.selector_config = config.selector_config + self.additional_args = config.additional_args self.page_action = config.page_action self.user_data_dir = config.user_data_dir self._headers_keys = {header.lower() for header in self.extra_headers.keys()} if self.extra_headers else set() @@ -175,6 +176,7 @@ class DynamicSessionMixin: self.launch_options["extra_http_headers"] = dict(self.launch_options["extra_http_headers"]) self.launch_options["proxy"] = dict(self.launch_options["proxy"]) or None self.launch_options["user_data_dir"] = self.user_data_dir + self.launch_options.update(cast(Dict, self.additional_args)) self.context_options = dict() else: # while `context_options` is left to be used when cdp mode is enabled @@ -190,6 +192,7 @@ class DynamicSessionMixin: ) self.context_options["extra_http_headers"] = dict(self.context_options["extra_http_headers"]) self.context_options["proxy"] = dict(self.context_options["proxy"]) or None + self.context_options.update(cast(Dict, self.additional_args)) class StealthySessionMixin: diff --git a/scrapling/engines/_browsers/_controllers.py b/scrapling/engines/_browsers/_controllers.py index ccfbb11..ca8b45e 100644 --- a/scrapling/engines/_browsers/_controllers.py +++ b/scrapling/engines/_browsers/_controllers.py @@ -99,6 +99,7 @@ class DynamicSession(DynamicSessionMixin, SyncSession): wait_selector_state: SelectorWaitStates = "attached", user_data_dir: str = "", selector_config: Optional[Dict] = None, + additional_args: Optional[Dict] = None, ): """A Browser session manager with page pooling, it's using a persistent browser Context by default with a temporary user profile directory. @@ -127,6 +128,7 @@ class DynamicSession(DynamicSessionMixin, SyncSession): :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only. :param user_data_dir: Path to a User Data Directory, which stores browser session data like cookies and local storage. The default is to create a temporary directory. :param selector_config: The arguments that will be passed in the end while creating the final Selector's class. + :param additional_args: Additional arguments to be passed to Playwright's context as additional settings, and it takes higher priority than Scrapling's settings. """ self.__validate__( wait=wait, @@ -151,6 +153,7 @@ class DynamicSession(DynamicSessionMixin, SyncSession): wait_selector=wait_selector, disable_webgl=disable_webgl, selector_config=selector_config, + additional_args=additional_args, disable_resources=disable_resources, wait_selector_state=wait_selector_state, ) @@ -306,7 +309,7 @@ class DynamicSession(DynamicSessionMixin, SyncSession): page_info.page, first_response, final_response, params.selector_config ) - # Close the page, to free up resources + # Close the page to free up resources page_info.page.close() self.page_pool.pages.remove(page_info) @@ -346,6 +349,7 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession): wait_selector_state: SelectorWaitStates = "attached", user_data_dir: str = "", selector_config: Optional[Dict] = None, + additional_args: Optional[Dict] = None, ): """A Browser session manager with page pooling @@ -375,6 +379,7 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession): :param max_pages: The maximum number of tabs to be opened at the same time. It will be used in rotation through a PagePool. :param user_data_dir: Path to a User Data Directory, which stores browser session data like cookies and local storage. The default is to create a temporary directory. :param selector_config: The arguments that will be passed in the end while creating the final Selector's class. + :param additional_args: Additional arguments to be passed to Playwright's context as additional settings, and it takes higher priority than Scrapling's settings. """ self.__validate__( @@ -400,6 +405,7 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession): wait_selector=wait_selector, disable_webgl=disable_webgl, selector_config=selector_config, + additional_args=additional_args, disable_resources=disable_resources, wait_selector_state=wait_selector_state, ) @@ -560,7 +566,7 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession): page_info.page, first_response, final_response, params.selector_config ) - # Close the page, to free up resources + # Close the page to free up resources await page_info.page.close() self.page_pool.pages.remove(page_info) return response diff --git a/scrapling/engines/_browsers/_validators.py b/scrapling/engines/_browsers/_validators.py index af90d9e..0d7d236 100644 --- a/scrapling/engines/_browsers/_validators.py +++ b/scrapling/engines/_browsers/_validators.py @@ -89,6 +89,7 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False): wait_selector_state: SelectorWaitStates = "attached" user_data_dir: str = "" selector_config: Optional[Dict] = {} + additional_args: Optional[Dict] = {} def __post_init__(self): """Custom validation after msgspec validation""" @@ -103,6 +104,8 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False): self.cookies = [] if not self.selector_config: self.selector_config = {} + if not self.additional_args: + self.additional_args = {} if self.init_script is not None: _validate_file_path(self.init_script) diff --git a/scrapling/fetchers/chrome.py b/scrapling/fetchers/chrome.py index 3c33c47..9cc980d 100644 --- a/scrapling/fetchers/chrome.py +++ b/scrapling/fetchers/chrome.py @@ -50,6 +50,7 @@ class DynamicFetcher(BaseFetcher): network_idle: bool = False, load_dom: bool = True, wait_selector_state: SelectorWaitStates = "attached", + additional_args: Optional[Dict] = None, custom_config: Optional[Dict] = None, ) -> Response: """Opens up a browser and do your request based on your chosen options below. @@ -79,6 +80,7 @@ class DynamicFetcher(BaseFetcher): :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only. :param custom_config: A dictionary of custom parser arguments to use with this request. Any argument passed will override any class parameters values. + :param additional_args: Additional arguments to be passed to Playwright's context as additional settings, and it takes higher priority than Scrapling's settings. :return: A `Response` object. """ if not custom_config: @@ -106,6 +108,7 @@ class DynamicFetcher(BaseFetcher): extra_headers=extra_headers, wait_selector=wait_selector, disable_webgl=disable_webgl, + additional_args=additional_args, disable_resources=disable_resources, wait_selector_state=wait_selector_state, selector_config={**cls._generate_parser_arguments(), **custom_config}, @@ -137,6 +140,7 @@ class DynamicFetcher(BaseFetcher): network_idle: bool = False, load_dom: bool = True, wait_selector_state: SelectorWaitStates = "attached", + additional_args: Optional[Dict] = None, custom_config: Optional[Dict] = None, ) -> Response: """Opens up a browser and do your request based on your chosen options below. @@ -166,6 +170,7 @@ class DynamicFetcher(BaseFetcher): :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only. :param custom_config: A dictionary of custom parser arguments to use with this request. Any argument passed will override any class parameters values. + :param additional_args: Additional arguments to be passed to Playwright's context as additional settings, and it takes higher priority than Scrapling's settings. :return: A `Response` object. """ if not custom_config: @@ -194,6 +199,7 @@ class DynamicFetcher(BaseFetcher): extra_headers=extra_headers, wait_selector=wait_selector, disable_webgl=disable_webgl, + additional_args=additional_args, disable_resources=disable_resources, wait_selector_state=wait_selector_state, selector_config={**cls._generate_parser_arguments(), **custom_config},