freedak/Scrapling
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(Dynamic fetcher): Add additional_args option to customize browser as in StealthyFetcher

Browse Source
This commit is contained in:
Karim shoair
2025-10-12 05:07:22 +03:00
parent 4fea8c2e59
commit a38b92cc87
5 changed files with 21 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/fetching/dynamic.md
+1
View File
@@ -89,6 +89,7 @@ Scrapling provides many options with this fetcher and its session classes. To ma
| locale | Set the locale for the browser if wanted. The default value is `en-US`. | ✔️ |
| cdp_url | Instead of launching a new browser instance, connect to this CDP URL to control real browsers through CDP. | ✔️ |
| user_data_dir | Path to a User Data Directory, which stores browser session data like cookies and local storage. The default is to create a temporary directory. **Only Works with sessions** | ✔️ |
| additional_args | Additional arguments to be passed to Playwright's context as additional settings, and it takes higher priority than Scrapling's settings. | ✔️ |
| selector_config | A dictionary of custom parsing arguments to be used when creating the final `Selector`/`Response` class. | ✔️ |
In session classes, all these arguments can be set globally for the session. Still, you can configure each request individually by passing some of the arguments here that can be configured on the browser tab level like: `google_search`, `timeout`, `wait`, `page_action`, `extra_headers`, `disable_resources`, `wait_selector`, `wait_selector_state`, `network_idle`, `load_dom`, and `selector_config`.
scrapling/engines/_browsers/_base.py
+3
View File
@@ -148,6 +148,7 @@ class DynamicSessionMixin:
self.init_script = config.init_script
self.wait_selector_state = config.wait_selector_state
self.selector_config = config.selector_config
self.additional_args = config.additional_args
self.page_action = config.page_action
self.user_data_dir = config.user_data_dir
self._headers_keys = {header.lower() for header in self.extra_headers.keys()} if self.extra_headers else set()
@@ -175,6 +176,7 @@ class DynamicSessionMixin:
self.launch_options["extra_http_headers"] = dict(self.launch_options["extra_http_headers"])
self.launch_options["proxy"] = dict(self.launch_options["proxy"]) or None
self.launch_options["user_data_dir"] = self.user_data_dir
self.launch_options.update(cast(Dict, self.additional_args))
self.context_options = dict()
else:
# while `context_options` is left to be used when cdp mode is enabled
@@ -190,6 +192,7 @@ class DynamicSessionMixin:
)
self.context_options["extra_http_headers"] = dict(self.context_options["extra_http_headers"])
self.context_options["proxy"] = dict(self.context_options["proxy"]) or None
self.context_options.update(cast(Dict, self.additional_args))
class StealthySessionMixin:
scrapling/engines/_browsers/_controllers.py
+8 -2
View File
@@ -99,6 +99,7 @@ class DynamicSession(DynamicSessionMixin, SyncSession):
wait_selector_state: SelectorWaitStates = "attached",
user_data_dir: str = "",
selector_config: Optional[Dict] = None,
additional_args: Optional[Dict] = None,
):
"""A Browser session manager with page pooling, it's using a persistent browser Context by default with a temporary user profile directory.
@@ -127,6 +128,7 @@ class DynamicSession(DynamicSessionMixin, SyncSession):
:param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.
:param user_data_dir: Path to a User Data Directory, which stores browser session data like cookies and local storage. The default is to create a temporary directory.
:param selector_config: The arguments that will be passed in the end while creating the final Selector's class.
:param additional_args: Additional arguments to be passed to Playwright's context as additional settings, and it takes higher priority than Scrapling's settings.
"""
self.__validate__(
wait=wait,
@@ -151,6 +153,7 @@ class DynamicSession(DynamicSessionMixin, SyncSession):
wait_selector=wait_selector,
disable_webgl=disable_webgl,
selector_config=selector_config,
additional_args=additional_args,
disable_resources=disable_resources,
wait_selector_state=wait_selector_state,
)
@@ -306,7 +309,7 @@ class DynamicSession(DynamicSessionMixin, SyncSession):
page_info.page, first_response, final_response, params.selector_config
)
# Close the page, to free up resources
# Close the page to free up resources
page_info.page.close()
self.page_pool.pages.remove(page_info)
@@ -346,6 +349,7 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession):
wait_selector_state: SelectorWaitStates = "attached",
user_data_dir: str = "",
selector_config: Optional[Dict] = None,
additional_args: Optional[Dict] = None,
):
"""A Browser session manager with page pooling
@@ -375,6 +379,7 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession):
:param max_pages: The maximum number of tabs to be opened at the same time. It will be used in rotation through a PagePool.
:param user_data_dir: Path to a User Data Directory, which stores browser session data like cookies and local storage. The default is to create a temporary directory.
:param selector_config: The arguments that will be passed in the end while creating the final Selector's class.
:param additional_args: Additional arguments to be passed to Playwright's context as additional settings, and it takes higher priority than Scrapling's settings.
"""
self.__validate__(
@@ -400,6 +405,7 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession):
wait_selector=wait_selector,
disable_webgl=disable_webgl,
selector_config=selector_config,
additional_args=additional_args,
disable_resources=disable_resources,
wait_selector_state=wait_selector_state,
)
@@ -560,7 +566,7 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession):
page_info.page, first_response, final_response, params.selector_config
)
# Close the page, to free up resources
# Close the page to free up resources
await page_info.page.close()
self.page_pool.pages.remove(page_info)
return response
scrapling/engines/_browsers/_validators.py
+3
View File
@@ -89,6 +89,7 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False):
wait_selector_state: SelectorWaitStates = "attached"
user_data_dir: str = ""
selector_config: Optional[Dict] = {}
additional_args: Optional[Dict] = {}
def __post_init__(self):
"""Custom validation after msgspec validation"""
@@ -103,6 +104,8 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False):
self.cookies = []
if not self.selector_config:
self.selector_config = {}
if not self.additional_args:
self.additional_args = {}
if self.init_script is not None:
_validate_file_path(self.init_script)
scrapling/fetchers/chrome.py
+6
View File
@@ -50,6 +50,7 @@ class DynamicFetcher(BaseFetcher):
network_idle: bool = False,
load_dom: bool = True,
wait_selector_state: SelectorWaitStates = "attached",
additional_args: Optional[Dict] = None,
custom_config: Optional[Dict] = None,
) -> Response:
"""Opens up a browser and do your request based on your chosen options below.
@@ -79,6 +80,7 @@ class DynamicFetcher(BaseFetcher):
:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
:param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.
:param custom_config: A dictionary of custom parser arguments to use with this request. Any argument passed will override any class parameters values.
:param additional_args: Additional arguments to be passed to Playwright's context as additional settings, and it takes higher priority than Scrapling's settings.
:return: A `Response` object.
"""
if not custom_config:
@@ -106,6 +108,7 @@ class DynamicFetcher(BaseFetcher):
extra_headers=extra_headers,
wait_selector=wait_selector,
disable_webgl=disable_webgl,
additional_args=additional_args,
disable_resources=disable_resources,
wait_selector_state=wait_selector_state,
selector_config={**cls._generate_parser_arguments(), **custom_config},
@@ -137,6 +140,7 @@ class DynamicFetcher(BaseFetcher):
network_idle: bool = False,
load_dom: bool = True,
wait_selector_state: SelectorWaitStates = "attached",
additional_args: Optional[Dict] = None,
custom_config: Optional[Dict] = None,
) -> Response:
"""Opens up a browser and do your request based on your chosen options below.
@@ -166,6 +170,7 @@ class DynamicFetcher(BaseFetcher):
:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
:param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.
:param custom_config: A dictionary of custom parser arguments to use with this request. Any argument passed will override any class parameters values.
:param additional_args: Additional arguments to be passed to Playwright's context as additional settings, and it takes higher priority than Scrapling's settings.
:return: A `Response` object.
"""
if not custom_config:
@@ -194,6 +199,7 @@ class DynamicFetcher(BaseFetcher):
extra_headers=extra_headers,
wait_selector=wait_selector,
disable_webgl=disable_webgl,
additional_args=additional_args,
disable_resources=disable_resources,
wait_selector_state=wait_selector_state,
selector_config={**cls._generate_parser_arguments(), **custom_config},