From 6ad1bab2e4d689573609748975a3e03a6d2c3cf0 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Tue, 2 Sep 2025 16:25:13 +0300 Subject: [PATCH] feat(DynamicFetcher): add the ability to add `init_script` to context (#56) --- scrapling/engines/_browsers/_controllers.py | 14 ++++++++++++++ scrapling/engines/_browsers/_validators.py | 10 ++++++++++ scrapling/fetchers.py | 6 ++++++ 3 files changed, 30 insertions(+) diff --git a/scrapling/engines/_browsers/_controllers.py b/scrapling/engines/_browsers/_controllers.py index 285553f..2747ed9 100644 --- a/scrapling/engines/_browsers/_controllers.py +++ b/scrapling/engines/_browsers/_controllers.py @@ -60,6 +60,7 @@ class DynamicSession: "disable_resources", "network_idle", "wait_selector", + "init_script", "wait_selector_state", "wait", "playwright", @@ -94,6 +95,7 @@ class DynamicSession: timeout: int | float = 30000, disable_resources: bool = False, wait_selector: Optional[str] = None, + init_script: Optional[str] = None, cookies: Optional[List[Dict]] = None, network_idle: bool = False, wait_selector_state: SelectorWaitStates = "attached", @@ -112,6 +114,7 @@ class DynamicSession: :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object. :param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again. :param wait_selector: Wait for a specific CSS selector to be in a specific state. + :param init_script: An absolute path to a JavaScript file to be executed on page creation for all pages in this session. :param locale: Set the locale for the browser if wanted. The default value is `en-US`. :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`. :param stealth: Enables stealth mode, check the documentation to see what stealth mode does currently. @@ -143,6 +146,7 @@ class DynamicSession: "selector_config": selector_config, "disable_resources": disable_resources, "wait_selector": wait_selector, + "init_script": init_script, "cookies": cookies, "network_idle": network_idle, "wait_selector_state": wait_selector_state, @@ -168,6 +172,7 @@ class DynamicSession: self.cdp_url = config.cdp_url self.network_idle = config.network_idle self.wait_selector = config.wait_selector + self.init_script = config.init_script self.wait_selector_state = config.wait_selector_state self.playwright: Optional[Playwright] = None @@ -243,6 +248,9 @@ class DynamicSession: user_data_dir="", **self.launch_options ) + if self.init_script: # pragma: no cover + self.context.add_init_script(path=self.init_script) + if self.cookies: # pragma: no cover self.context.add_cookies(self.cookies) @@ -409,6 +417,7 @@ class AsyncDynamicSession(DynamicSession): timeout: int | float = 30000, disable_resources: bool = False, wait_selector: Optional[str] = None, + init_script: Optional[str] = None, cookies: Optional[List[Dict]] = None, network_idle: bool = False, wait_selector_state: SelectorWaitStates = "attached", @@ -427,6 +436,7 @@ class AsyncDynamicSession(DynamicSession): :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object. :param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again. :param wait_selector: Wait for a specific CSS selector to be in a specific state. + :param init_script: An absolute path to a JavaScript file to be executed on page creation for all pages in this session. :param locale: Set the locale for the browser if wanted. The default value is `en-US`. :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`. :param stealth: Enables stealth mode, check the documentation to see what stealth mode does currently. @@ -459,6 +469,7 @@ class AsyncDynamicSession(DynamicSession): timeout, disable_resources, wait_selector, + init_script, cookies, network_idle, wait_selector_state, @@ -494,6 +505,9 @@ class AsyncDynamicSession(DynamicSession): ) ) + if self.init_script: # pragma: no cover + await self.context.add_init_script(path=self.init_script) + if self.cookies: await self.context.add_cookies(self.cookies) diff --git a/scrapling/engines/_browsers/_validators.py b/scrapling/engines/_browsers/_validators.py index d2b321e..6363df7 100644 --- a/scrapling/engines/_browsers/_validators.py +++ b/scrapling/engines/_browsers/_validators.py @@ -32,6 +32,7 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False): extra_headers: Optional[Dict[str, str]] = None useragent: Optional[str] = None timeout: int | float = 30000 + init_script: Optional[str] = None disable_resources: bool = False wait_selector: Optional[str] = None cookies: Optional[List[Dict]] = None @@ -58,6 +59,15 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False): if not self.selector_config: self.selector_config = {} + if self.init_script is not None: + script_path = Path(self.init_script) + if not script_path.exists(): + raise ValueError("Init script path not found") + elif not script_path.is_file(): + raise ValueError("Init script is not a file") + elif not script_path.is_absolute(): + raise ValueError("Init script is not a absolute path") + @staticmethod def __validate_cdp(cdp_url): try: diff --git a/scrapling/fetchers.py b/scrapling/fetchers.py index 054906c..c1ce8f7 100644 --- a/scrapling/fetchers.py +++ b/scrapling/fetchers.py @@ -282,6 +282,7 @@ class DynamicFetcher(BaseFetcher): timeout: int | float = 30000, disable_resources: bool = False, wait_selector: Optional[str] = None, + init_script: Optional[str] = None, cookies: Optional[Iterable[Dict]] = None, network_idle: bool = False, wait_selector_state: SelectorWaitStates = "attached", @@ -301,6 +302,7 @@ class DynamicFetcher(BaseFetcher): :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object. :param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again. :param wait_selector: Wait for a specific CSS selector to be in a specific state. + :param init_script: An absolute path to a JavaScript file to be executed on page creation with this request. :param locale: Set the locale for the browser if wanted. The default value is `en-US`. :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`. :param stealth: Enables stealth mode, check the documentation to see what stealth mode does currently. @@ -334,6 +336,7 @@ class DynamicFetcher(BaseFetcher): real_chrome=real_chrome, page_action=page_action, hide_canvas=hide_canvas, + init_script=init_script, network_idle=network_idle, google_search=google_search, extra_headers=extra_headers, @@ -365,6 +368,7 @@ class DynamicFetcher(BaseFetcher): timeout: int | float = 30000, disable_resources: bool = False, wait_selector: Optional[str] = None, + init_script: Optional[str] = None, cookies: Optional[Iterable[Dict]] = None, network_idle: bool = False, wait_selector_state: SelectorWaitStates = "attached", @@ -384,6 +388,7 @@ class DynamicFetcher(BaseFetcher): :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object. :param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again. :param wait_selector: Wait for a specific CSS selector to be in a specific state. + :param init_script: An absolute path to a JavaScript file to be executed on page creation with this request. :param locale: Set the locale for the browser if wanted. The default value is `en-US`. :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`. :param stealth: Enables stealth mode, check the documentation to see what stealth mode does currently. @@ -418,6 +423,7 @@ class DynamicFetcher(BaseFetcher): real_chrome=real_chrome, page_action=page_action, hide_canvas=hide_canvas, + init_script=init_script, network_idle=network_idle, google_search=google_search, extra_headers=extra_headers,