From 5e13d3ece6d93d519f4a8cee83e95d7816400879 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Mon, 13 Apr 2026 03:57:27 +0200 Subject: [PATCH] feat(browsers): add a pre-navigation hook to allow page setup Solves #238 --- scrapling/core/_shell_signatures.py | 2 ++ scrapling/engines/_browsers/_controllers.py | 24 +++++++++++++++++---- scrapling/engines/_browsers/_stealth.py | 24 +++++++++++++++++---- scrapling/engines/_browsers/_types.py | 2 ++ scrapling/engines/_browsers/_validators.py | 4 ++++ scrapling/fetchers/chrome.py | 6 ++++-- scrapling/fetchers/stealth_chrome.py | 6 ++++-- 7 files changed, 56 insertions(+), 12 deletions(-) diff --git a/scrapling/core/_shell_signatures.py b/scrapling/core/_shell_signatures.py index b2340fb..00f1fe9 100644 --- a/scrapling/core/_shell_signatures.py +++ b/scrapling/core/_shell_signatures.py @@ -47,6 +47,7 @@ _FETCH_PARAMS = { "wait": int | float, "timezone_id": str | None, "page_action": Optional[Callable], + "page_setup": Optional[Callable], "proxy": Optional[str | Dict[str, str] | Tuple], "extra_headers": Optional[Dict[str, str]], "timeout": int | float, @@ -80,6 +81,7 @@ _STEALTHY_FETCH_PARAMS = { "wait": int | float, "timezone_id": str | None, "page_action": Optional[Callable], + "page_setup": Optional[Callable], "proxy": Optional[str | Dict[str, str] | Tuple], "extra_headers": Optional[Dict[str, str]], "timeout": int | float, diff --git a/scrapling/engines/_browsers/_controllers.py b/scrapling/engines/_browsers/_controllers.py index ce4b643..edabb17 100644 --- a/scrapling/engines/_browsers/_controllers.py +++ b/scrapling/engines/_browsers/_controllers.py @@ -47,7 +47,8 @@ class DynamicSession(SyncSession, DynamicSessionMixin): :param network_idle: Wait for the page until there are no network connections for at least 500 ms. :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000 :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object. - :param page_action: Added for automation. A function that takes the `page` object and does the automation you need. + :param page_action: Added for automation. A function that takes the `page` object, runs after navigation, and does the automation you need. + :param page_setup: A function that takes the `page` object, runs before navigation. Use it to register event listeners or routes that must be set up before the page loads. :param wait_selector: Wait for a specific CSS selector to be in a specific state. :param init_script: An absolute path to a JavaScript file to be executed on page creation for all pages in this session. :param locale: Specify user locale, for example, `en-GB`, `de-DE`, etc. Locale will affect navigator.language value, Accept-Language request header value as well as number and date formatting @@ -105,7 +106,8 @@ class DynamicSession(SyncSession, DynamicSessionMixin): :param google_search: Enabled by default, Scrapling will set a Google referer header. :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000 :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object. - :param page_action: Added for automation. A function that takes the `page` object and does the automation you need. + :param page_action: Added for automation. A function that takes the `page` object, runs after navigation, and does the automation you need. + :param page_setup: A function that takes the `page` object, runs before navigation. Use it to register event listeners or routes that must be set up before the page loads. :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by `google_search` takes priority over the referer set here if used together._ :param disable_resources: Drop requests for unnecessary resources for a speed boost. Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`. @@ -152,6 +154,12 @@ class DynamicSession(SyncSession, DynamicSessionMixin): ), ) + if params.page_setup: + try: + params.page_setup(page) + except Exception as e: # pragma: no cover + log.error(f"Error executing page_setup: {e}") + try: first_response = page.goto(url, referer=referer) self._wait_for_page_stability(page, params.load_dom, params.network_idle) @@ -228,7 +236,8 @@ class AsyncDynamicSession(AsyncSession, DynamicSessionMixin): :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute. :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000 :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object. - :param page_action: Added for automation. A function that takes the `page` object and does the automation you need. + :param page_action: Added for automation. A function that takes the `page` object, runs after navigation, and does the automation you need. + :param page_setup: A function that takes the `page` object, runs before navigation. Use it to register event listeners or routes that must be set up before the page loads. :param wait_selector: Wait for a specific CSS selector to be in a specific state. :param init_script: An absolute path to a JavaScript file to be executed on page creation for all pages in this session. :param locale: Specify user locale, for example, `en-GB`, `de-DE`, etc. Locale will affect navigator.language value, Accept-Language request header value as well as number and date formatting @@ -285,7 +294,8 @@ class AsyncDynamicSession(AsyncSession, DynamicSessionMixin): :param google_search: Enabled by default, Scrapling will set a Google referer header. :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000 :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object. - :param page_action: Added for automation. A function that takes the `page` object and does the automation you need. + :param page_action: Added for automation. A function that takes the `page` object, runs after navigation, and does the automation you need. + :param page_setup: A function that takes the `page` object, runs before navigation. Use it to register event listeners or routes that must be set up before the page loads. :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by `google_search` takes priority over the referer set here if used together._ :param disable_resources: Drop requests for unnecessary resources for a speed boost. Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`. @@ -333,6 +343,12 @@ class AsyncDynamicSession(AsyncSession, DynamicSessionMixin): ), ) + if params.page_setup: + try: + await params.page_setup(page) + except Exception as e: # pragma: no cover + log.error(f"Error executing page_setup: {e}") + try: first_response = await page.goto(url, referer=referer) await self._wait_for_page_stability(page, params.load_dom, params.network_idle) diff --git a/scrapling/engines/_browsers/_stealth.py b/scrapling/engines/_browsers/_stealth.py index f06c62e..efa195e 100644 --- a/scrapling/engines/_browsers/_stealth.py +++ b/scrapling/engines/_browsers/_stealth.py @@ -47,7 +47,8 @@ class StealthySession(SyncSession, StealthySessionMixin): :param network_idle: Wait for the page until there are no network connections for at least 500 ms. :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000 :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object. - :param page_action: Added for automation. A function that takes the `page` object and does the automation you need. + :param page_action: Added for automation. A function that takes the `page` object, runs after navigation, and does the automation you need. + :param page_setup: A function that takes the `page` object, runs before navigation. Use it to register event listeners or routes that must be set up before the page loads. :param wait_selector: Wait for a specific CSS selector to be in a specific state. :param init_script: An absolute path to a JavaScript file to be executed on page creation for all pages in this session. :param locale: Specify user locale, for example, `en-GB`, `de-DE`, etc. Locale will affect navigator.language value, Accept-Language request header value as well as number and date formatting @@ -187,7 +188,8 @@ class StealthySession(SyncSession, StealthySessionMixin): :param google_search: Enabled by default, Scrapling will set a Google referer header. :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000 :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object. - :param page_action: Added for automation. A function that takes the `page` object and does the automation you need. + :param page_action: Added for automation. A function that takes the `page` object, runs after navigation, and does the automation you need. + :param page_setup: A function that takes the `page` object, runs before navigation. Use it to register event listeners or routes that must be set up before the page loads. :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by `google_search` takes priority over the referer set here if used together._ :param disable_resources: Drop requests for unnecessary resources for a speed boost. Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`. @@ -235,6 +237,12 @@ class StealthySession(SyncSession, StealthySessionMixin): ), ) + if params.page_setup: + try: + params.page_setup(page) + except Exception as e: # pragma: no cover + log.error(f"Error executing page_setup: {e}") + try: first_response = page.goto(url, referer=referer) self._wait_for_page_stability(page, params.load_dom, params.network_idle) @@ -315,7 +323,8 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin): :param network_idle: Wait for the page until there are no network connections for at least 500 ms. :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000 :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object. - :param page_action: Added for automation. A function that takes the `page` object and does the automation you need. + :param page_action: Added for automation. A function that takes the `page` object, runs after navigation, and does the automation you need. + :param page_setup: A function that takes the `page` object, runs before navigation. Use it to register event listeners or routes that must be set up before the page loads. :param wait_selector: Wait for a specific CSS selector to be in a specific state. :param init_script: An absolute path to a JavaScript file to be executed on page creation for all pages in this session. :param locale: Specify user locale, for example, `en-GB`, `de-DE`, etc. Locale will affect navigator.language value, Accept-Language request header value as well as number and date formatting @@ -454,7 +463,8 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin): :param google_search: Enabled by default, Scrapling will set a Google referer header. :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000 :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object. - :param page_action: Added for automation. A function that takes the `page` object and does the automation you need. + :param page_action: Added for automation. A function that takes the `page` object, runs after navigation, and does the automation you need. + :param page_setup: A function that takes the `page` object, runs before navigation. Use it to register event listeners or routes that must be set up before the page loads. :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by `google_search` takes priority over the referer set here if used together._ :param disable_resources: Drop requests for unnecessary resources for a speed boost. Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`. @@ -503,6 +513,12 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin): ), ) + if params.page_setup: + try: + await params.page_setup(page) + except Exception as e: # pragma: no cover + log.error(f"Error executing page_setup: {e}") + try: first_response = await page.goto(url, referer=referer) await self._wait_for_page_stability(page, params.load_dom, params.network_idle) diff --git a/scrapling/engines/_browsers/_types.py b/scrapling/engines/_browsers/_types.py index 5030480..b436c59 100644 --- a/scrapling/engines/_browsers/_types.py +++ b/scrapling/engines/_browsers/_types.py @@ -74,6 +74,7 @@ class PlaywrightSession(TypedDict, total=False): wait: int | float timezone_id: str | None page_action: Optional[Callable] + page_setup: Optional[Callable] proxy: Optional[str | Dict[str, str] | Tuple] proxy_rotator: Optional[ProxyRotator] extra_headers: Optional[Dict[str, str]] @@ -105,6 +106,7 @@ class PlaywrightFetchParams(TypedDict, total=False): disable_resources: bool wait_selector: Optional[str] page_action: Optional[Callable] + page_setup: Optional[Callable] selector_config: Optional[Dict] extra_headers: Optional[Dict[str, str]] wait_selector_state: SelectorWaitStates diff --git a/scrapling/engines/_browsers/_validators.py b/scrapling/engines/_browsers/_validators.py index 504ba78..77dc81d 100644 --- a/scrapling/engines/_browsers/_validators.py +++ b/scrapling/engines/_browsers/_validators.py @@ -71,6 +71,7 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False, weakref=True): wait: Seconds = 0 timezone_id: str | None = "" page_action: Optional[Callable] = None + page_setup: Optional[Callable] = None proxy: Optional[str | Dict[str, str] | Tuple] = None # The default value for proxy in Playwright's source is `None` proxy_rotator: Optional[ProxyRotator] = None extra_headers: Optional[Dict[str, str]] = None @@ -96,6 +97,8 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False, weakref=True): """Custom validation after msgspec validation""" if self.page_action and not callable(self.page_action): raise TypeError(f"page_action must be callable, got {type(self.page_action).__name__}") + if self.page_setup and not callable(self.page_setup): + raise TypeError(f"page_setup must be callable, got {type(self.page_setup).__name__}") if self.proxy and self.proxy_rotator: raise ValueError( "Cannot use 'proxy_rotator' together with 'proxy'. " @@ -160,6 +163,7 @@ class _fetch_params: timeout: Seconds wait: Seconds page_action: Optional[Callable] + page_setup: Optional[Callable] extra_headers: Optional[Dict[str, str]] disable_resources: bool wait_selector: Optional[str] diff --git a/scrapling/fetchers/chrome.py b/scrapling/fetchers/chrome.py index f7795b7..3c2efcf 100644 --- a/scrapling/fetchers/chrome.py +++ b/scrapling/fetchers/chrome.py @@ -23,7 +23,8 @@ class DynamicFetcher(BaseFetcher): :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute. :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000 :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the Response object. - :param page_action: Added for automation. A function that takes the `page` object and does the automation you need. + :param page_action: Added for automation. A function that takes the `page` object, runs after navigation, and does the automation you need. + :param page_setup: A function that takes the `page` object, runs before navigation. Use it to register event listeners or routes that must be set up before the page loads. :param wait_selector: Wait for a specific CSS selector to be in a specific state. :param init_script: An absolute path to a JavaScript file to be executed on page creation with this request. :param locale: Set the locale for the browser if wanted. Defaults to the system default locale. @@ -65,7 +66,8 @@ class DynamicFetcher(BaseFetcher): :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute. :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000 :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the Response object. - :param page_action: Added for automation. A function that takes the `page` object and does the automation you need. + :param page_action: Added for automation. A function that takes the `page` object, runs after navigation, and does the automation you need. + :param page_setup: A function that takes the `page` object, runs before navigation. Use it to register event listeners or routes that must be set up before the page loads. :param wait_selector: Wait for a specific CSS selector to be in a specific state. :param init_script: An absolute path to a JavaScript file to be executed on page creation with this request. :param locale: Set the locale for the browser if wanted. Defaults to the system default locale. diff --git a/scrapling/fetchers/stealth_chrome.py b/scrapling/fetchers/stealth_chrome.py index 70ce937..ad6e01f 100644 --- a/scrapling/fetchers/stealth_chrome.py +++ b/scrapling/fetchers/stealth_chrome.py @@ -27,7 +27,8 @@ class StealthyFetcher(BaseFetcher): :param network_idle: Wait for the page until there are no network connections for at least 500 ms. :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000 :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object. - :param page_action: Added for automation. A function that takes the `page` object and does the automation you need. + :param page_action: Added for automation. A function that takes the `page` object, runs after navigation, and does the automation you need. + :param page_setup: A function that takes the `page` object, runs before navigation. Use it to register event listeners or routes that must be set up before the page loads. :param wait_selector: Wait for a specific CSS selector to be in a specific state. :param init_script: An absolute path to a JavaScript file to be executed on page creation for all pages in this session. :param locale: Specify user locale, for example, `en-GB`, `de-DE`, etc. Locale will affect navigator.language value, Accept-Language request header value as well as number and date formatting @@ -78,7 +79,8 @@ class StealthyFetcher(BaseFetcher): :param network_idle: Wait for the page until there are no network connections for at least 500 ms. :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000 :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object. - :param page_action: Added for automation. A function that takes the `page` object and does the automation you need. + :param page_action: Added for automation. A function that takes the `page` object, runs after navigation, and does the automation you need. + :param page_setup: A function that takes the `page` object, runs before navigation. Use it to register event listeners or routes that must be set up before the page loads. :param wait_selector: Wait for a specific CSS selector to be in a specific state. :param init_script: An absolute path to a JavaScript file to be executed on page creation for all pages in this session. :param locale: Specify user locale, for example, `en-GB`, `de-DE`, etc. Locale will affect navigator.language value, Accept-Language request header value as well as number and date formatting