From 8c4d6158d6d69e71e1e306ab9c2fe1c7378558a7 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Sat, 13 Sep 2025 03:59:12 +0300 Subject: [PATCH] feat(browser fetchers): A new option to control page JS --- scrapling/engines/_browsers/_base.py | 2 ++ scrapling/engines/_browsers/_camoufox.py | 31 +++++++++++++++++---- scrapling/engines/_browsers/_controllers.py | 25 ++++++++++++++--- scrapling/engines/_browsers/_validators.py | 2 ++ scrapling/fetchers.py | 14 +++++++++- 5 files changed, 63 insertions(+), 11 deletions(-) diff --git a/scrapling/engines/_browsers/_base.py b/scrapling/engines/_browsers/_base.py index 6412698..5bb86ab 100644 --- a/scrapling/engines/_browsers/_base.py +++ b/scrapling/engines/_browsers/_base.py @@ -162,6 +162,7 @@ class DynamicSessionMixin: self.disable_resources = config.disable_resources self.cdp_url = config.cdp_url self.network_idle = config.network_idle + self.load_dom = config.load_dom self.wait_selector = config.wait_selector self.init_script = config.init_script self.wait_selector_state = config.wait_selector_state @@ -216,6 +217,7 @@ class StealthySessionMixin: self.block_webrtc = config.block_webrtc self.allow_webgl = config.allow_webgl self.network_idle = config.network_idle + self.load_dom = config.load_dom self.humanize = config.humanize self.solve_cloudflare = config.solve_cloudflare self.wait = config.wait diff --git a/scrapling/engines/_browsers/_camoufox.py b/scrapling/engines/_browsers/_camoufox.py index fcb77ac..31113be 100644 --- a/scrapling/engines/_browsers/_camoufox.py +++ b/scrapling/engines/_browsers/_camoufox.py @@ -46,6 +46,7 @@ class StealthySession(StealthySessionMixin, SyncSession): "block_webrtc", "allow_webgl", "network_idle", + "load_dom", "humanize", "solve_cloudflare", "wait", @@ -82,6 +83,7 @@ class StealthySession(StealthySessionMixin, SyncSession): block_webrtc: bool = False, allow_webgl: bool = True, network_idle: bool = False, + load_dom: bool = True, humanize: bool | float = True, solve_cloudflare: bool = False, wait: int | float = 0, @@ -116,6 +118,7 @@ class StealthySession(StealthySessionMixin, SyncSession): :param solve_cloudflare: Solves all 3 types of the Cloudflare's Turnstile wait page before returning the response to you. :param allow_webgl: Enabled by default. Disabling WebGL is not recommended as many WAFs now check if WebGL is enabled. :param network_idle: Wait for the page until there are no network connections for at least 500 ms. + :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute. :param disable_ads: Disabled by default, this installs the `uBlock Origin` addon on the browser if enabled. :param os_randomize: If enabled, Scrapling will randomize the OS fingerprints used. The default is Scrapling matching the fingerprints with the current OS. :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object. @@ -142,6 +145,7 @@ class StealthySession(StealthySessionMixin, SyncSession): cookies=cookies, headless=headless, humanize=humanize, + load_dom=load_dom, max_pages=__max_pages, disable_ads=disable_ads, allow_webgl=allow_webgl, @@ -259,6 +263,7 @@ class StealthySession(StealthySessionMixin, SyncSession): wait_selector: Optional[str] = _UNSET, wait_selector_state: SelectorWaitStates = _UNSET, network_idle: bool = _UNSET, + load_dom: bool = _UNSET, solve_cloudflare: bool = _UNSET, selector_config: Optional[Dict] = _UNSET, ) -> Response: @@ -276,6 +281,7 @@ class StealthySession(StealthySessionMixin, SyncSession): :param wait_selector: Wait for a specific CSS selector to be in a specific state. :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`. :param network_idle: Wait for the page until there are no network connections for at least 500 ms. + :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute. :param solve_cloudflare: Solves all 3 types of the Cloudflare's Turnstile wait page before returning the response to you. :param selector_config: The arguments that will be passed in the end while creating the final Selector's class. :return: A `Response` object. @@ -292,6 +298,7 @@ class StealthySession(StealthySessionMixin, SyncSession): wait_selector=self._get_with_precedence(wait_selector, self.wait_selector, _UNSET), wait_selector_state=self._get_with_precedence(wait_selector_state, self.wait_selector_state, _UNSET), network_idle=self._get_with_precedence(network_idle, self.network_idle, _UNSET), + load_dom=self._get_with_precedence(load_dom, self.load_dom, _UNSET), solve_cloudflare=self._get_with_precedence(solve_cloudflare, self.solve_cloudflare, _UNSET), selector_config=self._get_with_precedence(selector_config, self.selector_config, _UNSET), ), @@ -321,7 +328,8 @@ class StealthySession(StealthySessionMixin, SyncSession): # Navigate to URL and wait for a specified state page_info.page.on("response", handle_response) first_response = page_info.page.goto(url, referer=referer) - page_info.page.wait_for_load_state(state="domcontentloaded") + if params.load_dom: + page_info.page.wait_for_load_state(state="domcontentloaded") if params.network_idle: page_info.page.wait_for_load_state("networkidle") @@ -333,7 +341,8 @@ class StealthySession(StealthySessionMixin, SyncSession): self._solve_cloudflare(page_info.page) # Make sure the page is fully loaded after the captcha page_info.page.wait_for_load_state(state="load") - page_info.page.wait_for_load_state(state="domcontentloaded") + if params.load_dom: + page_info.page.wait_for_load_state(state="domcontentloaded") if params.network_idle: page_info.page.wait_for_load_state("networkidle") @@ -349,7 +358,8 @@ class StealthySession(StealthySessionMixin, SyncSession): waiter.first.wait_for(state=params.wait_selector_state) # Wait again after waiting for the selector, helpful with protections like Cloudflare page_info.page.wait_for_load_state(state="load") - page_info.page.wait_for_load_state(state="domcontentloaded") + if params.load_dom: + page_info.page.wait_for_load_state(state="domcontentloaded") if params.network_idle: page_info.page.wait_for_load_state("networkidle") except Exception as e: @@ -382,6 +392,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession): block_webrtc: bool = False, allow_webgl: bool = True, network_idle: bool = False, + load_dom: bool = True, humanize: bool | float = True, solve_cloudflare: bool = False, wait: int | float = 0, @@ -416,6 +427,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession): :param solve_cloudflare: Solves all 3 types of the Cloudflare's Turnstile wait page before returning the response to you. :param allow_webgl: Enabled by default. Disabling WebGL is not recommended as many WAFs now check if WebGL is enabled. :param network_idle: Wait for the page until there are no network connections for at least 500 ms. + :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute. :param disable_ads: Disabled by default, this installs the `uBlock Origin` addon on the browser if enabled. :param os_randomize: If enabled, Scrapling will randomize the OS fingerprints used. The default is Scrapling matching the fingerprints with the current OS. :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object. @@ -441,6 +453,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession): timeout=timeout, cookies=cookies, headless=headless, + load_dom=load_dom, humanize=humanize, max_pages=max_pages, disable_ads=disable_ads, @@ -559,6 +572,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession): wait_selector: Optional[str] = _UNSET, wait_selector_state: SelectorWaitStates = _UNSET, network_idle: bool = _UNSET, + load_dom: bool = _UNSET, solve_cloudflare: bool = _UNSET, selector_config: Optional[Dict] = _UNSET, ) -> Response: @@ -576,6 +590,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession): :param wait_selector: Wait for a specific CSS selector to be in a specific state. :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`. :param network_idle: Wait for the page until there are no network connections for at least 500 ms. + :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute. :param solve_cloudflare: Solves all 3 types of the Cloudflare's Turnstile wait page before returning the response to you. :param selector_config: The arguments that will be passed in the end while creating the final Selector's class. :return: A `Response` object. @@ -591,6 +606,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession): wait_selector=self._get_with_precedence(wait_selector, self.wait_selector, _UNSET), wait_selector_state=self._get_with_precedence(wait_selector_state, self.wait_selector_state, _UNSET), network_idle=self._get_with_precedence(network_idle, self.network_idle, _UNSET), + load_dom=self._get_with_precedence(load_dom, self.load_dom, _UNSET), solve_cloudflare=self._get_with_precedence(solve_cloudflare, self.solve_cloudflare, _UNSET), selector_config=self._get_with_precedence(selector_config, self.selector_config, _UNSET), ), @@ -620,7 +636,8 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession): # Navigate to URL and wait for a specified state page_info.page.on("response", handle_response) first_response = await page_info.page.goto(url, referer=referer) - await page_info.page.wait_for_load_state(state="domcontentloaded") + if params.load_dom: + await page_info.page.wait_for_load_state(state="domcontentloaded") if params.network_idle: await page_info.page.wait_for_load_state("networkidle") @@ -632,7 +649,8 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession): await self._solve_cloudflare(page_info.page) # Make sure the page is fully loaded after the captcha await page_info.page.wait_for_load_state(state="load") - await page_info.page.wait_for_load_state(state="domcontentloaded") + if params.load_dom: + await page_info.page.wait_for_load_state(state="domcontentloaded") if params.network_idle: await page_info.page.wait_for_load_state("networkidle") @@ -648,7 +666,8 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession): await waiter.first.wait_for(state=params.wait_selector_state) # Wait again after waiting for the selector, helpful with protections like Cloudflare await page_info.page.wait_for_load_state(state="load") - await page_info.page.wait_for_load_state(state="domcontentloaded") + if params.load_dom: + await page_info.page.wait_for_load_state(state="domcontentloaded") if params.network_idle: await page_info.page.wait_for_load_state("networkidle") except Exception as e: diff --git a/scrapling/engines/_browsers/_controllers.py b/scrapling/engines/_browsers/_controllers.py index a53fa62..96d7291 100644 --- a/scrapling/engines/_browsers/_controllers.py +++ b/scrapling/engines/_browsers/_controllers.py @@ -54,6 +54,7 @@ class DynamicSession(DynamicSessionMixin, SyncSession): "cookies", "disable_resources", "network_idle", + "load_dom", "wait_selector", "init_script", "wait_selector_state", @@ -93,6 +94,7 @@ class DynamicSession(DynamicSessionMixin, SyncSession): init_script: Optional[str] = None, cookies: Optional[List[Dict]] = None, network_idle: bool = False, + load_dom: bool = True, wait_selector_state: SelectorWaitStates = "attached", selector_config: Optional[Dict] = None, ): @@ -116,6 +118,7 @@ class DynamicSession(DynamicSessionMixin, SyncSession): :param real_chrome: If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch an instance of your browser and use it. :param hide_canvas: Add random noise to canvas operations to prevent fingerprinting. :param disable_webgl: Disables WebGL and WebGL 2.0 support entirely. + :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute. :param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers/NSTBrowser through CDP. :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name. :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ @@ -130,6 +133,7 @@ class DynamicSession(DynamicSessionMixin, SyncSession): stealth=stealth, cdp_url=cdp_url, cookies=cookies, + load_dom=load_dom, headless=headless, useragent=useragent, max_pages=__max_pages, @@ -208,6 +212,7 @@ class DynamicSession(DynamicSessionMixin, SyncSession): wait_selector: Optional[str] = _UNSET, wait_selector_state: SelectorWaitStates = _UNSET, network_idle: bool = _UNSET, + load_dom: bool = _UNSET, selector_config: Optional[Dict] = _UNSET, ) -> Response: """Opens up the browser and do your request based on your chosen options. @@ -224,6 +229,7 @@ class DynamicSession(DynamicSessionMixin, SyncSession): :param wait_selector: Wait for a specific CSS selector to be in a specific state. :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`. :param network_idle: Wait for the page until there are no network connections for at least 500 ms. + :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute. :param selector_config: The arguments that will be passed in the end while creating the final Selector's class. :return: A `Response` object. """ @@ -239,6 +245,7 @@ class DynamicSession(DynamicSessionMixin, SyncSession): wait_selector=self._get_with_precedence(wait_selector, self.wait_selector, _UNSET), wait_selector_state=self._get_with_precedence(wait_selector_state, self.wait_selector_state, _UNSET), network_idle=self._get_with_precedence(network_idle, self.network_idle, _UNSET), + load_dom=self._get_with_precedence(load_dom, self.load_dom, _UNSET), selector_config=self._get_with_precedence(selector_config, self.selector_config, _UNSET), ), PlaywrightConfig, @@ -267,7 +274,8 @@ class DynamicSession(DynamicSessionMixin, SyncSession): # Navigate to URL and wait for a specified state page_info.page.on("response", handle_response) first_response = page_info.page.goto(url, referer=referer) - page_info.page.wait_for_load_state(state="domcontentloaded") + if params.load_dom: + page_info.page.wait_for_load_state(state="domcontentloaded") if params.network_idle: page_info.page.wait_for_load_state("networkidle") @@ -287,7 +295,8 @@ class DynamicSession(DynamicSessionMixin, SyncSession): waiter.first.wait_for(state=params.wait_selector_state) # Wait again after waiting for the selector, helpful with protections like Cloudflare page_info.page.wait_for_load_state(state="load") - page_info.page.wait_for_load_state(state="domcontentloaded") + if params.load_dom: + page_info.page.wait_for_load_state(state="domcontentloaded") if params.network_idle: page_info.page.wait_for_load_state("networkidle") except Exception as e: # pragma: no cover @@ -335,6 +344,7 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession): init_script: Optional[str] = None, cookies: Optional[List[Dict]] = None, network_idle: bool = False, + load_dom: bool = True, wait_selector_state: SelectorWaitStates = "attached", selector_config: Optional[Dict] = None, ): @@ -347,6 +357,7 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession): :param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it. :param cookies: Set cookies for the next request. :param network_idle: Wait for the page until there are no network connections for at least 500 ms. + :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute. :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000 :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object. :param page_action: Added for automation. A function that takes the `page` object and does the automation you need. @@ -374,6 +385,7 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession): stealth=stealth, cdp_url=cdp_url, cookies=cookies, + load_dom=load_dom, headless=headless, useragent=useragent, max_pages=max_pages, @@ -453,6 +465,7 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession): wait_selector: Optional[str] = _UNSET, wait_selector_state: SelectorWaitStates = _UNSET, network_idle: bool = _UNSET, + load_dom: bool = _UNSET, selector_config: Optional[Dict] = _UNSET, ) -> Response: """Opens up the browser and do your request based on your chosen options. @@ -469,6 +482,7 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession): :param wait_selector: Wait for a specific CSS selector to be in a specific state. :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`. :param network_idle: Wait for the page until there are no network connections for at least 500 ms. + :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute. :param selector_config: The arguments that will be passed in the end while creating the final Selector's class. :return: A `Response` object. """ @@ -484,6 +498,7 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession): wait_selector=self._get_with_precedence(wait_selector, self.wait_selector, _UNSET), wait_selector_state=self._get_with_precedence(wait_selector_state, self.wait_selector_state, _UNSET), network_idle=self._get_with_precedence(network_idle, self.network_idle, _UNSET), + load_dom=self._get_with_precedence(load_dom, self.load_dom, _UNSET), selector_config=self._get_with_precedence(selector_config, self.selector_config, _UNSET), ), PlaywrightConfig, @@ -512,7 +527,8 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession): # Navigate to URL and wait for a specified state page_info.page.on("response", handle_response) first_response = await page_info.page.goto(url, referer=referer) - await page_info.page.wait_for_load_state(state="domcontentloaded") + if self.load_dom: + await page_info.page.wait_for_load_state(state="domcontentloaded") if params.network_idle: await page_info.page.wait_for_load_state("networkidle") @@ -532,7 +548,8 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession): await waiter.first.wait_for(state=params.wait_selector_state) # Wait again after waiting for the selector, helpful with protections like Cloudflare await page_info.page.wait_for_load_state(state="load") - await page_info.page.wait_for_load_state(state="domcontentloaded") + if self.load_dom: + await page_info.page.wait_for_load_state(state="domcontentloaded") if params.network_idle: await page_info.page.wait_for_load_state("networkidle") except Exception as e: diff --git a/scrapling/engines/_browsers/_validators.py b/scrapling/engines/_browsers/_validators.py index c34d6fe..a20ae55 100644 --- a/scrapling/engines/_browsers/_validators.py +++ b/scrapling/engines/_browsers/_validators.py @@ -35,6 +35,7 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False): wait_selector: Optional[str] = None cookies: Optional[List[Dict]] = None network_idle: bool = False + load_dom: bool = True wait_selector_state: SelectorWaitStates = "attached" selector_config: Optional[Dict] = None @@ -92,6 +93,7 @@ class CamoufoxConfig(Struct, kw_only=True, frozen=False): block_webrtc: bool = False allow_webgl: bool = True network_idle: bool = False + load_dom: bool = True humanize: bool | float = True solve_cloudflare: bool = False wait: int | float = 0 diff --git a/scrapling/fetchers.py b/scrapling/fetchers.py index fc72de4..2421b42 100644 --- a/scrapling/fetchers.py +++ b/scrapling/fetchers.py @@ -56,6 +56,7 @@ class StealthyFetcher(BaseFetcher): block_webrtc: bool = False, allow_webgl: bool = True, network_idle: bool = False, + load_dom: bool = True, humanize: bool | float = True, solve_cloudflare: bool = False, wait: int | float = 0, @@ -92,6 +93,7 @@ class StealthyFetcher(BaseFetcher): :param solve_cloudflare: Solves all 3 types of the Cloudflare's Turnstile wait page before returning the response to you. :param allow_webgl: Enabled by default. Disabling WebGL is not recommended as many WAFs now check if WebGL is enabled. :param network_idle: Wait for the page until there are no network connections for at least 500 ms. + :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute. :param disable_ads: Disabled by default, this installs the `uBlock Origin` addon on the browser if enabled. :param os_randomize: If enabled, Scrapling will randomize the OS fingerprints used. The default is Scrapling matching the fingerprints with the current OS. :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object. @@ -123,6 +125,7 @@ class StealthyFetcher(BaseFetcher): cookies=cookies, headless=headless, humanize=humanize, + load_dom=load_dom, disable_ads=disable_ads, allow_webgl=allow_webgl, page_action=page_action, @@ -152,6 +155,7 @@ class StealthyFetcher(BaseFetcher): block_webrtc: bool = False, allow_webgl: bool = True, network_idle: bool = False, + load_dom: bool = True, humanize: bool | float = True, solve_cloudflare: bool = False, wait: int | float = 0, @@ -188,6 +192,7 @@ class StealthyFetcher(BaseFetcher): :param solve_cloudflare: Solves all 3 types of the Cloudflare's Turnstile wait page before returning the response to you. :param allow_webgl: Enabled by default. Disabling WebGL is not recommended as many WAFs now check if WebGL is enabled. :param network_idle: Wait for the page until there are no network connections for at least 500 ms. + :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute. :param disable_ads: Disabled by default, this installs the `uBlock Origin` addon on the browser if enabled. :param os_randomize: If enabled, Scrapling will randomize the OS fingerprints used. The default is Scrapling matching the fingerprints with the current OS. :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object. @@ -220,6 +225,7 @@ class StealthyFetcher(BaseFetcher): cookies=cookies, headless=headless, humanize=humanize, + load_dom=load_dom, disable_ads=disable_ads, allow_webgl=allow_webgl, page_action=page_action, @@ -280,6 +286,7 @@ class DynamicFetcher(BaseFetcher): init_script: Optional[str] = None, cookies: Optional[Iterable[Dict]] = None, network_idle: bool = False, + load_dom: bool = True, wait_selector_state: SelectorWaitStates = "attached", custom_config: Optional[Dict] = None, ) -> Response: @@ -293,6 +300,7 @@ class DynamicFetcher(BaseFetcher): :param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it. :param cookies: Set cookies for the next request. :param network_idle: Wait for the page until there are no network connections for at least 500 ms. + :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute. :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000 :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object. :param page_action: Added for automation. A function that takes the `page` object and does the automation you need. @@ -325,6 +333,7 @@ class DynamicFetcher(BaseFetcher): cdp_url=cdp_url, cookies=cookies, headless=headless, + load_dom=load_dom, useragent=useragent, real_chrome=real_chrome, page_action=page_action, @@ -364,6 +373,7 @@ class DynamicFetcher(BaseFetcher): init_script: Optional[str] = None, cookies: Optional[Iterable[Dict]] = None, network_idle: bool = False, + load_dom: bool = True, wait_selector_state: SelectorWaitStates = "attached", custom_config: Optional[Dict] = None, ) -> Response: @@ -377,6 +387,7 @@ class DynamicFetcher(BaseFetcher): :param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it. :param cookies: Set cookies for the next request. :param network_idle: Wait for the page until there are no network connections for at least 500 ms. + :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute. :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000 :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object. :param page_action: Added for automation. A function that takes the `page` object and does the automation you need. @@ -402,6 +413,7 @@ class DynamicFetcher(BaseFetcher): async with AsyncDynamicSession( wait=wait, + max_pages=1, proxy=proxy, locale=locale, timeout=timeout, @@ -409,8 +421,8 @@ class DynamicFetcher(BaseFetcher): cdp_url=cdp_url, cookies=cookies, headless=headless, + load_dom=load_dom, useragent=useragent, - max_pages=1, real_chrome=real_chrome, page_action=page_action, hide_canvas=hide_canvas,