From 9838d741d0def5ca2235b3baf3b0dc4157b1c07a Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Thu, 11 Sep 2025 03:48:55 +0300 Subject: [PATCH] refactor/feat(browser fetchers): make it possible to have a configuration per page in sessions - Also, no need for the `page_action` argument function to return the page again --- scrapling/engines/_browsers/_base.py | 37 +++-- scrapling/engines/_browsers/_camoufox.py | 171 ++++++++++++++++---- scrapling/engines/_browsers/_controllers.py | 149 ++++++++++++++--- 3 files changed, 291 insertions(+), 66 deletions(-) diff --git a/scrapling/engines/_browsers/_base.py b/scrapling/engines/_browsers/_base.py index 55b3310..6b91d8f 100644 --- a/scrapling/engines/_browsers/_base.py +++ b/scrapling/engines/_browsers/_base.py @@ -22,6 +22,7 @@ from ._config_tools import _compiled_stealth_scripts from ._validators import validate, PlaywrightConfig, CamoufoxConfig from ._config_tools import _launch_kwargs, _context_kwargs from scrapling.core._types import ( + Any, Dict, Optional, ) @@ -38,7 +39,12 @@ class SyncSession: self.context: Optional[BrowserContext] = None self._closed = False - def _get_page(self) -> PageInfo: # pragma: no cover + def _get_page( + self, + timeout: int | float, + extra_headers: Optional[Dict[str, str]], + disable_resources: bool, + ) -> PageInfo: # pragma: no cover """Get a new page to use""" # Close all finished pages to ensure clean state @@ -59,13 +65,12 @@ class SyncSession: ) page = self.context.new_page() - timeout = getattr(self, "timeout", 30000) page.set_default_navigation_timeout(timeout) page.set_default_timeout(timeout) - if getattr(self, "extra_headers", False): - page.set_extra_http_headers(getattr(self, "extra_headers")) + if extra_headers: + page.set_extra_http_headers(extra_headers) - if getattr(self, "disable_resources", False): + if disable_resources: page.route("**/*", intercept_route) if getattr(self, "stealth", False): @@ -74,6 +79,13 @@ class SyncSession: return self.page_pool.add_page(page) + @staticmethod + def _get_with_precedence( + request_value: Any, session_value: Any, sentinel_value: object + ) -> Any: + """Get value with request-level priority over session-level""" + return request_value if request_value is not sentinel_value else session_value + def get_pool_stats(self) -> Dict[str, int]: """Get statistics about the current page pool""" return { @@ -90,7 +102,12 @@ class AsyncSession(SyncSession): self.context: Optional[AsyncBrowserContext] = None self._lock = Lock() - async def _get_page(self) -> PageInfo: # pragma: no cover + async def _get_page( + self, + timeout: int | float, + extra_headers: Optional[Dict[str, str]], + disable_resources: bool, + ) -> PageInfo: # pragma: no cover """Get a new page to use""" async with self._lock: # Close all finished pages to ensure clean state @@ -111,13 +128,12 @@ class AsyncSession(SyncSession): ) page = await self.context.new_page() - timeout = getattr(self, "timeout", 30000) page.set_default_navigation_timeout(timeout) page.set_default_timeout(timeout) - if getattr(self, "extra_headers", False): - await page.set_extra_http_headers(getattr(self, "extra_headers")) + if extra_headers: + await page.set_extra_http_headers(extra_headers) - if getattr(self, "disable_resources", False): + if disable_resources: await page.route("**/*", async_intercept_route) if getattr(self, "stealth", False): @@ -334,7 +350,6 @@ class StealthySessionMixin: self.geoip = config.geoip self.selector_config = config.selector_config self.additional_args = config.additional_args - self.selector_config = config.selector_config self.page_action = config.page_action self._headers_keys = ( set(map(str.lower, self.extra_headers.keys())) diff --git a/scrapling/engines/_browsers/_camoufox.py b/scrapling/engines/_browsers/_camoufox.py index cd814c1..926191a 100644 --- a/scrapling/engines/_browsers/_camoufox.py +++ b/scrapling/engines/_browsers/_camoufox.py @@ -31,6 +31,7 @@ from scrapling.engines.toolbelt import ( ) __CF_PATTERN__ = re_compile("challenges.cloudflare.com/cdn-cgi/challenge-platform/.*") +_UNSET = object() class StealthySession(StealthySessionMixin, SyncSession): @@ -247,19 +248,74 @@ class StealthySession(StealthySessionMixin, SyncSession): log.info("Cloudflare captcha is solved") return - def fetch(self, url: str) -> Response: + def fetch( + self, + url: str, + google_search: bool = _UNSET, + timeout: int | float = _UNSET, + wait: int | float = _UNSET, + page_action: Optional[Callable] = _UNSET, + extra_headers: Optional[Dict[str, str]] = _UNSET, + disable_resources: bool = _UNSET, + wait_selector: Optional[str] = _UNSET, + wait_selector_state: SelectorWaitStates = _UNSET, + network_idle: bool = _UNSET, + solve_cloudflare: bool = _UNSET, + selector_config: Optional[Dict] = _UNSET, + ) -> Response: """Opens up the browser and do your request based on your chosen options. :param url: The Target url. + :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name. + :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000 + :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object. + :param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again. + :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ + :param disable_resources: Drop requests of unnecessary resources for a speed boost. It depends, but it made requests ~25% faster in my tests for some websites. + Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`. + This can help save your proxy usage but be careful with this option as it makes some websites never finish loading. + :param wait_selector: Wait for a specific CSS selector to be in a specific state. + :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`. + :param network_idle: Wait for the page until there are no network connections for at least 500 ms. + :param solve_cloudflare: Solves all 3 types of the Cloudflare's Turnstile wait page before returning the response to you. + :param selector_config: The arguments that will be passed in the end while creating the final Selector's class. :return: A `Response` object. """ + google_search = self._get_with_precedence( + google_search, self.google_search, _UNSET + ) + timeout = self._get_with_precedence(timeout, self.timeout, _UNSET) + wait = self._get_with_precedence(wait, self.wait, _UNSET) + page_action = self._get_with_precedence(page_action, self.page_action, _UNSET) + extra_headers = self._get_with_precedence( + extra_headers, self.extra_headers, _UNSET + ) + disable_resources = self._get_with_precedence( + disable_resources, self.disable_resources, _UNSET + ) + wait_selector = self._get_with_precedence( + wait_selector, self.wait_selector, _UNSET + ) + wait_selector_state = self._get_with_precedence( + wait_selector_state, self.wait_selector_state, _UNSET + ) + network_idle = self._get_with_precedence( + network_idle, self.network_idle, _UNSET + ) + solve_cloudflare = self._get_with_precedence( + solve_cloudflare, self.solve_cloudflare, _UNSET + ) + selector_config = self._get_with_precedence( + selector_config, self.selector_config, _UNSET + ) + if self._closed: # pragma: no cover raise RuntimeError("Context manager has been closed") final_response = None referer = ( generate_convincing_referer(url) - if (self.google_search and "referer" not in self._headers_keys) + if (google_search and "referer" not in self._headers_keys) else None ) @@ -271,7 +327,7 @@ class StealthySession(StealthySessionMixin, SyncSession): ): final_response = finished_response - page_info = self._get_page() + page_info = self._get_page(timeout, extra_headers, disable_resources) page_info.mark_busy(url=url) try: # pragma: no cover @@ -280,41 +336,41 @@ class StealthySession(StealthySessionMixin, SyncSession): first_response = page_info.page.goto(url, referer=referer) page_info.page.wait_for_load_state(state="domcontentloaded") - if self.network_idle: + if network_idle: page_info.page.wait_for_load_state("networkidle") if not first_response: raise RuntimeError(f"Failed to get response for {url}") - if self.solve_cloudflare: + if solve_cloudflare: self._solve_cloudflare(page_info.page) # Make sure the page is fully loaded after the captcha page_info.page.wait_for_load_state(state="load") page_info.page.wait_for_load_state(state="domcontentloaded") - if self.network_idle: + if network_idle: page_info.page.wait_for_load_state("networkidle") - if self.page_action is not None: + if page_action is not None: try: - page_info.page = self.page_action(page_info.page) + _ = page_action(page_info.page) except Exception as e: log.error(f"Error executing page_action: {e}") - if self.wait_selector: + if wait_selector: try: - waiter: Locator = page_info.page.locator(self.wait_selector) - waiter.first.wait_for(state=self.wait_selector_state) + waiter: Locator = page_info.page.locator(wait_selector) + waiter.first.wait_for(state=wait_selector_state) # Wait again after waiting for the selector, helpful with protections like Cloudflare page_info.page.wait_for_load_state(state="load") page_info.page.wait_for_load_state(state="domcontentloaded") - if self.network_idle: + if network_idle: page_info.page.wait_for_load_state("networkidle") except Exception as e: - log.error(f"Error waiting for selector {self.wait_selector}: {e}") + log.error(f"Error waiting for selector {wait_selector}: {e}") - page_info.page.wait_for_timeout(self.wait) + page_info.page.wait_for_timeout(wait) response = ResponseFactory.from_playwright_response( - page_info.page, first_response, final_response, self.selector_config + page_info.page, first_response, final_response, selector_config ) # Mark the page as finished for next use @@ -508,19 +564,74 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession): log.info("Cloudflare captcha is solved") return - async def fetch(self, url: str) -> Response: + async def fetch( + self, + url: str, + google_search: bool = _UNSET, + timeout: int | float = _UNSET, + wait: int | float = _UNSET, + page_action: Optional[Callable] = _UNSET, + extra_headers: Optional[Dict[str, str]] = _UNSET, + disable_resources: bool = _UNSET, + wait_selector: Optional[str] = _UNSET, + wait_selector_state: SelectorWaitStates = _UNSET, + network_idle: bool = _UNSET, + solve_cloudflare: bool = _UNSET, + selector_config: Optional[Dict] = _UNSET, + ) -> Response: """Opens up the browser and do your request based on your chosen options. :param url: The Target url. + :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name. + :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000 + :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object. + :param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again. + :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ + :param disable_resources: Drop requests of unnecessary resources for a speed boost. It depends, but it made requests ~25% faster in my tests for some websites. + Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`. + This can help save your proxy usage but be careful with this option as it makes some websites never finish loading. + :param wait_selector: Wait for a specific CSS selector to be in a specific state. + :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`. + :param network_idle: Wait for the page until there are no network connections for at least 500 ms. + :param solve_cloudflare: Solves all 3 types of the Cloudflare's Turnstile wait page before returning the response to you. + :param selector_config: The arguments that will be passed in the end while creating the final Selector's class. :return: A `Response` object. """ + google_search = self._get_with_precedence( + google_search, self.google_search, _UNSET + ) + timeout = self._get_with_precedence(timeout, self.timeout, _UNSET) + wait = self._get_with_precedence(wait, self.wait, _UNSET) + page_action = self._get_with_precedence(page_action, self.page_action, _UNSET) + extra_headers = self._get_with_precedence( + extra_headers, self.extra_headers, _UNSET + ) + disable_resources = self._get_with_precedence( + disable_resources, self.disable_resources, _UNSET + ) + wait_selector = self._get_with_precedence( + wait_selector, self.wait_selector, _UNSET + ) + wait_selector_state = self._get_with_precedence( + wait_selector_state, self.wait_selector_state, _UNSET + ) + network_idle = self._get_with_precedence( + network_idle, self.network_idle, _UNSET + ) + solve_cloudflare = self._get_with_precedence( + solve_cloudflare, self.solve_cloudflare, _UNSET + ) + selector_config = self._get_with_precedence( + selector_config, self.selector_config, _UNSET + ) + if self._closed: # pragma: no cover raise RuntimeError("Context manager has been closed") final_response = None referer = ( generate_convincing_referer(url) - if (self.google_search and "referer" not in self._headers_keys) + if (google_search and "referer" not in self._headers_keys) else None ) @@ -532,7 +643,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession): ): final_response = finished_response - page_info = await self._get_page() + page_info = await self._get_page(timeout, extra_headers, disable_resources) page_info.mark_busy(url=url) try: @@ -541,43 +652,43 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession): first_response = await page_info.page.goto(url, referer=referer) await page_info.page.wait_for_load_state(state="domcontentloaded") - if self.network_idle: + if network_idle: await page_info.page.wait_for_load_state("networkidle") if not first_response: raise RuntimeError(f"Failed to get response for {url}") - if self.solve_cloudflare: + if solve_cloudflare: await self._solve_cloudflare(page_info.page) # Make sure the page is fully loaded after the captcha await page_info.page.wait_for_load_state(state="load") await page_info.page.wait_for_load_state(state="domcontentloaded") - if self.network_idle: + if network_idle: await page_info.page.wait_for_load_state("networkidle") - if self.page_action is not None: + if page_action is not None: try: - page_info.page = await self.page_action(page_info.page) + _ = await page_action(page_info.page) except Exception as e: log.error(f"Error executing page_action: {e}") - if self.wait_selector: + if wait_selector: try: - waiter: AsyncLocator = page_info.page.locator(self.wait_selector) - await waiter.first.wait_for(state=self.wait_selector_state) + waiter: AsyncLocator = page_info.page.locator(wait_selector) + await waiter.first.wait_for(state=wait_selector_state) # Wait again after waiting for the selector, helpful with protections like Cloudflare await page_info.page.wait_for_load_state(state="load") await page_info.page.wait_for_load_state(state="domcontentloaded") - if self.network_idle: + if network_idle: await page_info.page.wait_for_load_state("networkidle") except Exception as e: - log.error(f"Error waiting for selector {self.wait_selector}: {e}") + log.error(f"Error waiting for selector {wait_selector}: {e}") - await page_info.page.wait_for_timeout(self.wait) + await page_info.page.wait_for_timeout(wait) # Create response object response = await ResponseFactory.from_async_playwright_response( - page_info.page, first_response, final_response, self.selector_config + page_info.page, first_response, final_response, selector_config ) # Mark the page as finished for next use diff --git a/scrapling/engines/_browsers/_controllers.py b/scrapling/engines/_browsers/_controllers.py index ef3b906..04a9d35 100644 --- a/scrapling/engines/_browsers/_controllers.py +++ b/scrapling/engines/_browsers/_controllers.py @@ -31,6 +31,8 @@ from scrapling.engines.toolbelt import ( generate_convincing_referer, ) +_UNSET = object() + class DynamicSession(DynamicSessionMixin, SyncSession): """A Browser session manager with page pooling.""" @@ -198,19 +200,66 @@ class DynamicSession(DynamicSessionMixin, SyncSession): def fetch( self, url: str, + google_search: bool = _UNSET, + timeout: int | float = _UNSET, + wait: int | float = _UNSET, + page_action: Optional[Callable] = _UNSET, + extra_headers: Optional[Dict[str, str]] = _UNSET, + disable_resources: bool = _UNSET, + wait_selector: Optional[str] = _UNSET, + wait_selector_state: SelectorWaitStates = _UNSET, + network_idle: bool = _UNSET, + selector_config: Optional[Dict] = _UNSET, ) -> Response: """Opens up the browser and do your request based on your chosen options. :param url: The Target url. + :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name. + :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000 + :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object. + :param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again. + :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ + :param disable_resources: Drop requests of unnecessary resources for a speed boost. It depends, but it made requests ~25% faster in my tests for some websites. + Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`. + This can help save your proxy usage but be careful with this option as it makes some websites never finish loading. + :param wait_selector: Wait for a specific CSS selector to be in a specific state. + :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`. + :param network_idle: Wait for the page until there are no network connections for at least 500 ms. + :param selector_config: The arguments that will be passed in the end while creating the final Selector's class. :return: A `Response` object. """ + google_search = self._get_with_precedence( + google_search, self.google_search, _UNSET + ) + timeout = self._get_with_precedence(timeout, self.timeout, _UNSET) + wait = self._get_with_precedence(wait, self.wait, _UNSET) + page_action = self._get_with_precedence(page_action, self.page_action, _UNSET) + extra_headers = self._get_with_precedence( + extra_headers, self.extra_headers, _UNSET + ) + disable_resources = self._get_with_precedence( + disable_resources, self.disable_resources, _UNSET + ) + wait_selector = self._get_with_precedence( + wait_selector, self.wait_selector, _UNSET + ) + wait_selector_state = self._get_with_precedence( + wait_selector_state, self.wait_selector_state, _UNSET + ) + network_idle = self._get_with_precedence( + network_idle, self.network_idle, _UNSET + ) + selector_config = self._get_with_precedence( + selector_config, self.selector_config, _UNSET + ) + if self._closed: # pragma: no cover raise RuntimeError("Context manager has been closed") final_response = None referer = ( generate_convincing_referer(url) - if (self.google_search and "referer" not in self._headers_keys) + if (google_search and "referer" not in self._headers_keys) else None ) @@ -222,7 +271,7 @@ class DynamicSession(DynamicSessionMixin, SyncSession): ): final_response = finished_response - page_info = self._get_page() + page_info = self._get_page(timeout, extra_headers, disable_resources) page_info.mark_busy(url=url) try: # pragma: no cover @@ -231,35 +280,35 @@ class DynamicSession(DynamicSessionMixin, SyncSession): first_response = page_info.page.goto(url, referer=referer) page_info.page.wait_for_load_state(state="domcontentloaded") - if self.network_idle: + if network_idle: page_info.page.wait_for_load_state("networkidle") if not first_response: raise RuntimeError(f"Failed to get response for {url}") - if self.page_action is not None: + if page_action is not None: try: - page_info.page = self.page_action(page_info.page) + _ = page_action(page_info.page) except Exception as e: # pragma: no cover log.error(f"Error executing page_action: {e}") - if self.wait_selector: + if wait_selector: try: - waiter: Locator = page_info.page.locator(self.wait_selector) - waiter.first.wait_for(state=self.wait_selector_state) + waiter: Locator = page_info.page.locator(wait_selector) + waiter.first.wait_for(state=wait_selector_state) # Wait again after waiting for the selector, helpful with protections like Cloudflare page_info.page.wait_for_load_state(state="load") page_info.page.wait_for_load_state(state="domcontentloaded") - if self.network_idle: + if network_idle: page_info.page.wait_for_load_state("networkidle") except Exception as e: # pragma: no cover - log.error(f"Error waiting for selector {self.wait_selector}: {e}") + log.error(f"Error waiting for selector {wait_selector}: {e}") - page_info.page.wait_for_timeout(self.wait) + page_info.page.wait_for_timeout(wait) # Create response object response = ResponseFactory.from_playwright_response( - page_info.page, first_response, final_response, self.selector_config + page_info.page, first_response, final_response, selector_config ) # Mark the page as finished for next use @@ -409,19 +458,69 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession): self._closed = True - async def fetch(self, url: str) -> Response: + async def fetch( + self, + url: str, + google_search: bool = _UNSET, + timeout: int | float = _UNSET, + wait: int | float = _UNSET, + page_action: Optional[Callable] = _UNSET, + extra_headers: Optional[Dict[str, str]] = _UNSET, + disable_resources: bool = _UNSET, + wait_selector: Optional[str] = _UNSET, + wait_selector_state: SelectorWaitStates = _UNSET, + network_idle: bool = _UNSET, + selector_config: Optional[Dict] = _UNSET, + ) -> Response: """Opens up the browser and do your request based on your chosen options. :param url: The Target url. + :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name. + :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000 + :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object. + :param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again. + :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ + :param disable_resources: Drop requests of unnecessary resources for a speed boost. It depends, but it made requests ~25% faster in my tests for some websites. + Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`. + This can help save your proxy usage but be careful with this option as it makes some websites never finish loading. + :param wait_selector: Wait for a specific CSS selector to be in a specific state. + :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`. + :param network_idle: Wait for the page until there are no network connections for at least 500 ms. + :param selector_config: The arguments that will be passed in the end while creating the final Selector's class. :return: A `Response` object. """ + google_search = self._get_with_precedence( + google_search, self.google_search, _UNSET + ) + timeout = self._get_with_precedence(timeout, self.timeout, _UNSET) + wait = self._get_with_precedence(wait, self.wait, _UNSET) + page_action = self._get_with_precedence(page_action, self.page_action, _UNSET) + extra_headers = self._get_with_precedence( + extra_headers, self.extra_headers, _UNSET + ) + disable_resources = self._get_with_precedence( + disable_resources, self.disable_resources, _UNSET + ) + wait_selector = self._get_with_precedence( + wait_selector, self.wait_selector, _UNSET + ) + wait_selector_state = self._get_with_precedence( + wait_selector_state, self.wait_selector_state, _UNSET + ) + network_idle = self._get_with_precedence( + network_idle, self.network_idle, _UNSET + ) + selector_config = self._get_with_precedence( + selector_config, self.selector_config, _UNSET + ) + if self._closed: # pragma: no cover raise RuntimeError("Context manager has been closed") final_response = None referer = ( generate_convincing_referer(url) - if (self.google_search and "referer" not in self._headers_keys) + if (google_search and "referer" not in self._headers_keys) else None ) @@ -433,7 +532,7 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession): ): final_response = finished_response - page_info = await self._get_page() + page_info = await self._get_page(timeout, extra_headers, disable_resources) page_info.mark_busy(url=url) try: @@ -442,35 +541,35 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession): first_response = await page_info.page.goto(url, referer=referer) await page_info.page.wait_for_load_state(state="domcontentloaded") - if self.network_idle: + if network_idle: await page_info.page.wait_for_load_state("networkidle") if not first_response: raise RuntimeError(f"Failed to get response for {url}") - if self.page_action is not None: + if page_action is not None: try: - page_info.page = await self.page_action(page_info.page) + _ = await page_action(page_info.page) except Exception as e: log.error(f"Error executing page_action: {e}") - if self.wait_selector: + if wait_selector: try: - waiter: AsyncLocator = page_info.page.locator(self.wait_selector) - await waiter.first.wait_for(state=self.wait_selector_state) + waiter: AsyncLocator = page_info.page.locator(wait_selector) + await waiter.first.wait_for(state=wait_selector_state) # Wait again after waiting for the selector, helpful with protections like Cloudflare await page_info.page.wait_for_load_state(state="load") await page_info.page.wait_for_load_state(state="domcontentloaded") - if self.network_idle: + if network_idle: await page_info.page.wait_for_load_state("networkidle") except Exception as e: - log.error(f"Error waiting for selector {self.wait_selector}: {e}") + log.error(f"Error waiting for selector {wait_selector}: {e}") - await page_info.page.wait_for_timeout(self.wait) + await page_info.page.wait_for_timeout(wait) # Create response object response = await ResponseFactory.from_async_playwright_response( - page_info.page, first_response, final_response, self.selector_config + page_info.page, first_response, final_response, selector_config ) # Mark the page as finished for next use