From 60be9dc816b5bf82adf83534c712c28156f9a17f Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Sat, 13 Sep 2025 03:22:08 +0300 Subject: [PATCH] feat: Validate all fetch-level parameters So it's validated like the session-level ones. Needs improvement, but that's for later --- scrapling/engines/_browsers/_camoufox.py | 160 ++++++++------------ scrapling/engines/_browsers/_controllers.py | 153 ++++++++----------- 2 files changed, 128 insertions(+), 185 deletions(-) diff --git a/scrapling/engines/_browsers/_camoufox.py b/scrapling/engines/_browsers/_camoufox.py index 3412fd6..fcb77ac 100644 --- a/scrapling/engines/_browsers/_camoufox.py +++ b/scrapling/engines/_browsers/_camoufox.py @@ -15,6 +15,7 @@ from playwright.async_api import ( Page as async_Page, ) +from ._validators import validate, CamoufoxConfig from ._base import SyncSession, AsyncSession, StealthySessionMixin from scrapling.core.utils import log from scrapling.core._types import ( @@ -164,10 +165,8 @@ class StealthySession(StealthySessionMixin, SyncSession): def __create__(self): """Create a browser for this instance and context.""" self.playwright = sync_playwright().start() - self.context = ( - self.playwright.firefox.launch_persistent_context( # pragma: no cover - **self.launch_options - ) + self.context = self.playwright.firefox.launch_persistent_context( # pragma: no cover + **self.launch_options ) # Get the default page and close it @@ -281,32 +280,22 @@ class StealthySession(StealthySessionMixin, SyncSession): :param selector_config: The arguments that will be passed in the end while creating the final Selector's class. :return: A `Response` object. """ - google_search = self._get_with_precedence( - google_search, self.google_search, _UNSET - ) - timeout = self._get_with_precedence(timeout, self.timeout, _UNSET) - wait = self._get_with_precedence(wait, self.wait, _UNSET) - page_action = self._get_with_precedence(page_action, self.page_action, _UNSET) - extra_headers = self._get_with_precedence( - extra_headers, self.extra_headers, _UNSET - ) - disable_resources = self._get_with_precedence( - disable_resources, self.disable_resources, _UNSET - ) - wait_selector = self._get_with_precedence( - wait_selector, self.wait_selector, _UNSET - ) - wait_selector_state = self._get_with_precedence( - wait_selector_state, self.wait_selector_state, _UNSET - ) - network_idle = self._get_with_precedence( - network_idle, self.network_idle, _UNSET - ) - solve_cloudflare = self._get_with_precedence( - solve_cloudflare, self.solve_cloudflare, _UNSET - ) - selector_config = self._get_with_precedence( - selector_config, self.selector_config, _UNSET + # Validate all resolved parameters + params = validate( + dict( + google_search=self._get_with_precedence(google_search, self.google_search, _UNSET), + timeout=self._get_with_precedence(timeout, self.timeout, _UNSET), + wait=self._get_with_precedence(wait, self.wait, _UNSET), + page_action=self._get_with_precedence(page_action, self.page_action, _UNSET), + extra_headers=self._get_with_precedence(extra_headers, self.extra_headers, _UNSET), + disable_resources=self._get_with_precedence(disable_resources, self.disable_resources, _UNSET), + wait_selector=self._get_with_precedence(wait_selector, self.wait_selector, _UNSET), + wait_selector_state=self._get_with_precedence(wait_selector_state, self.wait_selector_state, _UNSET), + network_idle=self._get_with_precedence(network_idle, self.network_idle, _UNSET), + solve_cloudflare=self._get_with_precedence(solve_cloudflare, self.solve_cloudflare, _UNSET), + selector_config=self._get_with_precedence(selector_config, self.selector_config, _UNSET), + ), + CamoufoxConfig, ) if self._closed: # pragma: no cover @@ -314,9 +303,7 @@ class StealthySession(StealthySessionMixin, SyncSession): final_response = None referer = ( - generate_convincing_referer(url) - if (google_search and "referer" not in self._headers_keys) - else None + generate_convincing_referer(url) if (params.google_search and "referer" not in self._headers_keys) else None ) def handle_response(finished_response: SyncPlaywrightResponse): @@ -327,7 +314,7 @@ class StealthySession(StealthySessionMixin, SyncSession): ): final_response = finished_response - page_info = self._get_page(timeout, extra_headers, disable_resources) + page_info = self._get_page(params.timeout, params.extra_headers, params.disable_resources) page_info.mark_busy(url=url) try: # pragma: no cover @@ -336,41 +323,41 @@ class StealthySession(StealthySessionMixin, SyncSession): first_response = page_info.page.goto(url, referer=referer) page_info.page.wait_for_load_state(state="domcontentloaded") - if network_idle: + if params.network_idle: page_info.page.wait_for_load_state("networkidle") if not first_response: raise RuntimeError(f"Failed to get response for {url}") - if solve_cloudflare: + if params.solve_cloudflare: self._solve_cloudflare(page_info.page) # Make sure the page is fully loaded after the captcha page_info.page.wait_for_load_state(state="load") page_info.page.wait_for_load_state(state="domcontentloaded") - if network_idle: + if params.network_idle: page_info.page.wait_for_load_state("networkidle") - if page_action is not None: + if params.page_action: try: - _ = page_action(page_info.page) + _ = params.page_action(page_info.page) except Exception as e: log.error(f"Error executing page_action: {e}") - if wait_selector: + if params.wait_selector: try: - waiter: Locator = page_info.page.locator(wait_selector) - waiter.first.wait_for(state=wait_selector_state) + waiter: Locator = page_info.page.locator(params.wait_selector) + waiter.first.wait_for(state=params.wait_selector_state) # Wait again after waiting for the selector, helpful with protections like Cloudflare page_info.page.wait_for_load_state(state="load") page_info.page.wait_for_load_state(state="domcontentloaded") - if network_idle: + if params.network_idle: page_info.page.wait_for_load_state("networkidle") except Exception as e: - log.error(f"Error waiting for selector {wait_selector}: {e}") + log.error(f"Error waiting for selector {params.wait_selector}: {e}") - page_info.page.wait_for_timeout(wait) + page_info.page.wait_for_timeout(params.wait) response = ResponseFactory.from_playwright_response( - page_info.page, first_response, final_response, selector_config + page_info.page, first_response, final_response, params.selector_config ) # Mark the page as finished for next use @@ -478,10 +465,8 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession): async def __create__(self): """Create a browser for this instance and context.""" self.playwright: AsyncPlaywright = await async_playwright().start() - self.context: AsyncBrowserContext = ( - await self.playwright.firefox.launch_persistent_context( - **self.launch_options - ) + self.context: AsyncBrowserContext = await self.playwright.firefox.launch_persistent_context( + **self.launch_options ) # Get the default page and close it @@ -551,9 +536,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession): await page.wait_for_timeout(500) # Calculate the Captcha coordinates for any viewport - outer_box = await page.locator( - ".main-content p+div>div>div" - ).bounding_box() + outer_box = await page.locator(".main-content p+div>div>div").bounding_box() captcha_x, captcha_y = outer_box["x"] + 26, outer_box["y"] + 25 # Move the mouse to the center of the window, then press and hold the left mouse button @@ -597,32 +580,21 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession): :param selector_config: The arguments that will be passed in the end while creating the final Selector's class. :return: A `Response` object. """ - google_search = self._get_with_precedence( - google_search, self.google_search, _UNSET - ) - timeout = self._get_with_precedence(timeout, self.timeout, _UNSET) - wait = self._get_with_precedence(wait, self.wait, _UNSET) - page_action = self._get_with_precedence(page_action, self.page_action, _UNSET) - extra_headers = self._get_with_precedence( - extra_headers, self.extra_headers, _UNSET - ) - disable_resources = self._get_with_precedence( - disable_resources, self.disable_resources, _UNSET - ) - wait_selector = self._get_with_precedence( - wait_selector, self.wait_selector, _UNSET - ) - wait_selector_state = self._get_with_precedence( - wait_selector_state, self.wait_selector_state, _UNSET - ) - network_idle = self._get_with_precedence( - network_idle, self.network_idle, _UNSET - ) - solve_cloudflare = self._get_with_precedence( - solve_cloudflare, self.solve_cloudflare, _UNSET - ) - selector_config = self._get_with_precedence( - selector_config, self.selector_config, _UNSET + params = validate( + dict( + google_search=self._get_with_precedence(google_search, self.google_search, _UNSET), + timeout=self._get_with_precedence(timeout, self.timeout, _UNSET), + wait=self._get_with_precedence(wait, self.wait, _UNSET), + page_action=self._get_with_precedence(page_action, self.page_action, _UNSET), + extra_headers=self._get_with_precedence(extra_headers, self.extra_headers, _UNSET), + disable_resources=self._get_with_precedence(disable_resources, self.disable_resources, _UNSET), + wait_selector=self._get_with_precedence(wait_selector, self.wait_selector, _UNSET), + wait_selector_state=self._get_with_precedence(wait_selector_state, self.wait_selector_state, _UNSET), + network_idle=self._get_with_precedence(network_idle, self.network_idle, _UNSET), + solve_cloudflare=self._get_with_precedence(solve_cloudflare, self.solve_cloudflare, _UNSET), + selector_config=self._get_with_precedence(selector_config, self.selector_config, _UNSET), + ), + CamoufoxConfig, ) if self._closed: # pragma: no cover @@ -630,9 +602,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession): final_response = None referer = ( - generate_convincing_referer(url) - if (google_search and "referer" not in self._headers_keys) - else None + generate_convincing_referer(url) if (params.google_search and "referer" not in self._headers_keys) else None ) async def handle_response(finished_response: AsyncPlaywrightResponse): @@ -643,7 +613,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession): ): final_response = finished_response - page_info = await self._get_page(timeout, extra_headers, disable_resources) + page_info = await self._get_page(params.timeout, params.extra_headers, params.disable_resources) page_info.mark_busy(url=url) try: @@ -652,43 +622,43 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession): first_response = await page_info.page.goto(url, referer=referer) await page_info.page.wait_for_load_state(state="domcontentloaded") - if network_idle: + if params.network_idle: await page_info.page.wait_for_load_state("networkidle") if not first_response: raise RuntimeError(f"Failed to get response for {url}") - if solve_cloudflare: + if params.solve_cloudflare: await self._solve_cloudflare(page_info.page) # Make sure the page is fully loaded after the captcha await page_info.page.wait_for_load_state(state="load") await page_info.page.wait_for_load_state(state="domcontentloaded") - if network_idle: + if params.network_idle: await page_info.page.wait_for_load_state("networkidle") - if page_action is not None: + if params.page_action: try: - _ = await page_action(page_info.page) + _ = await params.page_action(page_info.page) except Exception as e: log.error(f"Error executing page_action: {e}") - if wait_selector: + if params.wait_selector: try: - waiter: AsyncLocator = page_info.page.locator(wait_selector) - await waiter.first.wait_for(state=wait_selector_state) + waiter: AsyncLocator = page_info.page.locator(params.wait_selector) + await waiter.first.wait_for(state=params.wait_selector_state) # Wait again after waiting for the selector, helpful with protections like Cloudflare await page_info.page.wait_for_load_state(state="load") await page_info.page.wait_for_load_state(state="domcontentloaded") - if network_idle: + if params.network_idle: await page_info.page.wait_for_load_state("networkidle") except Exception as e: - log.error(f"Error waiting for selector {wait_selector}: {e}") + log.error(f"Error waiting for selector {params.wait_selector}: {e}") - await page_info.page.wait_for_timeout(wait) + await page_info.page.wait_for_timeout(params.wait) # Create response object response = await ResponseFactory.from_async_playwright_response( - page_info.page, first_response, final_response, selector_config + page_info.page, first_response, final_response, params.selector_config ) # Mark the page as finished for next use diff --git a/scrapling/engines/_browsers/_controllers.py b/scrapling/engines/_browsers/_controllers.py index 49f6c9d..a53fa62 100644 --- a/scrapling/engines/_browsers/_controllers.py +++ b/scrapling/engines/_browsers/_controllers.py @@ -18,6 +18,7 @@ from rebrowser_playwright.async_api import ( from scrapling.core.utils import log from ._base import SyncSession, AsyncSession, DynamicSessionMixin +from ._validators import validate, PlaywrightConfig from scrapling.core._types import ( Dict, List, @@ -157,13 +158,11 @@ class DynamicSession(DynamicSessionMixin, SyncSession): self.playwright: Playwright = sync_context().start() if self.cdp_url: # pragma: no cover - self.context = self.playwright.chromium.connect_over_cdp( - endpoint_url=self.cdp_url - ).new_context(**self.context_options) - else: - self.context = self.playwright.chromium.launch_persistent_context( - user_data_dir="", **self.launch_options + self.context = self.playwright.chromium.connect_over_cdp(endpoint_url=self.cdp_url).new_context( + **self.context_options ) + else: + self.context = self.playwright.chromium.launch_persistent_context(user_data_dir="", **self.launch_options) # Get the default page and close it default_page = self.context.pages[0] @@ -228,29 +227,21 @@ class DynamicSession(DynamicSessionMixin, SyncSession): :param selector_config: The arguments that will be passed in the end while creating the final Selector's class. :return: A `Response` object. """ - google_search = self._get_with_precedence( - google_search, self.google_search, _UNSET - ) - timeout = self._get_with_precedence(timeout, self.timeout, _UNSET) - wait = self._get_with_precedence(wait, self.wait, _UNSET) - page_action = self._get_with_precedence(page_action, self.page_action, _UNSET) - extra_headers = self._get_with_precedence( - extra_headers, self.extra_headers, _UNSET - ) - disable_resources = self._get_with_precedence( - disable_resources, self.disable_resources, _UNSET - ) - wait_selector = self._get_with_precedence( - wait_selector, self.wait_selector, _UNSET - ) - wait_selector_state = self._get_with_precedence( - wait_selector_state, self.wait_selector_state, _UNSET - ) - network_idle = self._get_with_precedence( - network_idle, self.network_idle, _UNSET - ) - selector_config = self._get_with_precedence( - selector_config, self.selector_config, _UNSET + # Validate all resolved parameters + params = validate( + dict( + google_search=self._get_with_precedence(google_search, self.google_search, _UNSET), + timeout=self._get_with_precedence(timeout, self.timeout, _UNSET), + wait=self._get_with_precedence(wait, self.wait, _UNSET), + page_action=self._get_with_precedence(page_action, self.page_action, _UNSET), + extra_headers=self._get_with_precedence(extra_headers, self.extra_headers, _UNSET), + disable_resources=self._get_with_precedence(disable_resources, self.disable_resources, _UNSET), + wait_selector=self._get_with_precedence(wait_selector, self.wait_selector, _UNSET), + wait_selector_state=self._get_with_precedence(wait_selector_state, self.wait_selector_state, _UNSET), + network_idle=self._get_with_precedence(network_idle, self.network_idle, _UNSET), + selector_config=self._get_with_precedence(selector_config, self.selector_config, _UNSET), + ), + PlaywrightConfig, ) if self._closed: # pragma: no cover @@ -258,9 +249,7 @@ class DynamicSession(DynamicSessionMixin, SyncSession): final_response = None referer = ( - generate_convincing_referer(url) - if (google_search and "referer" not in self._headers_keys) - else None + generate_convincing_referer(url) if (params.google_search and "referer" not in self._headers_keys) else None ) def handle_response(finished_response: SyncPlaywrightResponse): @@ -271,7 +260,7 @@ class DynamicSession(DynamicSessionMixin, SyncSession): ): final_response = finished_response - page_info = self._get_page(timeout, extra_headers, disable_resources) + page_info = self._get_page(params.timeout, params.extra_headers, params.disable_resources) page_info.mark_busy(url=url) try: # pragma: no cover @@ -280,35 +269,35 @@ class DynamicSession(DynamicSessionMixin, SyncSession): first_response = page_info.page.goto(url, referer=referer) page_info.page.wait_for_load_state(state="domcontentloaded") - if network_idle: + if params.network_idle: page_info.page.wait_for_load_state("networkidle") if not first_response: raise RuntimeError(f"Failed to get response for {url}") - if page_action is not None: + if params.page_action: try: - _ = page_action(page_info.page) + _ = params.page_action(page_info.page) except Exception as e: # pragma: no cover log.error(f"Error executing page_action: {e}") - if wait_selector: + if params.wait_selector: try: - waiter: Locator = page_info.page.locator(wait_selector) - waiter.first.wait_for(state=wait_selector_state) + waiter: Locator = page_info.page.locator(params.wait_selector) + waiter.first.wait_for(state=params.wait_selector_state) # Wait again after waiting for the selector, helpful with protections like Cloudflare page_info.page.wait_for_load_state(state="load") page_info.page.wait_for_load_state(state="domcontentloaded") - if network_idle: + if params.network_idle: page_info.page.wait_for_load_state("networkidle") except Exception as e: # pragma: no cover - log.error(f"Error waiting for selector {wait_selector}: {e}") + log.error(f"Error waiting for selector {params.wait_selector}: {e}") - page_info.page.wait_for_timeout(wait) + page_info.page.wait_for_timeout(params.wait) # Create response object response = ResponseFactory.from_playwright_response( - page_info.page, first_response, final_response, selector_config + page_info.page, first_response, final_response, params.selector_config ) # Mark the page as finished for next use @@ -413,17 +402,11 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession): self.playwright: AsyncPlaywright = await async_context().start() if self.cdp_url: - browser = await self.playwright.chromium.connect_over_cdp( - endpoint_url=self.cdp_url - ) - self.context: AsyncBrowserContext = await browser.new_context( - **self.context_options - ) + browser = await self.playwright.chromium.connect_over_cdp(endpoint_url=self.cdp_url) + self.context: AsyncBrowserContext = await browser.new_context(**self.context_options) else: - self.context: AsyncBrowserContext = ( - await self.playwright.chromium.launch_persistent_context( - user_data_dir="", **self.launch_options - ) + self.context: AsyncBrowserContext = await self.playwright.chromium.launch_persistent_context( + user_data_dir="", **self.launch_options ) # Get the default page and close it @@ -489,29 +472,21 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession): :param selector_config: The arguments that will be passed in the end while creating the final Selector's class. :return: A `Response` object. """ - google_search = self._get_with_precedence( - google_search, self.google_search, _UNSET - ) - timeout = self._get_with_precedence(timeout, self.timeout, _UNSET) - wait = self._get_with_precedence(wait, self.wait, _UNSET) - page_action = self._get_with_precedence(page_action, self.page_action, _UNSET) - extra_headers = self._get_with_precedence( - extra_headers, self.extra_headers, _UNSET - ) - disable_resources = self._get_with_precedence( - disable_resources, self.disable_resources, _UNSET - ) - wait_selector = self._get_with_precedence( - wait_selector, self.wait_selector, _UNSET - ) - wait_selector_state = self._get_with_precedence( - wait_selector_state, self.wait_selector_state, _UNSET - ) - network_idle = self._get_with_precedence( - network_idle, self.network_idle, _UNSET - ) - selector_config = self._get_with_precedence( - selector_config, self.selector_config, _UNSET + # Validate all resolved parameters + params = validate( + dict( + google_search=self._get_with_precedence(google_search, self.google_search, _UNSET), + timeout=self._get_with_precedence(timeout, self.timeout, _UNSET), + wait=self._get_with_precedence(wait, self.wait, _UNSET), + page_action=self._get_with_precedence(page_action, self.page_action, _UNSET), + extra_headers=self._get_with_precedence(extra_headers, self.extra_headers, _UNSET), + disable_resources=self._get_with_precedence(disable_resources, self.disable_resources, _UNSET), + wait_selector=self._get_with_precedence(wait_selector, self.wait_selector, _UNSET), + wait_selector_state=self._get_with_precedence(wait_selector_state, self.wait_selector_state, _UNSET), + network_idle=self._get_with_precedence(network_idle, self.network_idle, _UNSET), + selector_config=self._get_with_precedence(selector_config, self.selector_config, _UNSET), + ), + PlaywrightConfig, ) if self._closed: # pragma: no cover @@ -519,9 +494,7 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession): final_response = None referer = ( - generate_convincing_referer(url) - if (google_search and "referer" not in self._headers_keys) - else None + generate_convincing_referer(url) if (params.google_search and "referer" not in self._headers_keys) else None ) async def handle_response(finished_response: AsyncPlaywrightResponse): @@ -532,7 +505,7 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession): ): final_response = finished_response - page_info = await self._get_page(timeout, extra_headers, disable_resources) + page_info = await self._get_page(params.timeout, params.extra_headers, params.disable_resources) page_info.mark_busy(url=url) try: @@ -541,35 +514,35 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession): first_response = await page_info.page.goto(url, referer=referer) await page_info.page.wait_for_load_state(state="domcontentloaded") - if network_idle: + if params.network_idle: await page_info.page.wait_for_load_state("networkidle") if not first_response: raise RuntimeError(f"Failed to get response for {url}") - if page_action is not None: + if params.page_action: try: - _ = await page_action(page_info.page) + _ = await params.page_action(page_info.page) except Exception as e: log.error(f"Error executing page_action: {e}") - if wait_selector: + if params.wait_selector: try: - waiter: AsyncLocator = page_info.page.locator(wait_selector) - await waiter.first.wait_for(state=wait_selector_state) + waiter: AsyncLocator = page_info.page.locator(params.wait_selector) + await waiter.first.wait_for(state=params.wait_selector_state) # Wait again after waiting for the selector, helpful with protections like Cloudflare await page_info.page.wait_for_load_state(state="load") await page_info.page.wait_for_load_state(state="domcontentloaded") - if network_idle: + if params.network_idle: await page_info.page.wait_for_load_state("networkidle") except Exception as e: - log.error(f"Error waiting for selector {wait_selector}: {e}") + log.error(f"Error waiting for selector {params.wait_selector}: {e}") - await page_info.page.wait_for_timeout(wait) + await page_info.page.wait_for_timeout(params.wait) # Create response object response = await ResponseFactory.from_async_playwright_response( - page_info.page, first_response, final_response, selector_config + page_info.page, first_response, final_response, params.selector_config ) # Mark the page as finished for next use