From a17bb1d97607908a5bf8b4ab145968a02449c2c3 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Sun, 11 Jan 2026 23:40:26 +0200 Subject: [PATCH] feat(browsers): Add option to retry tabs that gives errors --- scrapling/engines/_browsers/_base.py | 8 + scrapling/engines/_browsers/_controllers.py | 168 +++++++++--------- scrapling/engines/_browsers/_stealth.py | 184 +++++++++++--------- scrapling/engines/_browsers/_types.py | 2 + scrapling/engines/_browsers/_validators.py | 3 + 5 files changed, 205 insertions(+), 160 deletions(-) diff --git a/scrapling/engines/_browsers/_base.py b/scrapling/engines/_browsers/_base.py index cbbdc24..810c51c 100644 --- a/scrapling/engines/_browsers/_base.py +++ b/scrapling/engines/_browsers/_base.py @@ -310,6 +310,14 @@ class BaseSessionMixin: if config.additional_args: self._context_options.update(config.additional_args) + @staticmethod + def _is_retriable(error: Exception) -> bool: + """Check if an error is retriable (transient network/timeout issues).""" + if isinstance(error, TimeoutError): + return True + error_msg = str(error).lower() + return "net::" in error_msg or "failed to get response" in error_msg + class DynamicSessionMixin(BaseSessionMixin): def __validate__(self, **params): diff --git a/scrapling/engines/_browsers/_controllers.py b/scrapling/engines/_browsers/_controllers.py index 3d39e9d..bd5d455 100644 --- a/scrapling/engines/_browsers/_controllers.py +++ b/scrapling/engines/_browsers/_controllers.py @@ -1,3 +1,6 @@ +from time import sleep as time_sleep +from asyncio import sleep as asyncio_sleep + from playwright.sync_api import ( Locator, Playwright, @@ -115,52 +118,56 @@ class DynamicSession(SyncSession, DynamicSessionMixin): else None ) - page_info = self._get_page(params.timeout, params.extra_headers, params.disable_resources) - final_response = [None] - handle_response = self._create_response_handler(page_info, final_response) + for attempt in range(self._config.retries): + page_info = self._get_page(params.timeout, params.extra_headers, params.disable_resources) + final_response = [None] + handle_response = self._create_response_handler(page_info, final_response) - try: # pragma: no cover - # Navigate to URL and wait for a specified state - page_info.page.on("response", handle_response) - first_response = page_info.page.goto(url, referer=referer) - self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle) + try: # pragma: no cover + page_info.page.on("response", handle_response) + first_response = page_info.page.goto(url, referer=referer) + self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle) - if not first_response: - raise RuntimeError(f"Failed to get response for {url}") + if not first_response: + raise RuntimeError(f"Failed to get response for {url}") - if params.page_action: - try: - _ = params.page_action(page_info.page) - except Exception as e: # pragma: no cover - log.error(f"Error executing page_action: {e}") + if params.page_action: + try: + _ = params.page_action(page_info.page) + except Exception as e: # pragma: no cover + log.error(f"Error executing page_action: {e}") - if params.wait_selector: - try: - waiter: Locator = page_info.page.locator(params.wait_selector) - waiter.first.wait_for(state=params.wait_selector_state) - # Wait again after waiting for the selector, helpful with protections like Cloudflare - self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle) - except Exception as e: # pragma: no cover - log.error(f"Error waiting for selector {params.wait_selector}: {e}") + if params.wait_selector: + try: + waiter: Locator = page_info.page.locator(params.wait_selector) + waiter.first.wait_for(state=params.wait_selector_state) + self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle) + except Exception as e: # pragma: no cover + log.error(f"Error waiting for selector {params.wait_selector}: {e}") - page_info.page.wait_for_timeout(params.wait) + page_info.page.wait_for_timeout(params.wait) - # Create response object - response = ResponseFactory.from_playwright_response( - page_info.page, first_response, final_response[0], params.selector_config - ) + response = ResponseFactory.from_playwright_response( + page_info.page, first_response, final_response[0], params.selector_config + ) - # Close the page to free up resources - page_info.page.close() - self.page_pool.pages.remove(page_info) + page_info.page.close() + self.page_pool.pages.remove(page_info) + return response - return response + except Exception as e: + page_info.mark_error() + page_info.page.close() + self.page_pool.pages.remove(page_info) - except Exception as e: - page_info.mark_error() - page_info.page.close() - self.page_pool.pages.remove(page_info) - raise e + if attempt < self._config.retries - 1 and self._is_retriable(e): + log.warning(f"Attempt {attempt + 1} failed: {e}. Retrying in {self._config.retry_delay}s...") + time_sleep(self._config.retry_delay) + else: + raise + + # For type checking purposes only + raise AssertionError("Unreachable: retry loop must return or raise") # pragma: no cover class AsyncDynamicSession(AsyncSession, DynamicSessionMixin): @@ -252,54 +259,59 @@ class AsyncDynamicSession(AsyncSession, DynamicSessionMixin): else None ) - page_info = await self._get_page(params.timeout, params.extra_headers, params.disable_resources) - final_response = [None] - handle_response = self._create_response_handler(page_info, final_response) + for attempt in range(self._config.retries): + page_info = await self._get_page(params.timeout, params.extra_headers, params.disable_resources) + final_response = [None] + handle_response = self._create_response_handler(page_info, final_response) - if TYPE_CHECKING: - from playwright.async_api import Page as async_Page + if TYPE_CHECKING: + from playwright.async_api import Page as async_Page - if not isinstance(page_info.page, async_Page): - raise TypeError + if not isinstance(page_info.page, async_Page): + raise TypeError - try: - # Navigate to URL and wait for a specified state - page_info.page.on("response", handle_response) - first_response = await page_info.page.goto(url, referer=referer) - await self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle) + try: + page_info.page.on("response", handle_response) + first_response = await page_info.page.goto(url, referer=referer) + await self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle) - if not first_response: - raise RuntimeError(f"Failed to get response for {url}") + if not first_response: + raise RuntimeError(f"Failed to get response for {url}") - if params.page_action: - try: - _ = await params.page_action(page_info.page) - except Exception as e: - log.error(f"Error executing page_action: {e}") + if params.page_action: + try: + _ = await params.page_action(page_info.page) + except Exception as e: + log.error(f"Error executing page_action: {e}") - if params.wait_selector: - try: - waiter: AsyncLocator = page_info.page.locator(params.wait_selector) - await waiter.first.wait_for(state=params.wait_selector_state) - # Wait again after waiting for the selector, helpful with protections like Cloudflare - await self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle) - except Exception as e: - log.error(f"Error waiting for selector {params.wait_selector}: {e}") + if params.wait_selector: + try: + waiter: AsyncLocator = page_info.page.locator(params.wait_selector) + await waiter.first.wait_for(state=params.wait_selector_state) + await self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle) + except Exception as e: + log.error(f"Error waiting for selector {params.wait_selector}: {e}") - await page_info.page.wait_for_timeout(params.wait) + await page_info.page.wait_for_timeout(params.wait) - # Create response object - response = await ResponseFactory.from_async_playwright_response( - page_info.page, first_response, final_response[0], params.selector_config - ) + response = await ResponseFactory.from_async_playwright_response( + page_info.page, first_response, final_response[0], params.selector_config + ) - # Close the page to free up resources - await page_info.page.close() - self.page_pool.pages.remove(page_info) - return response + await page_info.page.close() + self.page_pool.pages.remove(page_info) + return response - except Exception as e: # pragma: no cover - page_info.mark_error() - await page_info.page.close() - self.page_pool.pages.remove(page_info) - raise e + except Exception as e: # pragma: no cover + page_info.mark_error() + await page_info.page.close() + self.page_pool.pages.remove(page_info) + + if attempt < self._config.retries - 1 and self._is_retriable(e): + log.warning(f"Attempt {attempt + 1} failed: {e}. Retrying in {self._config.retry_delay}s...") + await asyncio_sleep(self._config.retry_delay) + else: + raise + + # For type checking purposes only + raise AssertionError("Unreachable: retry loop must return or raise") # pragma: no cover diff --git a/scrapling/engines/_browsers/_stealth.py b/scrapling/engines/_browsers/_stealth.py index ade3ab9..27c4d79 100644 --- a/scrapling/engines/_browsers/_stealth.py +++ b/scrapling/engines/_browsers/_stealth.py @@ -1,5 +1,7 @@ from random import randint from re import compile as re_compile +from time import sleep as time_sleep +from asyncio import sleep as asyncio_sleep from playwright.sync_api import ( Locator, @@ -202,57 +204,66 @@ class StealthySession(SyncSession, StealthySessionMixin): else None ) - page_info = self._get_page(params.timeout, params.extra_headers, params.disable_resources) - final_response = [None] - handle_response = self._create_response_handler(page_info, final_response) + for attempt in range(self._config.retries): + page_info = self._get_page(params.timeout, params.extra_headers, params.disable_resources) + final_response = [None] + handle_response = self._create_response_handler(page_info, final_response) - try: # pragma: no cover - # Navigate to URL and wait for a specified state - page_info.page.on("response", handle_response) - first_response = page_info.page.goto(url, referer=referer) - self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle) - - if not first_response: - raise RuntimeError(f"Failed to get response for {url}") - - if params.solve_cloudflare: - self._cloudflare_solver(page_info.page) - # Make sure the page is fully loaded after the captcha + try: # pragma: no cover + # Navigate to URL and wait for a specified state + page_info.page.on("response", handle_response) + first_response = page_info.page.goto(url, referer=referer) self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle) - if params.page_action: - try: - _ = params.page_action(page_info.page) - except Exception as e: # pragma: no cover - log.error(f"Error executing page_action: {e}") + if not first_response: + raise RuntimeError(f"Failed to get response for {url}") - if params.wait_selector: - try: - waiter: Locator = page_info.page.locator(params.wait_selector) - waiter.first.wait_for(state=params.wait_selector_state) - # Wait again after waiting for the selector, helpful with protections like Cloudflare + if params.solve_cloudflare: + self._cloudflare_solver(page_info.page) + # Make sure the page is fully loaded after the captcha self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle) - except Exception as e: # pragma: no cover - log.error(f"Error waiting for selector {params.wait_selector}: {e}") - page_info.page.wait_for_timeout(params.wait) + if params.page_action: + try: + _ = params.page_action(page_info.page) + except Exception as e: # pragma: no cover + log.error(f"Error executing page_action: {e}") - # Create response object - response = ResponseFactory.from_playwright_response( - page_info.page, first_response, final_response[0], params.selector_config - ) + if params.wait_selector: + try: + waiter: Locator = page_info.page.locator(params.wait_selector) + waiter.first.wait_for(state=params.wait_selector_state) + # Wait again after waiting for the selector, helpful with protections like Cloudflare + self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle) + except Exception as e: # pragma: no cover + log.error(f"Error waiting for selector {params.wait_selector}: {e}") - # Close the page to free up resources - page_info.page.close() - self.page_pool.pages.remove(page_info) + page_info.page.wait_for_timeout(params.wait) - return response + # Create response object + response = ResponseFactory.from_playwright_response( + page_info.page, first_response, final_response[0], params.selector_config + ) - except Exception as e: - page_info.mark_error() - page_info.page.close() - self.page_pool.pages.remove(page_info) - raise e + # Close the page to free up resources + page_info.page.close() + self.page_pool.pages.remove(page_info) + + return response + + except Exception as e: + page_info.mark_error() + page_info.page.close() + self.page_pool.pages.remove(page_info) + + if attempt < self._config.retries - 1 and self._is_retriable(e): + log.warning(f"Attempt {attempt + 1} failed: {e}. Retrying in {self._config.retry_delay}s...") + time_sleep(self._config.retry_delay) + else: + raise + + # For type checking purposes only + raise AssertionError("Unreachable: retry loop must return or raise") # pragma: no cover class AsyncStealthySession(AsyncSession, StealthySessionMixin): @@ -422,53 +433,62 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin): else None ) - page_info = await self._get_page(params.timeout, params.extra_headers, params.disable_resources) - final_response = [None] - handle_response = self._create_response_handler(page_info, final_response) + for attempt in range(self._config.retries): + page_info = await self._get_page(params.timeout, params.extra_headers, params.disable_resources) + final_response = [None] + handle_response = self._create_response_handler(page_info, final_response) - try: - # Navigate to URL and wait for a specified state - page_info.page.on("response", handle_response) - first_response = await page_info.page.goto(url, referer=referer) - await self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle) - - if not first_response: - raise RuntimeError(f"Failed to get response for {url}") - - if params.solve_cloudflare: - await self._cloudflare_solver(page_info.page) - # Make sure the page is fully loaded after the captcha + try: + # Navigate to URL and wait for a specified state + page_info.page.on("response", handle_response) + first_response = await page_info.page.goto(url, referer=referer) await self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle) - if params.page_action: - try: - _ = await params.page_action(page_info.page) - except Exception as e: - log.error(f"Error executing page_action: {e}") + if not first_response: + raise RuntimeError(f"Failed to get response for {url}") - if params.wait_selector: - try: - waiter: AsyncLocator = page_info.page.locator(params.wait_selector) - await waiter.first.wait_for(state=params.wait_selector_state) - # Wait again after waiting for the selector, helpful with protections like Cloudflare + if params.solve_cloudflare: + await self._cloudflare_solver(page_info.page) + # Make sure the page is fully loaded after the captcha await self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle) - except Exception as e: - log.error(f"Error waiting for selector {params.wait_selector}: {e}") - await page_info.page.wait_for_timeout(params.wait) + if params.page_action: + try: + _ = await params.page_action(page_info.page) + except Exception as e: + log.error(f"Error executing page_action: {e}") - # Create response object - response = await ResponseFactory.from_async_playwright_response( - page_info.page, first_response, final_response[0], params.selector_config - ) + if params.wait_selector: + try: + waiter: AsyncLocator = page_info.page.locator(params.wait_selector) + await waiter.first.wait_for(state=params.wait_selector_state) + # Wait again after waiting for the selector, helpful with protections like Cloudflare + await self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle) + except Exception as e: + log.error(f"Error waiting for selector {params.wait_selector}: {e}") - # Close the page to free up resources - await page_info.page.close() - self.page_pool.pages.remove(page_info) - return response + await page_info.page.wait_for_timeout(params.wait) - except Exception as e: # pragma: no cover - page_info.mark_error() - await page_info.page.close() - self.page_pool.pages.remove(page_info) - raise e + # Create response object + response = await ResponseFactory.from_async_playwright_response( + page_info.page, first_response, final_response[0], params.selector_config + ) + + # Close the page to free up resources + await page_info.page.close() + self.page_pool.pages.remove(page_info) + return response + + except Exception as e: # pragma: no cover + page_info.mark_error() + await page_info.page.close() + self.page_pool.pages.remove(page_info) + + if attempt < self._config.retries - 1 and self._is_retriable(e): + log.warning(f"Attempt {attempt + 1} failed: {e}. Retrying in {self._config.retry_delay}s...") + await asyncio_sleep(self._config.retry_delay) + else: + raise + + # For type checking purposes only + raise AssertionError("Unreachable: retry loop must return or raise") # pragma: no cover diff --git a/scrapling/engines/_browsers/_types.py b/scrapling/engines/_browsers/_types.py index cb8d760..8391e9c 100644 --- a/scrapling/engines/_browsers/_types.py +++ b/scrapling/engines/_browsers/_types.py @@ -79,6 +79,8 @@ if TYPE_CHECKING: # pragma: no cover cdp_url: Optional[str] useragent: Optional[str] extra_flags: Optional[List[str]] + retries: int + retry_delay: int | float class PlaywrightFetchParams(TypedDict, total=False): load_dom: bool diff --git a/scrapling/engines/_browsers/_validators.py b/scrapling/engines/_browsers/_validators.py index 82270ef..0b08f2f 100644 --- a/scrapling/engines/_browsers/_validators.py +++ b/scrapling/engines/_browsers/_validators.py @@ -50,6 +50,7 @@ def _is_invalid_cdp_url(cdp_url: str) -> bool | str: # Type aliases for cleaner annotations PagesCount = Annotated[int, Meta(ge=1, le=50)] +RetriesCount = Annotated[int, Meta(ge=1, le=10)] Seconds = Annotated[int, float, Meta(ge=0)] @@ -80,6 +81,8 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False, weakref=True): cdp_url: Optional[str] = None useragent: Optional[str] = None extra_flags: Optional[List[str]] = None + retries: RetriesCount = 3 + retry_delay: Seconds = 1 def __post_init__(self): # pragma: no cover """Custom validation after msgspec validation"""