freedak/Scrapling
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(browsers): Add option to retry tabs that gives errors

Browse Source
This commit is contained in:
Karim shoair
2026-01-11 23:40:26 +02:00
parent 5a11c2466b
commit a17bb1d976
5 changed files with 205 additions and 160 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scrapling/engines/_browsers/_base.py
+8
View File
@@ -310,6 +310,14 @@ class BaseSessionMixin:
if config.additional_args: if config.additional_args:
self._context_options.update(config.additional_args) self._context_options.update(config.additional_args)
@staticmethod
def _is_retriable(error: Exception) -> bool:
"""Check if an error is retriable (transient network/timeout issues)."""
if isinstance(error, TimeoutError):
return True
error_msg = str(error).lower()
return "net::" in error_msg or "failed to get response" in error_msg
class DynamicSessionMixin(BaseSessionMixin): class DynamicSessionMixin(BaseSessionMixin):
def __validate__(self, **params): def __validate__(self, **params):
scrapling/engines/_browsers/_controllers.py
+90 -78
View File
@@ -1,3 +1,6 @@
from time import sleep as time_sleep
from asyncio import sleep as asyncio_sleep
from playwright.sync_api import ( from playwright.sync_api import (
Locator, Locator,
Playwright, Playwright,
@@ -115,52 +118,56 @@ class DynamicSession(SyncSession, DynamicSessionMixin):
else None else None
) )
page_info = self._get_page(params.timeout, params.extra_headers, params.disable_resources) for attempt in range(self._config.retries):
final_response = [None] page_info = self._get_page(params.timeout, params.extra_headers, params.disable_resources)
handle_response = self._create_response_handler(page_info, final_response) final_response = [None]
handle_response = self._create_response_handler(page_info, final_response)
try: # pragma: no cover try: # pragma: no cover
# Navigate to URL and wait for a specified state page_info.page.on("response", handle_response)
page_info.page.on("response", handle_response) first_response = page_info.page.goto(url, referer=referer)
first_response = page_info.page.goto(url, referer=referer) self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle)
self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle)
if not first_response: if not first_response:
raise RuntimeError(f"Failed to get response for {url}") raise RuntimeError(f"Failed to get response for {url}")
if params.page_action: if params.page_action:
try: try:
_ = params.page_action(page_info.page) _ = params.page_action(page_info.page)
except Exception as e: # pragma: no cover except Exception as e: # pragma: no cover
log.error(f"Error executing page_action: {e}") log.error(f"Error executing page_action: {e}")
if params.wait_selector: if params.wait_selector:
try: try:
waiter: Locator = page_info.page.locator(params.wait_selector) waiter: Locator = page_info.page.locator(params.wait_selector)
waiter.first.wait_for(state=params.wait_selector_state) waiter.first.wait_for(state=params.wait_selector_state)
# Wait again after waiting for the selector, helpful with protections like Cloudflare self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle)
self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle) except Exception as e: # pragma: no cover
except Exception as e: # pragma: no cover log.error(f"Error waiting for selector {params.wait_selector}: {e}")
log.error(f"Error waiting for selector {params.wait_selector}: {e}")
page_info.page.wait_for_timeout(params.wait) page_info.page.wait_for_timeout(params.wait)
# Create response object response = ResponseFactory.from_playwright_response(
response = ResponseFactory.from_playwright_response( page_info.page, first_response, final_response[0], params.selector_config
page_info.page, first_response, final_response[0], params.selector_config )
)
# Close the page to free up resources page_info.page.close()
page_info.page.close() self.page_pool.pages.remove(page_info)
self.page_pool.pages.remove(page_info) return response
return response except Exception as e:
page_info.mark_error()
page_info.page.close()
self.page_pool.pages.remove(page_info)
except Exception as e: if attempt < self._config.retries - 1 and self._is_retriable(e):
page_info.mark_error() log.warning(f"Attempt {attempt + 1} failed: {e}. Retrying in {self._config.retry_delay}s...")
page_info.page.close() time_sleep(self._config.retry_delay)
self.page_pool.pages.remove(page_info) else:
raise e raise
# For type checking purposes only
raise AssertionError("Unreachable: retry loop must return or raise") # pragma: no cover
class AsyncDynamicSession(AsyncSession, DynamicSessionMixin): class AsyncDynamicSession(AsyncSession, DynamicSessionMixin):
@@ -252,54 +259,59 @@ class AsyncDynamicSession(AsyncSession, DynamicSessionMixin):
else None else None
) )
page_info = await self._get_page(params.timeout, params.extra_headers, params.disable_resources) for attempt in range(self._config.retries):
final_response = [None] page_info = await self._get_page(params.timeout, params.extra_headers, params.disable_resources)
handle_response = self._create_response_handler(page_info, final_response) final_response = [None]
handle_response = self._create_response_handler(page_info, final_response)
if TYPE_CHECKING: if TYPE_CHECKING:
from playwright.async_api import Page as async_Page from playwright.async_api import Page as async_Page
if not isinstance(page_info.page, async_Page): if not isinstance(page_info.page, async_Page):
raise TypeError raise TypeError
try: try:
# Navigate to URL and wait for a specified state page_info.page.on("response", handle_response)
page_info.page.on("response", handle_response) first_response = await page_info.page.goto(url, referer=referer)
first_response = await page_info.page.goto(url, referer=referer) await self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle)
await self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle)
if not first_response: if not first_response:
raise RuntimeError(f"Failed to get response for {url}") raise RuntimeError(f"Failed to get response for {url}")
if params.page_action: if params.page_action:
try: try:
_ = await params.page_action(page_info.page) _ = await params.page_action(page_info.page)
except Exception as e: except Exception as e:
log.error(f"Error executing page_action: {e}") log.error(f"Error executing page_action: {e}")
if params.wait_selector: if params.wait_selector:
try: try:
waiter: AsyncLocator = page_info.page.locator(params.wait_selector) waiter: AsyncLocator = page_info.page.locator(params.wait_selector)
await waiter.first.wait_for(state=params.wait_selector_state) await waiter.first.wait_for(state=params.wait_selector_state)
# Wait again after waiting for the selector, helpful with protections like Cloudflare await self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle)
await self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle) except Exception as e:
except Exception as e: log.error(f"Error waiting for selector {params.wait_selector}: {e}")
log.error(f"Error waiting for selector {params.wait_selector}: {e}")
await page_info.page.wait_for_timeout(params.wait) await page_info.page.wait_for_timeout(params.wait)
# Create response object response = await ResponseFactory.from_async_playwright_response(
response = await ResponseFactory.from_async_playwright_response( page_info.page, first_response, final_response[0], params.selector_config
page_info.page, first_response, final_response[0], params.selector_config )
)
# Close the page to free up resources await page_info.page.close()
await page_info.page.close() self.page_pool.pages.remove(page_info)
self.page_pool.pages.remove(page_info) return response
return response
except Exception as e: # pragma: no cover except Exception as e: # pragma: no cover
page_info.mark_error() page_info.mark_error()
await page_info.page.close() await page_info.page.close()
self.page_pool.pages.remove(page_info) self.page_pool.pages.remove(page_info)
raise e
if attempt < self._config.retries - 1 and self._is_retriable(e):
log.warning(f"Attempt {attempt + 1} failed: {e}. Retrying in {self._config.retry_delay}s...")
await asyncio_sleep(self._config.retry_delay)
else:
raise
# For type checking purposes only
raise AssertionError("Unreachable: retry loop must return or raise") # pragma: no cover
scrapling/engines/_browsers/_stealth.py
+102 -82
View File
@@ -1,5 +1,7 @@
from random import randint from random import randint
from re import compile as re_compile from re import compile as re_compile
from time import sleep as time_sleep
from asyncio import sleep as asyncio_sleep
from playwright.sync_api import ( from playwright.sync_api import (
Locator, Locator,
@@ -202,57 +204,66 @@ class StealthySession(SyncSession, StealthySessionMixin):
else None else None
) )
page_info = self._get_page(params.timeout, params.extra_headers, params.disable_resources) for attempt in range(self._config.retries):
final_response = [None] page_info = self._get_page(params.timeout, params.extra_headers, params.disable_resources)
handle_response = self._create_response_handler(page_info, final_response) final_response = [None]
handle_response = self._create_response_handler(page_info, final_response)
try: # pragma: no cover try: # pragma: no cover
# Navigate to URL and wait for a specified state # Navigate to URL and wait for a specified state
page_info.page.on("response", handle_response) page_info.page.on("response", handle_response)
first_response = page_info.page.goto(url, referer=referer) first_response = page_info.page.goto(url, referer=referer)
self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle)
if not first_response:
raise RuntimeError(f"Failed to get response for {url}")
if params.solve_cloudflare:
self._cloudflare_solver(page_info.page)
# Make sure the page is fully loaded after the captcha
self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle) self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle)
if params.page_action: if not first_response:
try: raise RuntimeError(f"Failed to get response for {url}")
_ = params.page_action(page_info.page)
except Exception as e: # pragma: no cover
log.error(f"Error executing page_action: {e}")
if params.wait_selector: if params.solve_cloudflare:
try: self._cloudflare_solver(page_info.page)
waiter: Locator = page_info.page.locator(params.wait_selector) # Make sure the page is fully loaded after the captcha
waiter.first.wait_for(state=params.wait_selector_state)
# Wait again after waiting for the selector, helpful with protections like Cloudflare
self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle) self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle)
except Exception as e: # pragma: no cover
log.error(f"Error waiting for selector {params.wait_selector}: {e}")
page_info.page.wait_for_timeout(params.wait) if params.page_action:
try:
_ = params.page_action(page_info.page)
except Exception as e: # pragma: no cover
log.error(f"Error executing page_action: {e}")
# Create response object if params.wait_selector:
response = ResponseFactory.from_playwright_response( try:
page_info.page, first_response, final_response[0], params.selector_config waiter: Locator = page_info.page.locator(params.wait_selector)
) waiter.first.wait_for(state=params.wait_selector_state)
# Wait again after waiting for the selector, helpful with protections like Cloudflare
self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle)
except Exception as e: # pragma: no cover
log.error(f"Error waiting for selector {params.wait_selector}: {e}")
# Close the page to free up resources page_info.page.wait_for_timeout(params.wait)
page_info.page.close()
self.page_pool.pages.remove(page_info)
return response # Create response object
response = ResponseFactory.from_playwright_response(
page_info.page, first_response, final_response[0], params.selector_config
)
except Exception as e: # Close the page to free up resources
page_info.mark_error() page_info.page.close()
page_info.page.close() self.page_pool.pages.remove(page_info)
self.page_pool.pages.remove(page_info)
raise e return response
except Exception as e:
page_info.mark_error()
page_info.page.close()
self.page_pool.pages.remove(page_info)
if attempt < self._config.retries - 1 and self._is_retriable(e):
log.warning(f"Attempt {attempt + 1} failed: {e}. Retrying in {self._config.retry_delay}s...")
time_sleep(self._config.retry_delay)
else:
raise
# For type checking purposes only
raise AssertionError("Unreachable: retry loop must return or raise") # pragma: no cover
class AsyncStealthySession(AsyncSession, StealthySessionMixin): class AsyncStealthySession(AsyncSession, StealthySessionMixin):
@@ -422,53 +433,62 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin):
else None else None
) )
page_info = await self._get_page(params.timeout, params.extra_headers, params.disable_resources) for attempt in range(self._config.retries):
final_response = [None] page_info = await self._get_page(params.timeout, params.extra_headers, params.disable_resources)
handle_response = self._create_response_handler(page_info, final_response) final_response = [None]
handle_response = self._create_response_handler(page_info, final_response)
try: try:
# Navigate to URL and wait for a specified state # Navigate to URL and wait for a specified state
page_info.page.on("response", handle_response) page_info.page.on("response", handle_response)
first_response = await page_info.page.goto(url, referer=referer) first_response = await page_info.page.goto(url, referer=referer)
await self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle)
if not first_response:
raise RuntimeError(f"Failed to get response for {url}")
if params.solve_cloudflare:
await self._cloudflare_solver(page_info.page)
# Make sure the page is fully loaded after the captcha
await self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle) await self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle)
if params.page_action: if not first_response:
try: raise RuntimeError(f"Failed to get response for {url}")
_ = await params.page_action(page_info.page)
except Exception as e:
log.error(f"Error executing page_action: {e}")
if params.wait_selector: if params.solve_cloudflare:
try: await self._cloudflare_solver(page_info.page)
waiter: AsyncLocator = page_info.page.locator(params.wait_selector) # Make sure the page is fully loaded after the captcha
await waiter.first.wait_for(state=params.wait_selector_state)
# Wait again after waiting for the selector, helpful with protections like Cloudflare
await self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle) await self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle)
except Exception as e:
log.error(f"Error waiting for selector {params.wait_selector}: {e}")
await page_info.page.wait_for_timeout(params.wait) if params.page_action:
try:
_ = await params.page_action(page_info.page)
except Exception as e:
log.error(f"Error executing page_action: {e}")
# Create response object if params.wait_selector:
response = await ResponseFactory.from_async_playwright_response( try:
page_info.page, first_response, final_response[0], params.selector_config waiter: AsyncLocator = page_info.page.locator(params.wait_selector)
) await waiter.first.wait_for(state=params.wait_selector_state)
# Wait again after waiting for the selector, helpful with protections like Cloudflare
await self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle)
except Exception as e:
log.error(f"Error waiting for selector {params.wait_selector}: {e}")
# Close the page to free up resources await page_info.page.wait_for_timeout(params.wait)
await page_info.page.close()
self.page_pool.pages.remove(page_info)
return response
except Exception as e: # pragma: no cover # Create response object
page_info.mark_error() response = await ResponseFactory.from_async_playwright_response(
await page_info.page.close() page_info.page, first_response, final_response[0], params.selector_config
self.page_pool.pages.remove(page_info) )
raise e
# Close the page to free up resources
await page_info.page.close()
self.page_pool.pages.remove(page_info)
return response
except Exception as e: # pragma: no cover
page_info.mark_error()
await page_info.page.close()
self.page_pool.pages.remove(page_info)
if attempt < self._config.retries - 1 and self._is_retriable(e):
log.warning(f"Attempt {attempt + 1} failed: {e}. Retrying in {self._config.retry_delay}s...")
await asyncio_sleep(self._config.retry_delay)
else:
raise
# For type checking purposes only
raise AssertionError("Unreachable: retry loop must return or raise") # pragma: no cover
scrapling/engines/_browsers/_types.py
+2
View File
@@ -79,6 +79,8 @@ if TYPE_CHECKING: # pragma: no cover
cdp_url: Optional[str] cdp_url: Optional[str]
useragent: Optional[str] useragent: Optional[str]
extra_flags: Optional[List[str]] extra_flags: Optional[List[str]]
retries: int
retry_delay: int | float
class PlaywrightFetchParams(TypedDict, total=False): class PlaywrightFetchParams(TypedDict, total=False):
load_dom: bool load_dom: bool
scrapling/engines/_browsers/_validators.py
+3
View File
@@ -50,6 +50,7 @@ def _is_invalid_cdp_url(cdp_url: str) -> bool | str:
# Type aliases for cleaner annotations # Type aliases for cleaner annotations
PagesCount = Annotated[int, Meta(ge=1, le=50)] PagesCount = Annotated[int, Meta(ge=1, le=50)]
RetriesCount = Annotated[int, Meta(ge=1, le=10)]
Seconds = Annotated[int, float, Meta(ge=0)] Seconds = Annotated[int, float, Meta(ge=0)]
@@ -80,6 +81,8 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False, weakref=True):
cdp_url: Optional[str] = None cdp_url: Optional[str] = None
useragent: Optional[str] = None useragent: Optional[str] = None
extra_flags: Optional[List[str]] = None extra_flags: Optional[List[str]] = None
retries: RetriesCount = 3
retry_delay: Seconds = 1
def __post_init__(self): # pragma: no cover def __post_init__(self): # pragma: no cover
"""Custom validation after msgspec validation""" """Custom validation after msgspec validation"""