fix(fetchers/content): increase the default max number of retries and raise error on max retries

Ref.: https://github.com/D4Vinci/Scrapling/pull/197#issuecomment-4077705587
This commit is contained in:
Karim shoair
2026-03-17 22:14:02 +02:00
parent 17ea982162
commit 1dc0b7a1bd
+6 -6
View File
@@ -187,11 +187,11 @@ class ResponseFactory:
return history return history
@classmethod @classmethod
def _get_page_content(cls, page: SyncPage, max_retries: int = 10) -> str: def _get_page_content(cls, page: SyncPage, max_retries: int = 20) -> str:
""" """
A workaround for the Playwright issue with `page.content()` on Windows. Ref.: https://github.com/microsoft/playwright/issues/16108 A workaround for the Playwright issue with `page.content()` on Windows. Ref.: https://github.com/microsoft/playwright/issues/16108
:param page: The page to extract content from. :param page: The page to extract content from.
:param max_retries: Maximum number of retry attempts before returning empty string. :param max_retries: Maximum number of retry attempts before raising `RuntimeError`.
:return: :return:
""" """
for _ in range(max_retries): for _ in range(max_retries):
@@ -199,14 +199,14 @@ class ResponseFactory:
return page.content() or "" return page.content() or ""
except PlaywrightError: except PlaywrightError:
page.wait_for_timeout(500) page.wait_for_timeout(500)
return "" raise RuntimeError(f"Failed to retrieve the page content after retrying for {max_retries * 500}ms.")
@classmethod @classmethod
async def _get_async_page_content(cls, page: AsyncPage, max_retries: int = 10) -> str: async def _get_async_page_content(cls, page: AsyncPage, max_retries: int = 20) -> str:
""" """
A workaround for the Playwright issue with `page.content()` on Windows. Ref.: https://github.com/microsoft/playwright/issues/16108 A workaround for the Playwright issue with `page.content()` on Windows. Ref.: https://github.com/microsoft/playwright/issues/16108
:param page: The page to extract content from. :param page: The page to extract content from.
:param max_retries: Maximum number of retry attempts before returning empty string. :param max_retries: Maximum number of retry attempts before raising `RuntimeError`.
:return: :return:
""" """
for _ in range(max_retries): for _ in range(max_retries):
@@ -214,7 +214,7 @@ class ResponseFactory:
return (await page.content()) or "" return (await page.content()) or ""
except PlaywrightError: except PlaywrightError:
await page.wait_for_timeout(500) await page.wait_for_timeout(500)
return "" raise RuntimeError(f"Failed to retrieve the page content after retrying for {max_retries * 500}ms.")
@classmethod @classmethod
async def from_async_playwright_response( async def from_async_playwright_response(