fix(fetchers): add max retry limit to _get_page_content to prevent infinite loop (#197)
This commit is contained in:
@@ -187,34 +187,34 @@ class ResponseFactory:
|
|||||||
return history
|
return history
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def _get_page_content(cls, page: SyncPage) -> str:
|
def _get_page_content(cls, page: SyncPage, max_retries: int = 10) -> str:
|
||||||
"""
|
"""
|
||||||
A workaround for the Playwright issue with `page.content()` on Windows. Ref.: https://github.com/microsoft/playwright/issues/16108
|
A workaround for the Playwright issue with `page.content()` on Windows. Ref.: https://github.com/microsoft/playwright/issues/16108
|
||||||
:param page: The page to extract content from.
|
:param page: The page to extract content from.
|
||||||
|
:param max_retries: Maximum number of retry attempts before returning empty string.
|
||||||
:return:
|
:return:
|
||||||
"""
|
"""
|
||||||
while True:
|
for _ in range(max_retries):
|
||||||
try:
|
try:
|
||||||
return page.content() or ""
|
return page.content() or ""
|
||||||
except PlaywrightError:
|
except PlaywrightError:
|
||||||
page.wait_for_timeout(500)
|
page.wait_for_timeout(500)
|
||||||
continue
|
return ""
|
||||||
return "" # pyright: ignore
|
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
async def _get_async_page_content(cls, page: AsyncPage) -> str:
|
async def _get_async_page_content(cls, page: AsyncPage, max_retries: int = 10) -> str:
|
||||||
"""
|
"""
|
||||||
A workaround for the Playwright issue with `page.content()` on Windows. Ref.: https://github.com/microsoft/playwright/issues/16108
|
A workaround for the Playwright issue with `page.content()` on Windows. Ref.: https://github.com/microsoft/playwright/issues/16108
|
||||||
:param page: The page to extract content from.
|
:param page: The page to extract content from.
|
||||||
|
:param max_retries: Maximum number of retry attempts before returning empty string.
|
||||||
:return:
|
:return:
|
||||||
"""
|
"""
|
||||||
while True:
|
for _ in range(max_retries):
|
||||||
try:
|
try:
|
||||||
return (await page.content()) or ""
|
return (await page.content()) or ""
|
||||||
except PlaywrightError:
|
except PlaywrightError:
|
||||||
await page.wait_for_timeout(500)
|
await page.wait_for_timeout(500)
|
||||||
continue
|
return ""
|
||||||
return "" # pyright: ignore
|
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
async def from_async_playwright_response(
|
async def from_async_playwright_response(
|
||||||
|
|||||||
Reference in New Issue
Block a user