refactor(fetchers): Less duplicated code and better handling for unstable websites
- Websites that never finish loading their requests won't crash the code now if you used `network_idle` with them - Fixed a typo in `load_dom` in DynamicSession's async_fetch - Removed dead code
This commit is contained in:
Karim shoair
@@ -1,16 +1,13 @@
|
||||
from playwright.sync_api import (
|
||||
Response as SyncPlaywrightResponse,
|
||||
sync_playwright,
|
||||
Playwright,
|
||||
Locator,
|
||||
Playwright,
|
||||
sync_playwright,
|
||||
)
|
||||
from playwright.async_api import (
|
||||
async_playwright,
|
||||
Response as AsyncPlaywrightResponse,
|
||||
BrowserContext as AsyncBrowserContext,
|
||||
Playwright as AsyncPlaywright,
|
||||
Locator as AsyncLocator,
|
||||
Page as async_Page,
|
||||
Playwright as AsyncPlaywright,
|
||||
BrowserContext as AsyncBrowserContext,
|
||||
)
|
||||
from patchright.sync_api import sync_playwright as sync_patchright
|
||||
from patchright.async_api import async_playwright as async_patchright
|
||||
@@ -178,28 +175,6 @@ class DynamicSession(DynamicSessionMixin, SyncSession):
|
||||
if self.cookies: # pragma: no cover
|
||||
self.context.add_cookies(self.cookies)
|
||||
|
||||
def __enter__(self):
|
||||
self.__create__()
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||
self.close()
|
||||
|
||||
def close(self): # pragma: no cover
|
||||
"""Close all resources"""
|
||||
if self._closed:
|
||||
return
|
||||
|
||||
if self.context:
|
||||
self.context.close()
|
||||
self.context = None
|
||||
|
||||
if self.playwright:
|
||||
self.playwright.stop()
|
||||
self.playwright = None # pyright: ignore
|
||||
|
||||
self._closed = True
|
||||
|
||||
def fetch(
|
||||
self,
|
||||
url: str,
|
||||
@@ -253,32 +228,19 @@ class DynamicSession(DynamicSessionMixin, SyncSession):
|
||||
if self._closed: # pragma: no cover
|
||||
raise RuntimeError("Context manager has been closed")
|
||||
|
||||
final_response = None
|
||||
referer = (
|
||||
generate_convincing_referer(url) if (params.google_search and "referer" not in self._headers_keys) else None
|
||||
)
|
||||
|
||||
def handle_response(finished_response: SyncPlaywrightResponse):
|
||||
nonlocal final_response
|
||||
if (
|
||||
finished_response.request.resource_type == "document"
|
||||
and finished_response.request.is_navigation_request()
|
||||
and finished_response.request.frame == page_info.page.main_frame
|
||||
):
|
||||
final_response = finished_response
|
||||
|
||||
page_info = self._get_page(params.timeout, params.extra_headers, params.disable_resources)
|
||||
page_info.mark_busy(url=url)
|
||||
final_response = [None]
|
||||
handle_response = self._create_response_handler(page_info, final_response)
|
||||
|
||||
try: # pragma: no cover
|
||||
# Navigate to URL and wait for a specified state
|
||||
page_info.page.on("response", handle_response)
|
||||
first_response = page_info.page.goto(url, referer=referer)
|
||||
if params.load_dom:
|
||||
page_info.page.wait_for_load_state(state="domcontentloaded")
|
||||
|
||||
if params.network_idle:
|
||||
page_info.page.wait_for_load_state("networkidle")
|
||||
self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle)
|
||||
|
||||
if not first_response:
|
||||
raise RuntimeError(f"Failed to get response for {url}")
|
||||
@@ -294,11 +256,7 @@ class DynamicSession(DynamicSessionMixin, SyncSession):
|
||||
waiter: Locator = page_info.page.locator(params.wait_selector)
|
||||
waiter.first.wait_for(state=params.wait_selector_state)
|
||||
# Wait again after waiting for the selector, helpful with protections like Cloudflare
|
||||
page_info.page.wait_for_load_state(state="load")
|
||||
if params.load_dom:
|
||||
page_info.page.wait_for_load_state(state="domcontentloaded")
|
||||
if params.network_idle:
|
||||
page_info.page.wait_for_load_state("networkidle")
|
||||
self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle)
|
||||
except Exception as e: # pragma: no cover
|
||||
log.error(f"Error waiting for selector {params.wait_selector}: {e}")
|
||||
|
||||
@@ -306,7 +264,7 @@ class DynamicSession(DynamicSessionMixin, SyncSession):
|
||||
|
||||
# Create response object
|
||||
response = ResponseFactory.from_playwright_response(
|
||||
page_info.page, first_response, final_response, params.selector_config, bool(params.page_action)
|
||||
page_info.page, first_response, final_response[0], params.selector_config, bool(params.page_action)
|
||||
)
|
||||
|
||||
# Close the page to free up resources
|
||||
@@ -431,28 +389,6 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession):
|
||||
if self.cookies:
|
||||
await self.context.add_cookies(self.cookies) # pyright: ignore
|
||||
|
||||
async def __aenter__(self):
|
||||
await self.__create__()
|
||||
return self
|
||||
|
||||
async def __aexit__(self, exc_type, exc_val, exc_tb):
|
||||
await self.close()
|
||||
|
||||
async def close(self):
|
||||
"""Close all resources"""
|
||||
if self._closed: # pragma: no cover
|
||||
return
|
||||
|
||||
if self.context:
|
||||
await self.context.close()
|
||||
self.context = None # pyright: ignore
|
||||
|
||||
if self.playwright:
|
||||
await self.playwright.stop()
|
||||
self.playwright = None # pyright: ignore
|
||||
|
||||
self._closed = True
|
||||
|
||||
async def fetch(
|
||||
self,
|
||||
url: str,
|
||||
@@ -506,24 +442,17 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession):
|
||||
if self._closed: # pragma: no cover
|
||||
raise RuntimeError("Context manager has been closed")
|
||||
|
||||
final_response = None
|
||||
referer = (
|
||||
generate_convincing_referer(url) if (params.google_search and "referer" not in self._headers_keys) else None
|
||||
)
|
||||
|
||||
async def handle_response(finished_response: AsyncPlaywrightResponse):
|
||||
nonlocal final_response
|
||||
if (
|
||||
finished_response.request.resource_type == "document"
|
||||
and finished_response.request.is_navigation_request()
|
||||
and finished_response.request.frame == page_info.page.main_frame
|
||||
):
|
||||
final_response = finished_response
|
||||
|
||||
page_info = await self._get_page(params.timeout, params.extra_headers, params.disable_resources)
|
||||
page_info.mark_busy(url=url)
|
||||
final_response = [None]
|
||||
handle_response = self._create_response_handler(page_info, final_response)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from playwright.async_api import Page as async_Page
|
||||
|
||||
if not isinstance(page_info.page, async_Page):
|
||||
raise TypeError
|
||||
|
||||
@@ -531,11 +460,7 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession):
|
||||
# Navigate to URL and wait for a specified state
|
||||
page_info.page.on("response", handle_response)
|
||||
first_response = await page_info.page.goto(url, referer=referer)
|
||||
if self.load_dom:
|
||||
await page_info.page.wait_for_load_state(state="domcontentloaded")
|
||||
|
||||
if params.network_idle:
|
||||
await page_info.page.wait_for_load_state("networkidle")
|
||||
await self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle)
|
||||
|
||||
if not first_response:
|
||||
raise RuntimeError(f"Failed to get response for {url}")
|
||||
@@ -551,11 +476,7 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession):
|
||||
waiter: AsyncLocator = page_info.page.locator(params.wait_selector)
|
||||
await waiter.first.wait_for(state=params.wait_selector_state)
|
||||
# Wait again after waiting for the selector, helpful with protections like Cloudflare
|
||||
await page_info.page.wait_for_load_state(state="load")
|
||||
if self.load_dom:
|
||||
await page_info.page.wait_for_load_state(state="domcontentloaded")
|
||||
if params.network_idle:
|
||||
await page_info.page.wait_for_load_state("networkidle")
|
||||
await self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle)
|
||||
except Exception as e:
|
||||
log.error(f"Error waiting for selector {params.wait_selector}: {e}")
|
||||
|
||||
@@ -563,7 +484,7 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession):
|
||||
|
||||
# Create response object
|
||||
response = await ResponseFactory.from_async_playwright_response(
|
||||
page_info.page, first_response, final_response, params.selector_config, bool(params.page_action)
|
||||
page_info.page, first_response, final_response[0], params.selector_config, bool(params.page_action)
|
||||
)
|
||||
|
||||
# Close the page to free up resources
|
||||
|
||||
Reference in New Issue
Block a user