freedak/Scrapling
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Validate all fetch-level parameters

Browse Source 
So it's validated like the session-level ones. Needs improvement, but that's for later
This commit is contained in:
Karim shoair
2025-09-13 03:22:08 +03:00
parent 831eafca34
commit 60be9dc816
2 changed files with 128 additions and 185 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scrapling/engines/_browsers/_camoufox.py
+65 -95
View File
@@ -15,6 +15,7 @@ from playwright.async_api import (
Page as async_Page, Page as async_Page,
) )
from ._validators import validate, CamoufoxConfig
from ._base import SyncSession, AsyncSession, StealthySessionMixin from ._base import SyncSession, AsyncSession, StealthySessionMixin
from scrapling.core.utils import log from scrapling.core.utils import log
from scrapling.core._types import ( from scrapling.core._types import (
@@ -164,10 +165,8 @@ class StealthySession(StealthySessionMixin, SyncSession):
def __create__(self): def __create__(self):
"""Create a browser for this instance and context.""" """Create a browser for this instance and context."""
self.playwright = sync_playwright().start() self.playwright = sync_playwright().start()
self.context = ( self.context = self.playwright.firefox.launch_persistent_context( # pragma: no cover
self.playwright.firefox.launch_persistent_context( # pragma: no cover **self.launch_options
**self.launch_options
)
) )
# Get the default page and close it # Get the default page and close it
@@ -281,32 +280,22 @@ class StealthySession(StealthySessionMixin, SyncSession):
:param selector_config: The arguments that will be passed in the end while creating the final Selector's class. :param selector_config: The arguments that will be passed in the end while creating the final Selector's class.
:return: A `Response` object. :return: A `Response` object.
""" """
google_search = self._get_with_precedence( # Validate all resolved parameters
google_search, self.google_search, _UNSET params = validate(
) dict(
timeout = self._get_with_precedence(timeout, self.timeout, _UNSET) google_search=self._get_with_precedence(google_search, self.google_search, _UNSET),
wait = self._get_with_precedence(wait, self.wait, _UNSET) timeout=self._get_with_precedence(timeout, self.timeout, _UNSET),
page_action = self._get_with_precedence(page_action, self.page_action, _UNSET) wait=self._get_with_precedence(wait, self.wait, _UNSET),
extra_headers = self._get_with_precedence( page_action=self._get_with_precedence(page_action, self.page_action, _UNSET),
extra_headers, self.extra_headers, _UNSET extra_headers=self._get_with_precedence(extra_headers, self.extra_headers, _UNSET),
) disable_resources=self._get_with_precedence(disable_resources, self.disable_resources, _UNSET),
disable_resources = self._get_with_precedence( wait_selector=self._get_with_precedence(wait_selector, self.wait_selector, _UNSET),
disable_resources, self.disable_resources, _UNSET wait_selector_state=self._get_with_precedence(wait_selector_state, self.wait_selector_state, _UNSET),
) network_idle=self._get_with_precedence(network_idle, self.network_idle, _UNSET),
wait_selector = self._get_with_precedence( solve_cloudflare=self._get_with_precedence(solve_cloudflare, self.solve_cloudflare, _UNSET),
wait_selector, self.wait_selector, _UNSET selector_config=self._get_with_precedence(selector_config, self.selector_config, _UNSET),
) ),
wait_selector_state = self._get_with_precedence( CamoufoxConfig,
wait_selector_state, self.wait_selector_state, _UNSET
)
network_idle = self._get_with_precedence(
network_idle, self.network_idle, _UNSET
)
solve_cloudflare = self._get_with_precedence(
solve_cloudflare, self.solve_cloudflare, _UNSET
)
selector_config = self._get_with_precedence(
selector_config, self.selector_config, _UNSET
) )
if self._closed: # pragma: no cover if self._closed: # pragma: no cover
@@ -314,9 +303,7 @@ class StealthySession(StealthySessionMixin, SyncSession):
final_response = None final_response = None
referer = ( referer = (
generate_convincing_referer(url) generate_convincing_referer(url) if (params.google_search and "referer" not in self._headers_keys) else None
if (google_search and "referer" not in self._headers_keys)
else None
) )
def handle_response(finished_response: SyncPlaywrightResponse): def handle_response(finished_response: SyncPlaywrightResponse):
@@ -327,7 +314,7 @@ class StealthySession(StealthySessionMixin, SyncSession):
): ):
final_response = finished_response final_response = finished_response
page_info = self._get_page(timeout, extra_headers, disable_resources) page_info = self._get_page(params.timeout, params.extra_headers, params.disable_resources)
page_info.mark_busy(url=url) page_info.mark_busy(url=url)
try: # pragma: no cover try: # pragma: no cover
@@ -336,41 +323,41 @@ class StealthySession(StealthySessionMixin, SyncSession):
first_response = page_info.page.goto(url, referer=referer) first_response = page_info.page.goto(url, referer=referer)
page_info.page.wait_for_load_state(state="domcontentloaded") page_info.page.wait_for_load_state(state="domcontentloaded")
if network_idle: if params.network_idle:
page_info.page.wait_for_load_state("networkidle") page_info.page.wait_for_load_state("networkidle")
if not first_response: if not first_response:
raise RuntimeError(f"Failed to get response for {url}") raise RuntimeError(f"Failed to get response for {url}")
if solve_cloudflare: if params.solve_cloudflare:
self._solve_cloudflare(page_info.page) self._solve_cloudflare(page_info.page)
# Make sure the page is fully loaded after the captcha # Make sure the page is fully loaded after the captcha
page_info.page.wait_for_load_state(state="load") page_info.page.wait_for_load_state(state="load")
page_info.page.wait_for_load_state(state="domcontentloaded") page_info.page.wait_for_load_state(state="domcontentloaded")
if network_idle: if params.network_idle:
page_info.page.wait_for_load_state("networkidle") page_info.page.wait_for_load_state("networkidle")
if page_action is not None: if params.page_action:
try: try:
_ = page_action(page_info.page) _ = params.page_action(page_info.page)
except Exception as e: except Exception as e:
log.error(f"Error executing page_action: {e}") log.error(f"Error executing page_action: {e}")
if wait_selector: if params.wait_selector:
try: try:
waiter: Locator = page_info.page.locator(wait_selector) waiter: Locator = page_info.page.locator(params.wait_selector)
waiter.first.wait_for(state=wait_selector_state) waiter.first.wait_for(state=params.wait_selector_state)
# Wait again after waiting for the selector, helpful with protections like Cloudflare # Wait again after waiting for the selector, helpful with protections like Cloudflare
page_info.page.wait_for_load_state(state="load") page_info.page.wait_for_load_state(state="load")
page_info.page.wait_for_load_state(state="domcontentloaded") page_info.page.wait_for_load_state(state="domcontentloaded")
if network_idle: if params.network_idle:
page_info.page.wait_for_load_state("networkidle") page_info.page.wait_for_load_state("networkidle")
except Exception as e: except Exception as e:
log.error(f"Error waiting for selector {wait_selector}: {e}") log.error(f"Error waiting for selector {params.wait_selector}: {e}")
page_info.page.wait_for_timeout(wait) page_info.page.wait_for_timeout(params.wait)
response = ResponseFactory.from_playwright_response( response = ResponseFactory.from_playwright_response(
page_info.page, first_response, final_response, selector_config page_info.page, first_response, final_response, params.selector_config
) )
# Mark the page as finished for next use # Mark the page as finished for next use
@@ -478,10 +465,8 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):
async def __create__(self): async def __create__(self):
"""Create a browser for this instance and context.""" """Create a browser for this instance and context."""
self.playwright: AsyncPlaywright = await async_playwright().start() self.playwright: AsyncPlaywright = await async_playwright().start()
self.context: AsyncBrowserContext = ( self.context: AsyncBrowserContext = await self.playwright.firefox.launch_persistent_context(
await self.playwright.firefox.launch_persistent_context( **self.launch_options
**self.launch_options
)
) )
# Get the default page and close it # Get the default page and close it
@@ -551,9 +536,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):
await page.wait_for_timeout(500) await page.wait_for_timeout(500)
# Calculate the Captcha coordinates for any viewport # Calculate the Captcha coordinates for any viewport
outer_box = await page.locator( outer_box = await page.locator(".main-content p+div>div>div").bounding_box()
".main-content p+div>div>div"
).bounding_box()
captcha_x, captcha_y = outer_box["x"] + 26, outer_box["y"] + 25 captcha_x, captcha_y = outer_box["x"] + 26, outer_box["y"] + 25
# Move the mouse to the center of the window, then press and hold the left mouse button # Move the mouse to the center of the window, then press and hold the left mouse button
@@ -597,32 +580,21 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):
:param selector_config: The arguments that will be passed in the end while creating the final Selector's class. :param selector_config: The arguments that will be passed in the end while creating the final Selector's class.
:return: A `Response` object. :return: A `Response` object.
""" """
google_search = self._get_with_precedence( params = validate(
google_search, self.google_search, _UNSET dict(
) google_search=self._get_with_precedence(google_search, self.google_search, _UNSET),
timeout = self._get_with_precedence(timeout, self.timeout, _UNSET) timeout=self._get_with_precedence(timeout, self.timeout, _UNSET),
wait = self._get_with_precedence(wait, self.wait, _UNSET) wait=self._get_with_precedence(wait, self.wait, _UNSET),
page_action = self._get_with_precedence(page_action, self.page_action, _UNSET) page_action=self._get_with_precedence(page_action, self.page_action, _UNSET),
extra_headers = self._get_with_precedence( extra_headers=self._get_with_precedence(extra_headers, self.extra_headers, _UNSET),
extra_headers, self.extra_headers, _UNSET disable_resources=self._get_with_precedence(disable_resources, self.disable_resources, _UNSET),
) wait_selector=self._get_with_precedence(wait_selector, self.wait_selector, _UNSET),
disable_resources = self._get_with_precedence( wait_selector_state=self._get_with_precedence(wait_selector_state, self.wait_selector_state, _UNSET),
disable_resources, self.disable_resources, _UNSET network_idle=self._get_with_precedence(network_idle, self.network_idle, _UNSET),
) solve_cloudflare=self._get_with_precedence(solve_cloudflare, self.solve_cloudflare, _UNSET),
wait_selector = self._get_with_precedence( selector_config=self._get_with_precedence(selector_config, self.selector_config, _UNSET),
wait_selector, self.wait_selector, _UNSET ),
) CamoufoxConfig,
wait_selector_state = self._get_with_precedence(
wait_selector_state, self.wait_selector_state, _UNSET
)
network_idle = self._get_with_precedence(
network_idle, self.network_idle, _UNSET
)
solve_cloudflare = self._get_with_precedence(
solve_cloudflare, self.solve_cloudflare, _UNSET
)
selector_config = self._get_with_precedence(
selector_config, self.selector_config, _UNSET
) )
if self._closed: # pragma: no cover if self._closed: # pragma: no cover
@@ -630,9 +602,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):
final_response = None final_response = None
referer = ( referer = (
generate_convincing_referer(url) generate_convincing_referer(url) if (params.google_search and "referer" not in self._headers_keys) else None
if (google_search and "referer" not in self._headers_keys)
else None
) )
async def handle_response(finished_response: AsyncPlaywrightResponse): async def handle_response(finished_response: AsyncPlaywrightResponse):
@@ -643,7 +613,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):
): ):
final_response = finished_response final_response = finished_response
page_info = await self._get_page(timeout, extra_headers, disable_resources) page_info = await self._get_page(params.timeout, params.extra_headers, params.disable_resources)
page_info.mark_busy(url=url) page_info.mark_busy(url=url)
try: try:
@@ -652,43 +622,43 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):
first_response = await page_info.page.goto(url, referer=referer) first_response = await page_info.page.goto(url, referer=referer)
await page_info.page.wait_for_load_state(state="domcontentloaded") await page_info.page.wait_for_load_state(state="domcontentloaded")
if network_idle: if params.network_idle:
await page_info.page.wait_for_load_state("networkidle") await page_info.page.wait_for_load_state("networkidle")
if not first_response: if not first_response:
raise RuntimeError(f"Failed to get response for {url}") raise RuntimeError(f"Failed to get response for {url}")
if solve_cloudflare: if params.solve_cloudflare:
await self._solve_cloudflare(page_info.page) await self._solve_cloudflare(page_info.page)
# Make sure the page is fully loaded after the captcha # Make sure the page is fully loaded after the captcha
await page_info.page.wait_for_load_state(state="load") await page_info.page.wait_for_load_state(state="load")
await page_info.page.wait_for_load_state(state="domcontentloaded") await page_info.page.wait_for_load_state(state="domcontentloaded")
if network_idle: if params.network_idle:
await page_info.page.wait_for_load_state("networkidle") await page_info.page.wait_for_load_state("networkidle")
if page_action is not None: if params.page_action:
try: try:
_ = await page_action(page_info.page) _ = await params.page_action(page_info.page)
except Exception as e: except Exception as e:
log.error(f"Error executing page_action: {e}") log.error(f"Error executing page_action: {e}")
if wait_selector: if params.wait_selector:
try: try:
waiter: AsyncLocator = page_info.page.locator(wait_selector) waiter: AsyncLocator = page_info.page.locator(params.wait_selector)
await waiter.first.wait_for(state=wait_selector_state) await waiter.first.wait_for(state=params.wait_selector_state)
# Wait again after waiting for the selector, helpful with protections like Cloudflare # Wait again after waiting for the selector, helpful with protections like Cloudflare
await page_info.page.wait_for_load_state(state="load") await page_info.page.wait_for_load_state(state="load")
await page_info.page.wait_for_load_state(state="domcontentloaded") await page_info.page.wait_for_load_state(state="domcontentloaded")
if network_idle: if params.network_idle:
await page_info.page.wait_for_load_state("networkidle") await page_info.page.wait_for_load_state("networkidle")
except Exception as e: except Exception as e:
log.error(f"Error waiting for selector {wait_selector}: {e}") log.error(f"Error waiting for selector {params.wait_selector}: {e}")
await page_info.page.wait_for_timeout(wait) await page_info.page.wait_for_timeout(params.wait)
# Create response object # Create response object
response = await ResponseFactory.from_async_playwright_response( response = await ResponseFactory.from_async_playwright_response(
page_info.page, first_response, final_response, selector_config page_info.page, first_response, final_response, params.selector_config
) )
# Mark the page as finished for next use # Mark the page as finished for next use
scrapling/engines/_browsers/_controllers.py
+63 -90
View File
@@ -18,6 +18,7 @@ from rebrowser_playwright.async_api import (
from scrapling.core.utils import log from scrapling.core.utils import log
from ._base import SyncSession, AsyncSession, DynamicSessionMixin from ._base import SyncSession, AsyncSession, DynamicSessionMixin
from ._validators import validate, PlaywrightConfig
from scrapling.core._types import ( from scrapling.core._types import (
Dict, Dict,
List, List,
@@ -157,13 +158,11 @@ class DynamicSession(DynamicSessionMixin, SyncSession):
self.playwright: Playwright = sync_context().start() self.playwright: Playwright = sync_context().start()
if self.cdp_url: # pragma: no cover if self.cdp_url: # pragma: no cover
self.context = self.playwright.chromium.connect_over_cdp( self.context = self.playwright.chromium.connect_over_cdp(endpoint_url=self.cdp_url).new_context(
endpoint_url=self.cdp_url **self.context_options
).new_context(**self.context_options)
else:
self.context = self.playwright.chromium.launch_persistent_context(
user_data_dir="", **self.launch_options
) )
else:
self.context = self.playwright.chromium.launch_persistent_context(user_data_dir="", **self.launch_options)
# Get the default page and close it # Get the default page and close it
default_page = self.context.pages[0] default_page = self.context.pages[0]
@@ -228,29 +227,21 @@ class DynamicSession(DynamicSessionMixin, SyncSession):
:param selector_config: The arguments that will be passed in the end while creating the final Selector's class. :param selector_config: The arguments that will be passed in the end while creating the final Selector's class.
:return: A `Response` object. :return: A `Response` object.
""" """
google_search = self._get_with_precedence( # Validate all resolved parameters
google_search, self.google_search, _UNSET params = validate(
) dict(
timeout = self._get_with_precedence(timeout, self.timeout, _UNSET) google_search=self._get_with_precedence(google_search, self.google_search, _UNSET),
wait = self._get_with_precedence(wait, self.wait, _UNSET) timeout=self._get_with_precedence(timeout, self.timeout, _UNSET),
page_action = self._get_with_precedence(page_action, self.page_action, _UNSET) wait=self._get_with_precedence(wait, self.wait, _UNSET),
extra_headers = self._get_with_precedence( page_action=self._get_with_precedence(page_action, self.page_action, _UNSET),
extra_headers, self.extra_headers, _UNSET extra_headers=self._get_with_precedence(extra_headers, self.extra_headers, _UNSET),
) disable_resources=self._get_with_precedence(disable_resources, self.disable_resources, _UNSET),
disable_resources = self._get_with_precedence( wait_selector=self._get_with_precedence(wait_selector, self.wait_selector, _UNSET),
disable_resources, self.disable_resources, _UNSET wait_selector_state=self._get_with_precedence(wait_selector_state, self.wait_selector_state, _UNSET),
) network_idle=self._get_with_precedence(network_idle, self.network_idle, _UNSET),
wait_selector = self._get_with_precedence( selector_config=self._get_with_precedence(selector_config, self.selector_config, _UNSET),
wait_selector, self.wait_selector, _UNSET ),
) PlaywrightConfig,
wait_selector_state = self._get_with_precedence(
wait_selector_state, self.wait_selector_state, _UNSET
)
network_idle = self._get_with_precedence(
network_idle, self.network_idle, _UNSET
)
selector_config = self._get_with_precedence(
selector_config, self.selector_config, _UNSET
) )
if self._closed: # pragma: no cover if self._closed: # pragma: no cover
@@ -258,9 +249,7 @@ class DynamicSession(DynamicSessionMixin, SyncSession):
final_response = None final_response = None
referer = ( referer = (
generate_convincing_referer(url) generate_convincing_referer(url) if (params.google_search and "referer" not in self._headers_keys) else None
if (google_search and "referer" not in self._headers_keys)
else None
) )
def handle_response(finished_response: SyncPlaywrightResponse): def handle_response(finished_response: SyncPlaywrightResponse):
@@ -271,7 +260,7 @@ class DynamicSession(DynamicSessionMixin, SyncSession):
): ):
final_response = finished_response final_response = finished_response
page_info = self._get_page(timeout, extra_headers, disable_resources) page_info = self._get_page(params.timeout, params.extra_headers, params.disable_resources)
page_info.mark_busy(url=url) page_info.mark_busy(url=url)
try: # pragma: no cover try: # pragma: no cover
@@ -280,35 +269,35 @@ class DynamicSession(DynamicSessionMixin, SyncSession):
first_response = page_info.page.goto(url, referer=referer) first_response = page_info.page.goto(url, referer=referer)
page_info.page.wait_for_load_state(state="domcontentloaded") page_info.page.wait_for_load_state(state="domcontentloaded")
if network_idle: if params.network_idle:
page_info.page.wait_for_load_state("networkidle") page_info.page.wait_for_load_state("networkidle")
if not first_response: if not first_response:
raise RuntimeError(f"Failed to get response for {url}") raise RuntimeError(f"Failed to get response for {url}")
if page_action is not None: if params.page_action:
try: try:
_ = page_action(page_info.page) _ = params.page_action(page_info.page)
except Exception as e: # pragma: no cover except Exception as e: # pragma: no cover
log.error(f"Error executing page_action: {e}") log.error(f"Error executing page_action: {e}")
if wait_selector: if params.wait_selector:
try: try:
waiter: Locator = page_info.page.locator(wait_selector) waiter: Locator = page_info.page.locator(params.wait_selector)
waiter.first.wait_for(state=wait_selector_state) waiter.first.wait_for(state=params.wait_selector_state)
# Wait again after waiting for the selector, helpful with protections like Cloudflare # Wait again after waiting for the selector, helpful with protections like Cloudflare
page_info.page.wait_for_load_state(state="load") page_info.page.wait_for_load_state(state="load")
page_info.page.wait_for_load_state(state="domcontentloaded") page_info.page.wait_for_load_state(state="domcontentloaded")
if network_idle: if params.network_idle:
page_info.page.wait_for_load_state("networkidle") page_info.page.wait_for_load_state("networkidle")
except Exception as e: # pragma: no cover except Exception as e: # pragma: no cover
log.error(f"Error waiting for selector {wait_selector}: {e}") log.error(f"Error waiting for selector {params.wait_selector}: {e}")
page_info.page.wait_for_timeout(wait) page_info.page.wait_for_timeout(params.wait)
# Create response object # Create response object
response = ResponseFactory.from_playwright_response( response = ResponseFactory.from_playwright_response(
page_info.page, first_response, final_response, selector_config page_info.page, first_response, final_response, params.selector_config
) )
# Mark the page as finished for next use # Mark the page as finished for next use
@@ -413,17 +402,11 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession):
self.playwright: AsyncPlaywright = await async_context().start() self.playwright: AsyncPlaywright = await async_context().start()
if self.cdp_url: if self.cdp_url:
browser = await self.playwright.chromium.connect_over_cdp( browser = await self.playwright.chromium.connect_over_cdp(endpoint_url=self.cdp_url)
endpoint_url=self.cdp_url self.context: AsyncBrowserContext = await browser.new_context(**self.context_options)
)
self.context: AsyncBrowserContext = await browser.new_context(
**self.context_options
)
else: else:
self.context: AsyncBrowserContext = ( self.context: AsyncBrowserContext = await self.playwright.chromium.launch_persistent_context(
await self.playwright.chromium.launch_persistent_context( user_data_dir="", **self.launch_options
user_data_dir="", **self.launch_options
)
) )
# Get the default page and close it # Get the default page and close it
@@ -489,29 +472,21 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession):
:param selector_config: The arguments that will be passed in the end while creating the final Selector's class. :param selector_config: The arguments that will be passed in the end while creating the final Selector's class.
:return: A `Response` object. :return: A `Response` object.
""" """
google_search = self._get_with_precedence( # Validate all resolved parameters
google_search, self.google_search, _UNSET params = validate(
) dict(
timeout = self._get_with_precedence(timeout, self.timeout, _UNSET) google_search=self._get_with_precedence(google_search, self.google_search, _UNSET),
wait = self._get_with_precedence(wait, self.wait, _UNSET) timeout=self._get_with_precedence(timeout, self.timeout, _UNSET),
page_action = self._get_with_precedence(page_action, self.page_action, _UNSET) wait=self._get_with_precedence(wait, self.wait, _UNSET),
extra_headers = self._get_with_precedence( page_action=self._get_with_precedence(page_action, self.page_action, _UNSET),
extra_headers, self.extra_headers, _UNSET extra_headers=self._get_with_precedence(extra_headers, self.extra_headers, _UNSET),
) disable_resources=self._get_with_precedence(disable_resources, self.disable_resources, _UNSET),
disable_resources = self._get_with_precedence( wait_selector=self._get_with_precedence(wait_selector, self.wait_selector, _UNSET),
disable_resources, self.disable_resources, _UNSET wait_selector_state=self._get_with_precedence(wait_selector_state, self.wait_selector_state, _UNSET),
) network_idle=self._get_with_precedence(network_idle, self.network_idle, _UNSET),
wait_selector = self._get_with_precedence( selector_config=self._get_with_precedence(selector_config, self.selector_config, _UNSET),
wait_selector, self.wait_selector, _UNSET ),
) PlaywrightConfig,
wait_selector_state = self._get_with_precedence(
wait_selector_state, self.wait_selector_state, _UNSET
)
network_idle = self._get_with_precedence(
network_idle, self.network_idle, _UNSET
)
selector_config = self._get_with_precedence(
selector_config, self.selector_config, _UNSET
) )
if self._closed: # pragma: no cover if self._closed: # pragma: no cover
@@ -519,9 +494,7 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession):
final_response = None final_response = None
referer = ( referer = (
generate_convincing_referer(url) generate_convincing_referer(url) if (params.google_search and "referer" not in self._headers_keys) else None
if (google_search and "referer" not in self._headers_keys)
else None
) )
async def handle_response(finished_response: AsyncPlaywrightResponse): async def handle_response(finished_response: AsyncPlaywrightResponse):
@@ -532,7 +505,7 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession):
): ):
final_response = finished_response final_response = finished_response
page_info = await self._get_page(timeout, extra_headers, disable_resources) page_info = await self._get_page(params.timeout, params.extra_headers, params.disable_resources)
page_info.mark_busy(url=url) page_info.mark_busy(url=url)
try: try:
@@ -541,35 +514,35 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession):
first_response = await page_info.page.goto(url, referer=referer) first_response = await page_info.page.goto(url, referer=referer)
await page_info.page.wait_for_load_state(state="domcontentloaded") await page_info.page.wait_for_load_state(state="domcontentloaded")
if network_idle: if params.network_idle:
await page_info.page.wait_for_load_state("networkidle") await page_info.page.wait_for_load_state("networkidle")
if not first_response: if not first_response:
raise RuntimeError(f"Failed to get response for {url}") raise RuntimeError(f"Failed to get response for {url}")
if page_action is not None: if params.page_action:
try: try:
_ = await page_action(page_info.page) _ = await params.page_action(page_info.page)
except Exception as e: except Exception as e:
log.error(f"Error executing page_action: {e}") log.error(f"Error executing page_action: {e}")
if wait_selector: if params.wait_selector:
try: try:
waiter: AsyncLocator = page_info.page.locator(wait_selector) waiter: AsyncLocator = page_info.page.locator(params.wait_selector)
await waiter.first.wait_for(state=wait_selector_state) await waiter.first.wait_for(state=params.wait_selector_state)
# Wait again after waiting for the selector, helpful with protections like Cloudflare # Wait again after waiting for the selector, helpful with protections like Cloudflare
await page_info.page.wait_for_load_state(state="load") await page_info.page.wait_for_load_state(state="load")
await page_info.page.wait_for_load_state(state="domcontentloaded") await page_info.page.wait_for_load_state(state="domcontentloaded")
if network_idle: if params.network_idle:
await page_info.page.wait_for_load_state("networkidle") await page_info.page.wait_for_load_state("networkidle")
except Exception as e: except Exception as e:
log.error(f"Error waiting for selector {wait_selector}: {e}") log.error(f"Error waiting for selector {params.wait_selector}: {e}")
await page_info.page.wait_for_timeout(wait) await page_info.page.wait_for_timeout(params.wait)
# Create response object # Create response object
response = await ResponseFactory.from_async_playwright_response( response = await ResponseFactory.from_async_playwright_response(
page_info.page, first_response, final_response, selector_config page_info.page, first_response, final_response, params.selector_config
) )
# Mark the page as finished for next use # Mark the page as finished for next use