feat: Validate all fetch-level parameters
So it's validated like the session-level ones. Needs improvement, but that's for later
This commit is contained in:
Karim shoair
@@ -15,6 +15,7 @@ from playwright.async_api import (
|
||||
Page as async_Page,
|
||||
)
|
||||
|
||||
from ._validators import validate, CamoufoxConfig
|
||||
from ._base import SyncSession, AsyncSession, StealthySessionMixin
|
||||
from scrapling.core.utils import log
|
||||
from scrapling.core._types import (
|
||||
@@ -164,10 +165,8 @@ class StealthySession(StealthySessionMixin, SyncSession):
|
||||
def __create__(self):
|
||||
"""Create a browser for this instance and context."""
|
||||
self.playwright = sync_playwright().start()
|
||||
self.context = (
|
||||
self.playwright.firefox.launch_persistent_context( # pragma: no cover
|
||||
**self.launch_options
|
||||
)
|
||||
self.context = self.playwright.firefox.launch_persistent_context( # pragma: no cover
|
||||
**self.launch_options
|
||||
)
|
||||
|
||||
# Get the default page and close it
|
||||
@@ -281,32 +280,22 @@ class StealthySession(StealthySessionMixin, SyncSession):
|
||||
:param selector_config: The arguments that will be passed in the end while creating the final Selector's class.
|
||||
:return: A `Response` object.
|
||||
"""
|
||||
google_search = self._get_with_precedence(
|
||||
google_search, self.google_search, _UNSET
|
||||
)
|
||||
timeout = self._get_with_precedence(timeout, self.timeout, _UNSET)
|
||||
wait = self._get_with_precedence(wait, self.wait, _UNSET)
|
||||
page_action = self._get_with_precedence(page_action, self.page_action, _UNSET)
|
||||
extra_headers = self._get_with_precedence(
|
||||
extra_headers, self.extra_headers, _UNSET
|
||||
)
|
||||
disable_resources = self._get_with_precedence(
|
||||
disable_resources, self.disable_resources, _UNSET
|
||||
)
|
||||
wait_selector = self._get_with_precedence(
|
||||
wait_selector, self.wait_selector, _UNSET
|
||||
)
|
||||
wait_selector_state = self._get_with_precedence(
|
||||
wait_selector_state, self.wait_selector_state, _UNSET
|
||||
)
|
||||
network_idle = self._get_with_precedence(
|
||||
network_idle, self.network_idle, _UNSET
|
||||
)
|
||||
solve_cloudflare = self._get_with_precedence(
|
||||
solve_cloudflare, self.solve_cloudflare, _UNSET
|
||||
)
|
||||
selector_config = self._get_with_precedence(
|
||||
selector_config, self.selector_config, _UNSET
|
||||
# Validate all resolved parameters
|
||||
params = validate(
|
||||
dict(
|
||||
google_search=self._get_with_precedence(google_search, self.google_search, _UNSET),
|
||||
timeout=self._get_with_precedence(timeout, self.timeout, _UNSET),
|
||||
wait=self._get_with_precedence(wait, self.wait, _UNSET),
|
||||
page_action=self._get_with_precedence(page_action, self.page_action, _UNSET),
|
||||
extra_headers=self._get_with_precedence(extra_headers, self.extra_headers, _UNSET),
|
||||
disable_resources=self._get_with_precedence(disable_resources, self.disable_resources, _UNSET),
|
||||
wait_selector=self._get_with_precedence(wait_selector, self.wait_selector, _UNSET),
|
||||
wait_selector_state=self._get_with_precedence(wait_selector_state, self.wait_selector_state, _UNSET),
|
||||
network_idle=self._get_with_precedence(network_idle, self.network_idle, _UNSET),
|
||||
solve_cloudflare=self._get_with_precedence(solve_cloudflare, self.solve_cloudflare, _UNSET),
|
||||
selector_config=self._get_with_precedence(selector_config, self.selector_config, _UNSET),
|
||||
),
|
||||
CamoufoxConfig,
|
||||
)
|
||||
|
||||
if self._closed: # pragma: no cover
|
||||
@@ -314,9 +303,7 @@ class StealthySession(StealthySessionMixin, SyncSession):
|
||||
|
||||
final_response = None
|
||||
referer = (
|
||||
generate_convincing_referer(url)
|
||||
if (google_search and "referer" not in self._headers_keys)
|
||||
else None
|
||||
generate_convincing_referer(url) if (params.google_search and "referer" not in self._headers_keys) else None
|
||||
)
|
||||
|
||||
def handle_response(finished_response: SyncPlaywrightResponse):
|
||||
@@ -327,7 +314,7 @@ class StealthySession(StealthySessionMixin, SyncSession):
|
||||
):
|
||||
final_response = finished_response
|
||||
|
||||
page_info = self._get_page(timeout, extra_headers, disable_resources)
|
||||
page_info = self._get_page(params.timeout, params.extra_headers, params.disable_resources)
|
||||
page_info.mark_busy(url=url)
|
||||
|
||||
try: # pragma: no cover
|
||||
@@ -336,41 +323,41 @@ class StealthySession(StealthySessionMixin, SyncSession):
|
||||
first_response = page_info.page.goto(url, referer=referer)
|
||||
page_info.page.wait_for_load_state(state="domcontentloaded")
|
||||
|
||||
if network_idle:
|
||||
if params.network_idle:
|
||||
page_info.page.wait_for_load_state("networkidle")
|
||||
|
||||
if not first_response:
|
||||
raise RuntimeError(f"Failed to get response for {url}")
|
||||
|
||||
if solve_cloudflare:
|
||||
if params.solve_cloudflare:
|
||||
self._solve_cloudflare(page_info.page)
|
||||
# Make sure the page is fully loaded after the captcha
|
||||
page_info.page.wait_for_load_state(state="load")
|
||||
page_info.page.wait_for_load_state(state="domcontentloaded")
|
||||
if network_idle:
|
||||
if params.network_idle:
|
||||
page_info.page.wait_for_load_state("networkidle")
|
||||
|
||||
if page_action is not None:
|
||||
if params.page_action:
|
||||
try:
|
||||
_ = page_action(page_info.page)
|
||||
_ = params.page_action(page_info.page)
|
||||
except Exception as e:
|
||||
log.error(f"Error executing page_action: {e}")
|
||||
|
||||
if wait_selector:
|
||||
if params.wait_selector:
|
||||
try:
|
||||
waiter: Locator = page_info.page.locator(wait_selector)
|
||||
waiter.first.wait_for(state=wait_selector_state)
|
||||
waiter: Locator = page_info.page.locator(params.wait_selector)
|
||||
waiter.first.wait_for(state=params.wait_selector_state)
|
||||
# Wait again after waiting for the selector, helpful with protections like Cloudflare
|
||||
page_info.page.wait_for_load_state(state="load")
|
||||
page_info.page.wait_for_load_state(state="domcontentloaded")
|
||||
if network_idle:
|
||||
if params.network_idle:
|
||||
page_info.page.wait_for_load_state("networkidle")
|
||||
except Exception as e:
|
||||
log.error(f"Error waiting for selector {wait_selector}: {e}")
|
||||
log.error(f"Error waiting for selector {params.wait_selector}: {e}")
|
||||
|
||||
page_info.page.wait_for_timeout(wait)
|
||||
page_info.page.wait_for_timeout(params.wait)
|
||||
response = ResponseFactory.from_playwright_response(
|
||||
page_info.page, first_response, final_response, selector_config
|
||||
page_info.page, first_response, final_response, params.selector_config
|
||||
)
|
||||
|
||||
# Mark the page as finished for next use
|
||||
@@ -478,10 +465,8 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):
|
||||
async def __create__(self):
|
||||
"""Create a browser for this instance and context."""
|
||||
self.playwright: AsyncPlaywright = await async_playwright().start()
|
||||
self.context: AsyncBrowserContext = (
|
||||
await self.playwright.firefox.launch_persistent_context(
|
||||
**self.launch_options
|
||||
)
|
||||
self.context: AsyncBrowserContext = await self.playwright.firefox.launch_persistent_context(
|
||||
**self.launch_options
|
||||
)
|
||||
|
||||
# Get the default page and close it
|
||||
@@ -551,9 +536,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):
|
||||
await page.wait_for_timeout(500)
|
||||
|
||||
# Calculate the Captcha coordinates for any viewport
|
||||
outer_box = await page.locator(
|
||||
".main-content p+div>div>div"
|
||||
).bounding_box()
|
||||
outer_box = await page.locator(".main-content p+div>div>div").bounding_box()
|
||||
captcha_x, captcha_y = outer_box["x"] + 26, outer_box["y"] + 25
|
||||
|
||||
# Move the mouse to the center of the window, then press and hold the left mouse button
|
||||
@@ -597,32 +580,21 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):
|
||||
:param selector_config: The arguments that will be passed in the end while creating the final Selector's class.
|
||||
:return: A `Response` object.
|
||||
"""
|
||||
google_search = self._get_with_precedence(
|
||||
google_search, self.google_search, _UNSET
|
||||
)
|
||||
timeout = self._get_with_precedence(timeout, self.timeout, _UNSET)
|
||||
wait = self._get_with_precedence(wait, self.wait, _UNSET)
|
||||
page_action = self._get_with_precedence(page_action, self.page_action, _UNSET)
|
||||
extra_headers = self._get_with_precedence(
|
||||
extra_headers, self.extra_headers, _UNSET
|
||||
)
|
||||
disable_resources = self._get_with_precedence(
|
||||
disable_resources, self.disable_resources, _UNSET
|
||||
)
|
||||
wait_selector = self._get_with_precedence(
|
||||
wait_selector, self.wait_selector, _UNSET
|
||||
)
|
||||
wait_selector_state = self._get_with_precedence(
|
||||
wait_selector_state, self.wait_selector_state, _UNSET
|
||||
)
|
||||
network_idle = self._get_with_precedence(
|
||||
network_idle, self.network_idle, _UNSET
|
||||
)
|
||||
solve_cloudflare = self._get_with_precedence(
|
||||
solve_cloudflare, self.solve_cloudflare, _UNSET
|
||||
)
|
||||
selector_config = self._get_with_precedence(
|
||||
selector_config, self.selector_config, _UNSET
|
||||
params = validate(
|
||||
dict(
|
||||
google_search=self._get_with_precedence(google_search, self.google_search, _UNSET),
|
||||
timeout=self._get_with_precedence(timeout, self.timeout, _UNSET),
|
||||
wait=self._get_with_precedence(wait, self.wait, _UNSET),
|
||||
page_action=self._get_with_precedence(page_action, self.page_action, _UNSET),
|
||||
extra_headers=self._get_with_precedence(extra_headers, self.extra_headers, _UNSET),
|
||||
disable_resources=self._get_with_precedence(disable_resources, self.disable_resources, _UNSET),
|
||||
wait_selector=self._get_with_precedence(wait_selector, self.wait_selector, _UNSET),
|
||||
wait_selector_state=self._get_with_precedence(wait_selector_state, self.wait_selector_state, _UNSET),
|
||||
network_idle=self._get_with_precedence(network_idle, self.network_idle, _UNSET),
|
||||
solve_cloudflare=self._get_with_precedence(solve_cloudflare, self.solve_cloudflare, _UNSET),
|
||||
selector_config=self._get_with_precedence(selector_config, self.selector_config, _UNSET),
|
||||
),
|
||||
CamoufoxConfig,
|
||||
)
|
||||
|
||||
if self._closed: # pragma: no cover
|
||||
@@ -630,9 +602,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):
|
||||
|
||||
final_response = None
|
||||
referer = (
|
||||
generate_convincing_referer(url)
|
||||
if (google_search and "referer" not in self._headers_keys)
|
||||
else None
|
||||
generate_convincing_referer(url) if (params.google_search and "referer" not in self._headers_keys) else None
|
||||
)
|
||||
|
||||
async def handle_response(finished_response: AsyncPlaywrightResponse):
|
||||
@@ -643,7 +613,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):
|
||||
):
|
||||
final_response = finished_response
|
||||
|
||||
page_info = await self._get_page(timeout, extra_headers, disable_resources)
|
||||
page_info = await self._get_page(params.timeout, params.extra_headers, params.disable_resources)
|
||||
page_info.mark_busy(url=url)
|
||||
|
||||
try:
|
||||
@@ -652,43 +622,43 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):
|
||||
first_response = await page_info.page.goto(url, referer=referer)
|
||||
await page_info.page.wait_for_load_state(state="domcontentloaded")
|
||||
|
||||
if network_idle:
|
||||
if params.network_idle:
|
||||
await page_info.page.wait_for_load_state("networkidle")
|
||||
|
||||
if not first_response:
|
||||
raise RuntimeError(f"Failed to get response for {url}")
|
||||
|
||||
if solve_cloudflare:
|
||||
if params.solve_cloudflare:
|
||||
await self._solve_cloudflare(page_info.page)
|
||||
# Make sure the page is fully loaded after the captcha
|
||||
await page_info.page.wait_for_load_state(state="load")
|
||||
await page_info.page.wait_for_load_state(state="domcontentloaded")
|
||||
if network_idle:
|
||||
if params.network_idle:
|
||||
await page_info.page.wait_for_load_state("networkidle")
|
||||
|
||||
if page_action is not None:
|
||||
if params.page_action:
|
||||
try:
|
||||
_ = await page_action(page_info.page)
|
||||
_ = await params.page_action(page_info.page)
|
||||
except Exception as e:
|
||||
log.error(f"Error executing page_action: {e}")
|
||||
|
||||
if wait_selector:
|
||||
if params.wait_selector:
|
||||
try:
|
||||
waiter: AsyncLocator = page_info.page.locator(wait_selector)
|
||||
await waiter.first.wait_for(state=wait_selector_state)
|
||||
waiter: AsyncLocator = page_info.page.locator(params.wait_selector)
|
||||
await waiter.first.wait_for(state=params.wait_selector_state)
|
||||
# Wait again after waiting for the selector, helpful with protections like Cloudflare
|
||||
await page_info.page.wait_for_load_state(state="load")
|
||||
await page_info.page.wait_for_load_state(state="domcontentloaded")
|
||||
if network_idle:
|
||||
if params.network_idle:
|
||||
await page_info.page.wait_for_load_state("networkidle")
|
||||
except Exception as e:
|
||||
log.error(f"Error waiting for selector {wait_selector}: {e}")
|
||||
log.error(f"Error waiting for selector {params.wait_selector}: {e}")
|
||||
|
||||
await page_info.page.wait_for_timeout(wait)
|
||||
await page_info.page.wait_for_timeout(params.wait)
|
||||
|
||||
# Create response object
|
||||
response = await ResponseFactory.from_async_playwright_response(
|
||||
page_info.page, first_response, final_response, selector_config
|
||||
page_info.page, first_response, final_response, params.selector_config
|
||||
)
|
||||
|
||||
# Mark the page as finished for next use
|
||||
|
||||
Reference in New Issue
Block a user