freedak/Scrapling
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Validate all fetch-level parameters

Browse Source 
So it's validated like the session-level ones. Needs improvement, but that's for later
This commit is contained in:
Karim shoair
2025-09-13 03:22:08 +03:00
parent 831eafca34
commit 60be9dc816
2 changed files with 128 additions and 185 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scrapling/engines/_browsers/_camoufox.py
+65 -95
View File
@@ -15,6 +15,7 @@ from playwright.async_api import (
Page as async_Page,
)
from ._validators import validate, CamoufoxConfig
from ._base import SyncSession, AsyncSession, StealthySessionMixin
from scrapling.core.utils import log
from scrapling.core._types import (
@@ -164,10 +165,8 @@ class StealthySession(StealthySessionMixin, SyncSession):
def __create__(self):
"""Create a browser for this instance and context."""
self.playwright = sync_playwright().start()
self.context = (
self.playwright.firefox.launch_persistent_context( # pragma: no cover
**self.launch_options
)
self.context = self.playwright.firefox.launch_persistent_context( # pragma: no cover
**self.launch_options
)
# Get the default page and close it
@@ -281,32 +280,22 @@ class StealthySession(StealthySessionMixin, SyncSession):
:param selector_config: The arguments that will be passed in the end while creating the final Selector's class.
:return: A `Response` object.
"""
google_search = self._get_with_precedence(
google_search, self.google_search, _UNSET
)
timeout = self._get_with_precedence(timeout, self.timeout, _UNSET)
wait = self._get_with_precedence(wait, self.wait, _UNSET)
page_action = self._get_with_precedence(page_action, self.page_action, _UNSET)
extra_headers = self._get_with_precedence(
extra_headers, self.extra_headers, _UNSET
)
disable_resources = self._get_with_precedence(
disable_resources, self.disable_resources, _UNSET
)
wait_selector = self._get_with_precedence(
wait_selector, self.wait_selector, _UNSET
)
wait_selector_state = self._get_with_precedence(
wait_selector_state, self.wait_selector_state, _UNSET
)
network_idle = self._get_with_precedence(
network_idle, self.network_idle, _UNSET
)
solve_cloudflare = self._get_with_precedence(
solve_cloudflare, self.solve_cloudflare, _UNSET
)
selector_config = self._get_with_precedence(
selector_config, self.selector_config, _UNSET
# Validate all resolved parameters
params = validate(
dict(
google_search=self._get_with_precedence(google_search, self.google_search, _UNSET),
timeout=self._get_with_precedence(timeout, self.timeout, _UNSET),
wait=self._get_with_precedence(wait, self.wait, _UNSET),
page_action=self._get_with_precedence(page_action, self.page_action, _UNSET),
extra_headers=self._get_with_precedence(extra_headers, self.extra_headers, _UNSET),
disable_resources=self._get_with_precedence(disable_resources, self.disable_resources, _UNSET),
wait_selector=self._get_with_precedence(wait_selector, self.wait_selector, _UNSET),
wait_selector_state=self._get_with_precedence(wait_selector_state, self.wait_selector_state, _UNSET),
network_idle=self._get_with_precedence(network_idle, self.network_idle, _UNSET),
solve_cloudflare=self._get_with_precedence(solve_cloudflare, self.solve_cloudflare, _UNSET),
selector_config=self._get_with_precedence(selector_config, self.selector_config, _UNSET),
),
CamoufoxConfig,
)
if self._closed: # pragma: no cover
@@ -314,9 +303,7 @@ class StealthySession(StealthySessionMixin, SyncSession):
final_response = None
referer = (
generate_convincing_referer(url)
if (google_search and "referer" not in self._headers_keys)
else None
generate_convincing_referer(url) if (params.google_search and "referer" not in self._headers_keys) else None
)
def handle_response(finished_response: SyncPlaywrightResponse):
@@ -327,7 +314,7 @@ class StealthySession(StealthySessionMixin, SyncSession):
):
final_response = finished_response
page_info = self._get_page(timeout, extra_headers, disable_resources)
page_info = self._get_page(params.timeout, params.extra_headers, params.disable_resources)
page_info.mark_busy(url=url)
try: # pragma: no cover
@@ -336,41 +323,41 @@ class StealthySession(StealthySessionMixin, SyncSession):
first_response = page_info.page.goto(url, referer=referer)
page_info.page.wait_for_load_state(state="domcontentloaded")
if network_idle:
if params.network_idle:
page_info.page.wait_for_load_state("networkidle")
if not first_response:
raise RuntimeError(f"Failed to get response for {url}")
if solve_cloudflare:
if params.solve_cloudflare:
self._solve_cloudflare(page_info.page)
# Make sure the page is fully loaded after the captcha
page_info.page.wait_for_load_state(state="load")
page_info.page.wait_for_load_state(state="domcontentloaded")
if network_idle:
if params.network_idle:
page_info.page.wait_for_load_state("networkidle")
if page_action is not None:
if params.page_action:
try:
_ = page_action(page_info.page)
_ = params.page_action(page_info.page)
except Exception as e:
log.error(f"Error executing page_action: {e}")
if wait_selector:
if params.wait_selector:
try:
waiter: Locator = page_info.page.locator(wait_selector)
waiter.first.wait_for(state=wait_selector_state)
waiter: Locator = page_info.page.locator(params.wait_selector)
waiter.first.wait_for(state=params.wait_selector_state)
# Wait again after waiting for the selector, helpful with protections like Cloudflare
page_info.page.wait_for_load_state(state="load")
page_info.page.wait_for_load_state(state="domcontentloaded")
if network_idle:
if params.network_idle:
page_info.page.wait_for_load_state("networkidle")
except Exception as e:
log.error(f"Error waiting for selector {wait_selector}: {e}")
log.error(f"Error waiting for selector {params.wait_selector}: {e}")
page_info.page.wait_for_timeout(wait)
page_info.page.wait_for_timeout(params.wait)
response = ResponseFactory.from_playwright_response(
page_info.page, first_response, final_response, selector_config
page_info.page, first_response, final_response, params.selector_config
)
# Mark the page as finished for next use
@@ -478,10 +465,8 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):
async def __create__(self):
"""Create a browser for this instance and context."""
self.playwright: AsyncPlaywright = await async_playwright().start()
self.context: AsyncBrowserContext = (
await self.playwright.firefox.launch_persistent_context(
**self.launch_options
)
self.context: AsyncBrowserContext = await self.playwright.firefox.launch_persistent_context(
**self.launch_options
)
# Get the default page and close it
@@ -551,9 +536,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):
await page.wait_for_timeout(500)
# Calculate the Captcha coordinates for any viewport
outer_box = await page.locator(
".main-content p+div>div>div"
).bounding_box()
outer_box = await page.locator(".main-content p+div>div>div").bounding_box()
captcha_x, captcha_y = outer_box["x"] + 26, outer_box["y"] + 25
# Move the mouse to the center of the window, then press and hold the left mouse button
@@ -597,32 +580,21 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):
:param selector_config: The arguments that will be passed in the end while creating the final Selector's class.
:return: A `Response` object.
"""
google_search = self._get_with_precedence(
google_search, self.google_search, _UNSET
)
timeout = self._get_with_precedence(timeout, self.timeout, _UNSET)
wait = self._get_with_precedence(wait, self.wait, _UNSET)
page_action = self._get_with_precedence(page_action, self.page_action, _UNSET)
extra_headers = self._get_with_precedence(
extra_headers, self.extra_headers, _UNSET
)
disable_resources = self._get_with_precedence(
disable_resources, self.disable_resources, _UNSET
)
wait_selector = self._get_with_precedence(
wait_selector, self.wait_selector, _UNSET
)
wait_selector_state = self._get_with_precedence(
wait_selector_state, self.wait_selector_state, _UNSET
)
network_idle = self._get_with_precedence(
network_idle, self.network_idle, _UNSET
)
solve_cloudflare = self._get_with_precedence(
solve_cloudflare, self.solve_cloudflare, _UNSET
)
selector_config = self._get_with_precedence(
selector_config, self.selector_config, _UNSET
params = validate(
dict(
google_search=self._get_with_precedence(google_search, self.google_search, _UNSET),
timeout=self._get_with_precedence(timeout, self.timeout, _UNSET),
wait=self._get_with_precedence(wait, self.wait, _UNSET),
page_action=self._get_with_precedence(page_action, self.page_action, _UNSET),
extra_headers=self._get_with_precedence(extra_headers, self.extra_headers, _UNSET),
disable_resources=self._get_with_precedence(disable_resources, self.disable_resources, _UNSET),
wait_selector=self._get_with_precedence(wait_selector, self.wait_selector, _UNSET),
wait_selector_state=self._get_with_precedence(wait_selector_state, self.wait_selector_state, _UNSET),
network_idle=self._get_with_precedence(network_idle, self.network_idle, _UNSET),
solve_cloudflare=self._get_with_precedence(solve_cloudflare, self.solve_cloudflare, _UNSET),
selector_config=self._get_with_precedence(selector_config, self.selector_config, _UNSET),
),
CamoufoxConfig,
)
if self._closed: # pragma: no cover
@@ -630,9 +602,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):
final_response = None
referer = (
generate_convincing_referer(url)
if (google_search and "referer" not in self._headers_keys)
else None
generate_convincing_referer(url) if (params.google_search and "referer" not in self._headers_keys) else None
)
async def handle_response(finished_response: AsyncPlaywrightResponse):
@@ -643,7 +613,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):
):
final_response = finished_response
page_info = await self._get_page(timeout, extra_headers, disable_resources)
page_info = await self._get_page(params.timeout, params.extra_headers, params.disable_resources)
page_info.mark_busy(url=url)
try:
@@ -652,43 +622,43 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):
first_response = await page_info.page.goto(url, referer=referer)
await page_info.page.wait_for_load_state(state="domcontentloaded")
if network_idle:
if params.network_idle:
await page_info.page.wait_for_load_state("networkidle")
if not first_response:
raise RuntimeError(f"Failed to get response for {url}")
if solve_cloudflare:
if params.solve_cloudflare:
await self._solve_cloudflare(page_info.page)
# Make sure the page is fully loaded after the captcha
await page_info.page.wait_for_load_state(state="load")
await page_info.page.wait_for_load_state(state="domcontentloaded")
if network_idle:
if params.network_idle:
await page_info.page.wait_for_load_state("networkidle")
if page_action is not None:
if params.page_action:
try:
_ = await page_action(page_info.page)
_ = await params.page_action(page_info.page)
except Exception as e:
log.error(f"Error executing page_action: {e}")
if wait_selector:
if params.wait_selector:
try:
waiter: AsyncLocator = page_info.page.locator(wait_selector)
await waiter.first.wait_for(state=wait_selector_state)
waiter: AsyncLocator = page_info.page.locator(params.wait_selector)
await waiter.first.wait_for(state=params.wait_selector_state)
# Wait again after waiting for the selector, helpful with protections like Cloudflare
await page_info.page.wait_for_load_state(state="load")
await page_info.page.wait_for_load_state(state="domcontentloaded")
if network_idle:
if params.network_idle:
await page_info.page.wait_for_load_state("networkidle")
except Exception as e:
log.error(f"Error waiting for selector {wait_selector}: {e}")
log.error(f"Error waiting for selector {params.wait_selector}: {e}")
await page_info.page.wait_for_timeout(wait)
await page_info.page.wait_for_timeout(params.wait)
# Create response object
response = await ResponseFactory.from_async_playwright_response(
page_info.page, first_response, final_response, selector_config
page_info.page, first_response, final_response, params.selector_config
)
# Mark the page as finished for next use
scrapling/engines/_browsers/_controllers.py
+63 -90
View File
@@ -18,6 +18,7 @@ from rebrowser_playwright.async_api import (
from scrapling.core.utils import log
from ._base import SyncSession, AsyncSession, DynamicSessionMixin
from ._validators import validate, PlaywrightConfig
from scrapling.core._types import (
Dict,
List,
@@ -157,13 +158,11 @@ class DynamicSession(DynamicSessionMixin, SyncSession):
self.playwright: Playwright = sync_context().start()
if self.cdp_url: # pragma: no cover
self.context = self.playwright.chromium.connect_over_cdp(
endpoint_url=self.cdp_url
).new_context(**self.context_options)
else:
self.context = self.playwright.chromium.launch_persistent_context(
user_data_dir="", **self.launch_options
self.context = self.playwright.chromium.connect_over_cdp(endpoint_url=self.cdp_url).new_context(
**self.context_options
)
else:
self.context = self.playwright.chromium.launch_persistent_context(user_data_dir="", **self.launch_options)
# Get the default page and close it
default_page = self.context.pages[0]
@@ -228,29 +227,21 @@ class DynamicSession(DynamicSessionMixin, SyncSession):
:param selector_config: The arguments that will be passed in the end while creating the final Selector's class.
:return: A `Response` object.
"""
google_search = self._get_with_precedence(
google_search, self.google_search, _UNSET
)
timeout = self._get_with_precedence(timeout, self.timeout, _UNSET)
wait = self._get_with_precedence(wait, self.wait, _UNSET)
page_action = self._get_with_precedence(page_action, self.page_action, _UNSET)
extra_headers = self._get_with_precedence(
extra_headers, self.extra_headers, _UNSET
)
disable_resources = self._get_with_precedence(
disable_resources, self.disable_resources, _UNSET
)
wait_selector = self._get_with_precedence(
wait_selector, self.wait_selector, _UNSET
)
wait_selector_state = self._get_with_precedence(
wait_selector_state, self.wait_selector_state, _UNSET
)
network_idle = self._get_with_precedence(
network_idle, self.network_idle, _UNSET
)
selector_config = self._get_with_precedence(
selector_config, self.selector_config, _UNSET
# Validate all resolved parameters
params = validate(
dict(
google_search=self._get_with_precedence(google_search, self.google_search, _UNSET),
timeout=self._get_with_precedence(timeout, self.timeout, _UNSET),
wait=self._get_with_precedence(wait, self.wait, _UNSET),
page_action=self._get_with_precedence(page_action, self.page_action, _UNSET),
extra_headers=self._get_with_precedence(extra_headers, self.extra_headers, _UNSET),
disable_resources=self._get_with_precedence(disable_resources, self.disable_resources, _UNSET),
wait_selector=self._get_with_precedence(wait_selector, self.wait_selector, _UNSET),
wait_selector_state=self._get_with_precedence(wait_selector_state, self.wait_selector_state, _UNSET),
network_idle=self._get_with_precedence(network_idle, self.network_idle, _UNSET),
selector_config=self._get_with_precedence(selector_config, self.selector_config, _UNSET),
),
PlaywrightConfig,
)
if self._closed: # pragma: no cover
@@ -258,9 +249,7 @@ class DynamicSession(DynamicSessionMixin, SyncSession):
final_response = None
referer = (
generate_convincing_referer(url)
if (google_search and "referer" not in self._headers_keys)
else None
generate_convincing_referer(url) if (params.google_search and "referer" not in self._headers_keys) else None
)
def handle_response(finished_response: SyncPlaywrightResponse):
@@ -271,7 +260,7 @@ class DynamicSession(DynamicSessionMixin, SyncSession):
):
final_response = finished_response
page_info = self._get_page(timeout, extra_headers, disable_resources)
page_info = self._get_page(params.timeout, params.extra_headers, params.disable_resources)
page_info.mark_busy(url=url)
try: # pragma: no cover
@@ -280,35 +269,35 @@ class DynamicSession(DynamicSessionMixin, SyncSession):
first_response = page_info.page.goto(url, referer=referer)
page_info.page.wait_for_load_state(state="domcontentloaded")
if network_idle:
if params.network_idle:
page_info.page.wait_for_load_state("networkidle")
if not first_response:
raise RuntimeError(f"Failed to get response for {url}")
if page_action is not None:
if params.page_action:
try:
_ = page_action(page_info.page)
_ = params.page_action(page_info.page)
except Exception as e: # pragma: no cover
log.error(f"Error executing page_action: {e}")
if wait_selector:
if params.wait_selector:
try:
waiter: Locator = page_info.page.locator(wait_selector)
waiter.first.wait_for(state=wait_selector_state)
waiter: Locator = page_info.page.locator(params.wait_selector)
waiter.first.wait_for(state=params.wait_selector_state)
# Wait again after waiting for the selector, helpful with protections like Cloudflare
page_info.page.wait_for_load_state(state="load")
page_info.page.wait_for_load_state(state="domcontentloaded")
if network_idle:
if params.network_idle:
page_info.page.wait_for_load_state("networkidle")
except Exception as e: # pragma: no cover
log.error(f"Error waiting for selector {wait_selector}: {e}")
log.error(f"Error waiting for selector {params.wait_selector}: {e}")
page_info.page.wait_for_timeout(wait)
page_info.page.wait_for_timeout(params.wait)
# Create response object
response = ResponseFactory.from_playwright_response(
page_info.page, first_response, final_response, selector_config
page_info.page, first_response, final_response, params.selector_config
)
# Mark the page as finished for next use
@@ -413,17 +402,11 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession):
self.playwright: AsyncPlaywright = await async_context().start()
if self.cdp_url:
browser = await self.playwright.chromium.connect_over_cdp(
endpoint_url=self.cdp_url
)
self.context: AsyncBrowserContext = await browser.new_context(
**self.context_options
)
browser = await self.playwright.chromium.connect_over_cdp(endpoint_url=self.cdp_url)
self.context: AsyncBrowserContext = await browser.new_context(**self.context_options)
else:
self.context: AsyncBrowserContext = (
await self.playwright.chromium.launch_persistent_context(
user_data_dir="", **self.launch_options
)
self.context: AsyncBrowserContext = await self.playwright.chromium.launch_persistent_context(
user_data_dir="", **self.launch_options
)
# Get the default page and close it
@@ -489,29 +472,21 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession):
:param selector_config: The arguments that will be passed in the end while creating the final Selector's class.
:return: A `Response` object.
"""
google_search = self._get_with_precedence(
google_search, self.google_search, _UNSET
)
timeout = self._get_with_precedence(timeout, self.timeout, _UNSET)
wait = self._get_with_precedence(wait, self.wait, _UNSET)
page_action = self._get_with_precedence(page_action, self.page_action, _UNSET)
extra_headers = self._get_with_precedence(
extra_headers, self.extra_headers, _UNSET
)
disable_resources = self._get_with_precedence(
disable_resources, self.disable_resources, _UNSET
)
wait_selector = self._get_with_precedence(
wait_selector, self.wait_selector, _UNSET
)
wait_selector_state = self._get_with_precedence(
wait_selector_state, self.wait_selector_state, _UNSET
)
network_idle = self._get_with_precedence(
network_idle, self.network_idle, _UNSET
)
selector_config = self._get_with_precedence(
selector_config, self.selector_config, _UNSET
# Validate all resolved parameters
params = validate(
dict(
google_search=self._get_with_precedence(google_search, self.google_search, _UNSET),
timeout=self._get_with_precedence(timeout, self.timeout, _UNSET),
wait=self._get_with_precedence(wait, self.wait, _UNSET),
page_action=self._get_with_precedence(page_action, self.page_action, _UNSET),
extra_headers=self._get_with_precedence(extra_headers, self.extra_headers, _UNSET),
disable_resources=self._get_with_precedence(disable_resources, self.disable_resources, _UNSET),
wait_selector=self._get_with_precedence(wait_selector, self.wait_selector, _UNSET),
wait_selector_state=self._get_with_precedence(wait_selector_state, self.wait_selector_state, _UNSET),
network_idle=self._get_with_precedence(network_idle, self.network_idle, _UNSET),
selector_config=self._get_with_precedence(selector_config, self.selector_config, _UNSET),
),
PlaywrightConfig,
)
if self._closed: # pragma: no cover
@@ -519,9 +494,7 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession):
final_response = None
referer = (
generate_convincing_referer(url)
if (google_search and "referer" not in self._headers_keys)
else None
generate_convincing_referer(url) if (params.google_search and "referer" not in self._headers_keys) else None
)
async def handle_response(finished_response: AsyncPlaywrightResponse):
@@ -532,7 +505,7 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession):
):
final_response = finished_response
page_info = await self._get_page(timeout, extra_headers, disable_resources)
page_info = await self._get_page(params.timeout, params.extra_headers, params.disable_resources)
page_info.mark_busy(url=url)
try:
@@ -541,35 +514,35 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession):
first_response = await page_info.page.goto(url, referer=referer)
await page_info.page.wait_for_load_state(state="domcontentloaded")
if network_idle:
if params.network_idle:
await page_info.page.wait_for_load_state("networkidle")
if not first_response:
raise RuntimeError(f"Failed to get response for {url}")
if page_action is not None:
if params.page_action:
try:
_ = await page_action(page_info.page)
_ = await params.page_action(page_info.page)
except Exception as e:
log.error(f"Error executing page_action: {e}")
if wait_selector:
if params.wait_selector:
try:
waiter: AsyncLocator = page_info.page.locator(wait_selector)
await waiter.first.wait_for(state=wait_selector_state)
waiter: AsyncLocator = page_info.page.locator(params.wait_selector)
await waiter.first.wait_for(state=params.wait_selector_state)
# Wait again after waiting for the selector, helpful with protections like Cloudflare
await page_info.page.wait_for_load_state(state="load")
await page_info.page.wait_for_load_state(state="domcontentloaded")
if network_idle:
if params.network_idle:
await page_info.page.wait_for_load_state("networkidle")
except Exception as e:
log.error(f"Error waiting for selector {wait_selector}: {e}")
log.error(f"Error waiting for selector {params.wait_selector}: {e}")
await page_info.page.wait_for_timeout(wait)
await page_info.page.wait_for_timeout(params.wait)
# Create response object
response = await ResponseFactory.from_async_playwright_response(
page_info.page, first_response, final_response, selector_config
page_info.page, first_response, final_response, params.selector_config
)
# Mark the page as finished for next use