freedak/Scrapling
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor(browser fetchers): Make all the type hints dynamic + Faster validation

Browse Source 
Made the code shorter by an additional ~200 lines and easier to maintain in return for making the arguments autocompletion bad for shells that don't check for dynamic type hints like IPython.
This commit is contained in:
Karim shoair
2025-11-25 20:18:36 +02:00
parent 9e85311ff6
commit f0b01fc253
7 changed files with 364 additions and 637 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scrapling/core/_types.py
+1
View File
@@ -12,6 +12,7 @@ from typing import (
Callable,
Dict,
Generator,
Generic,
Iterable,
List,
Set,
scrapling/engines/_browsers/_base.py
+99 -94
View File
@@ -2,15 +2,15 @@ from time import time
from asyncio import sleep as asyncio_sleep, Lock
from camoufox import DefaultAddons
from playwright.sync_api._generated import Page
from playwright.sync_api import (
Page,
Frame,
BrowserContext,
Playwright,
Response as SyncPlaywrightResponse,
)
from playwright.async_api._generated import Page as AsyncPage
from playwright.async_api import (
Page as AsyncPage,
Frame as AsyncFrame,
Playwright as AsyncPlaywright,
Response as AsyncPlaywrightResponse,
@@ -70,7 +70,7 @@ class SyncSession:
timeout: int | float,
extra_headers: Optional[Dict[str, str]],
disable_resources: bool,
) -> PageInfo: # pragma: no cover
) -> PageInfo[Page]: # pragma: no cover
"""Get a new page to use"""
# No need to check if a page is available or not in sync code because the code blocked before reaching here till the page closed, ofc.
@@ -116,7 +116,7 @@ class SyncSession:
self._wait_for_networkidle(page)
@staticmethod
def _create_response_handler(page_info: PageInfo, response_container: List) -> Callable:
def _create_response_handler(page_info: PageInfo[Page], response_container: List) -> Callable:
"""Create a response handler that captures the final navigation response.
:param page_info: The PageInfo object containing the page
@@ -175,7 +175,7 @@ class AsyncSession:
timeout: int | float,
extra_headers: Optional[Dict[str, str]],
disable_resources: bool,
) -> PageInfo: # pragma: no cover
) -> PageInfo[AsyncPage]: # pragma: no cover
"""Get a new page to use"""
if TYPE_CHECKING:
assert self.context is not None, "Browser context not initialized"
@@ -232,7 +232,7 @@ class AsyncSession:
await self._wait_for_networkidle(page)
@staticmethod
def _create_response_handler(page_info: PageInfo, response_container: List) -> Callable:
def _create_response_handler(page_info: PageInfo[AsyncPage], response_container: List) -> Callable:
"""Create an async response handler that captures the final navigation response.
:param page_info: The PageInfo object containing the page
@@ -253,130 +253,135 @@ class AsyncSession:
class DynamicSessionMixin:
def __validate__(self, **params):
if "__max_pages" in params:
params["max_pages"] = params.pop("__max_pages")
config = validate(params, model=PlaywrightConfig)
self.max_pages = config.max_pages
self.headless = config.headless
self.hide_canvas = config.hide_canvas
self.disable_webgl = config.disable_webgl
self.real_chrome = config.real_chrome
self.stealth = config.stealth
self.google_search = config.google_search
self.wait = config.wait
self.proxy = config.proxy
self.locale = config.locale
self.extra_headers = config.extra_headers
self.useragent = config.useragent
self.timeout = config.timeout
self.cookies = config.cookies
self.disable_resources = config.disable_resources
self.cdp_url = config.cdp_url
self.network_idle = config.network_idle
self.load_dom = config.load_dom
self.wait_selector = config.wait_selector
self.init_script = config.init_script
self.wait_selector_state = config.wait_selector_state
self.extra_flags = config.extra_flags
self.selector_config = config.selector_config
self.additional_args = config.additional_args
self.page_action = config.page_action
self.user_data_dir = config.user_data_dir
self._headers_keys = {header.lower() for header in self.extra_headers.keys()} if self.extra_headers else set()
self._max_pages = config.max_pages
self._headless = config.headless
self._hide_canvas = config.hide_canvas
self._disable_webgl = config.disable_webgl
self._real_chrome = config.real_chrome
self._stealth = config.stealth
self._google_search = config.google_search
self._wait = config.wait
self._proxy = config.proxy
self._locale = config.locale
self._extra_headers = config.extra_headers
self._useragent = config.useragent
self._timeout = config.timeout
self._cookies = config.cookies
self._disable_resources = config.disable_resources
self._cdp_url = config.cdp_url
self._network_idle = config.network_idle
self._load_dom = config.load_dom
self._wait_selector = config.wait_selector
self._init_script = config.init_script
self._wait_selector_state = config.wait_selector_state
self._extra_flags = config.extra_flags
self._selector_config = config.selector_config
self._additional_args = config.additional_args
self._page_action = config.page_action
self._user_data_dir = config.user_data_dir
self._headers_keys = {header.lower() for header in self._extra_headers.keys()} if self._extra_headers else set()
self.__initiate_browser_options__()
def __initiate_browser_options__(self):
if TYPE_CHECKING:
assert isinstance(self.proxy, tuple)
assert isinstance(self._proxy, tuple)
if not self.cdp_url:
if not self._cdp_url:
# `launch_options` is used with persistent context
self.launch_options = dict(
_launch_kwargs(
self.headless,
self.proxy,
self.locale,
tuple(self.extra_headers.items()) if self.extra_headers else tuple(),
self.useragent,
self.real_chrome,
self.stealth,
self.hide_canvas,
self.disable_webgl,
tuple(self.extra_flags) if self.extra_flags else tuple(),
self._headless,
self._proxy,
self._locale,
tuple(self._extra_headers.items()) if self._extra_headers else tuple(),
self._useragent,
self._real_chrome,
self._stealth,
self._hide_canvas,
self._disable_webgl,
tuple(self._extra_flags) if self._extra_flags else tuple(),
)
)
self.launch_options["extra_http_headers"] = dict(self.launch_options["extra_http_headers"])
self.launch_options["proxy"] = dict(self.launch_options["proxy"]) or None
self.launch_options["user_data_dir"] = self.user_data_dir
self.launch_options.update(cast(Dict, self.additional_args))
self.launch_options["user_data_dir"] = self._user_data_dir
self.launch_options.update(cast(Dict, self._additional_args))
self.context_options = dict()
else:
# while `context_options` is left to be used when cdp mode is enabled
self.launch_options = dict()
self.context_options = dict(
_context_kwargs(
self.proxy,
self.locale,
tuple(self.extra_headers.items()) if self.extra_headers else tuple(),
self.useragent,
self.stealth,
self._proxy,
self._locale,
tuple(self._extra_headers.items()) if self._extra_headers else tuple(),
self._useragent,
self._stealth,
)
)
self.context_options["extra_http_headers"] = dict(self.context_options["extra_http_headers"])
self.context_options["proxy"] = dict(self.context_options["proxy"]) or None
self.context_options.update(cast(Dict, self.additional_args))
self.context_options.update(cast(Dict, self._additional_args))
class StealthySessionMixin:
def __validate__(self, **params):
if "__max_pages" in params:
params["max_pages"] = params.pop("__max_pages")
config: CamoufoxConfig = validate(params, model=CamoufoxConfig)
self.max_pages = config.max_pages
self.headless = config.headless
self.block_images = config.block_images
self.disable_resources = config.disable_resources
self.block_webrtc = config.block_webrtc
self.allow_webgl = config.allow_webgl
self.network_idle = config.network_idle
self.load_dom = config.load_dom
self.humanize = config.humanize
self.solve_cloudflare = config.solve_cloudflare
self.wait = config.wait
self.timeout = config.timeout
self.page_action = config.page_action
self.wait_selector = config.wait_selector
self.init_script = config.init_script
self.addons = config.addons
self.wait_selector_state = config.wait_selector_state
self.cookies = config.cookies
self.google_search = config.google_search
self.extra_headers = config.extra_headers
self.proxy = config.proxy
self.os_randomize = config.os_randomize
self.disable_ads = config.disable_ads
self.geoip = config.geoip
self.selector_config = config.selector_config
self.additional_args = config.additional_args
self.page_action = config.page_action
self.user_data_dir = config.user_data_dir
self._headers_keys = {header.lower() for header in self.extra_headers.keys()} if self.extra_headers else set()
self._max_pages = config.max_pages
self._headless = config.headless
self._block_images = config.block_images
self._disable_resources = config.disable_resources
self._block_webrtc = config.block_webrtc
self._allow_webgl = config.allow_webgl
self._network_idle = config.network_idle
self._load_dom = config.load_dom
self._humanize = config.humanize
self._solve_cloudflare = config.solve_cloudflare
self._wait = config.wait
self._timeout = config.timeout
self._page_action = config.page_action
self._wait_selector = config.wait_selector
self._init_script = config.init_script
self._addons = config.addons
self._wait_selector_state = config.wait_selector_state
self._cookies = config.cookies
self._google_search = config.google_search
self._extra_headers = config.extra_headers
self._proxy = config.proxy
self._os_randomize = config.os_randomize
self._disable_ads = config.disable_ads
self._geoip = config.geoip
self._selector_config = config.selector_config
self._additional_args = config.additional_args
self._user_data_dir = config.user_data_dir
self._headers_keys = {header.lower() for header in self._extra_headers.keys()} if self._extra_headers else set()
self.__initiate_browser_options__()
def __initiate_browser_options__(self):
"""Initiate browser options."""
self.launch_options: Dict[str, Any] = generate_launch_options(
**{
"geoip": self.geoip,
"proxy": dict(self.proxy) if self.proxy and isinstance(self.proxy, tuple) else self.proxy,
"addons": self.addons,
"exclude_addons": [] if self.disable_ads else [DefaultAddons.UBO],
"headless": self.headless,
"humanize": True if self.solve_cloudflare else self.humanize,
"geoip": self._geoip,
"proxy": dict(self._proxy) if self._proxy and isinstance(self._proxy, tuple) else self._proxy,
"addons": self._addons,
"exclude_addons": [] if self._disable_ads else [DefaultAddons.UBO],
"headless": self._headless,
"humanize": True if self._solve_cloudflare else self._humanize,
"i_know_what_im_doing": True, # To turn warnings off with the user configurations
"allow_webgl": self.allow_webgl,
"block_webrtc": self.block_webrtc,
"block_images": self.block_images, # Careful! it makes some websites don't finish loading at all like stackoverflow even in headful mode.
"os": None if self.os_randomize else get_os_name(),
"user_data_dir": self.user_data_dir,
"allow_webgl": self._allow_webgl,
"block_webrtc": self._block_webrtc,
"block_images": self._block_images, # Careful! it makes some websites don't finish loading at all like stackoverflow even in headful mode.
"os": None if self._os_randomize else get_os_name(),
"user_data_dir": self._user_data_dir,
"ff_version": __ff_version_str__,
"firefox_user_prefs": {
# This is what enabling `enable_cache` does internally, so we do it from here instead
@@ -386,7 +391,7 @@ class StealthySessionMixin:
"browser.cache.disk_cache_ssl": True,
"browser.cache.disk.smart_size.enabled": True,
},
**cast(Dict, self.additional_args),
**cast(Dict, self._additional_args),
}
)
scrapling/engines/_browsers/_camoufox.py
+84 -271
View File
@@ -14,58 +14,47 @@ from playwright.async_api import (
BrowserContext as AsyncBrowserContext,
)
from ._validators import validate_fetch as _validate, CamoufoxConfig
from ._base import SyncSession, AsyncSession, StealthySessionMixin
from scrapling.core.utils import log
from scrapling.core._types import (
Any,
Dict,
List,
Optional,
Callable,
TYPE_CHECKING,
SelectorWaitStates,
)
from scrapling.engines.toolbelt.convertor import (
Response,
ResponseFactory,
)
from ._types import CamoufoxSession, CamoufoxFetchParams
from scrapling.core._types import Any, Unpack, TYPE_CHECKING
from ._base import SyncSession, AsyncSession, StealthySessionMixin
from ._validators import validate_fetch as _validate, CamoufoxConfig
from scrapling.engines.toolbelt.convertor import Response, ResponseFactory
from scrapling.engines.toolbelt.fingerprints import generate_convincing_referer
__CF_PATTERN__ = re_compile("challenges.cloudflare.com/cdn-cgi/challenge-platform/.*")
_UNSET: Any = object()
class StealthySession(StealthySessionMixin, SyncSession):
"""A Stealthy session manager with page pooling."""
__slots__ = (
"max_pages",
"headless",
"block_images",
"disable_resources",
"block_webrtc",
"allow_webgl",
"network_idle",
"load_dom",
"humanize",
"solve_cloudflare",
"wait",
"timeout",
"page_action",
"wait_selector",
"init_script",
"addons",
"wait_selector_state",
"cookies",
"google_search",
"extra_headers",
"proxy",
"os_randomize",
"disable_ads",
"geoip",
"selector_config",
"additional_args",
"_max_pages",
"_headless",
"_block_images",
"_disable_resources",
"_block_webrtc",
"_allow_webgl",
"_network_idle",
"_load_dom",
"_humanize",
"_solve_cloudflare",
"_wait",
"_timeout",
"_page_action",
"_wait_selector",
"_init_script",
"_addons",
"_wait_selector_state",
"_cookies",
"_google_search",
"_extra_headers",
"_proxy",
"_os_randomize",
"_disable_ads",
"_geoip",
"_selector_config",
"_additional_args",
"playwright",
"browser",
"context",
@@ -73,38 +62,10 @@ class StealthySession(StealthySessionMixin, SyncSession):
"_closed",
"launch_options",
"_headers_keys",
"_user_data_dir",
)
def __init__(
self,
__max_pages: int = 1,
headless: bool = True, # noqa: F821
block_images: bool = False,
disable_resources: bool = False,
block_webrtc: bool = False,
allow_webgl: bool = True,
network_idle: bool = False,
load_dom: bool = True,
humanize: bool | float = True,
solve_cloudflare: bool = False,
wait: int | float = 0,
timeout: int | float = 30000,
page_action: Optional[Callable] = None,
wait_selector: Optional[str] = None,
init_script: Optional[str] = None,
addons: Optional[List[str]] = None,
wait_selector_state: SelectorWaitStates = "attached",
cookies: Optional[List[Dict]] = None,
google_search: bool = True,
extra_headers: Optional[Dict[str, str]] = None,
proxy: Optional[str | Dict[str, str]] = None,
os_randomize: bool = False,
disable_ads: bool = False,
geoip: bool = False,
user_data_dir: str = "",
selector_config: Optional[Dict] = None,
additional_args: Optional[Dict] = None,
):
def __init__(self, **kwargs: Unpack[CamoufoxSession]):
"""A Browser session manager with page pooling
:param headless: Run the browser in headless/hidden (default), or headful/visible mode.
@@ -138,50 +99,21 @@ class StealthySession(StealthySessionMixin, SyncSession):
:param selector_config: The arguments that will be passed in the end while creating the final Selector's class.
:param additional_args: Additional arguments to be passed to Camoufox as additional settings, and it takes higher priority than Scrapling's settings.
"""
self.__validate__(
wait=wait,
proxy=proxy,
geoip=geoip,
addons=addons,
timeout=timeout,
cookies=cookies,
headless=headless,
humanize=humanize,
load_dom=load_dom,
max_pages=__max_pages,
disable_ads=disable_ads,
allow_webgl=allow_webgl,
page_action=page_action,
init_script=init_script,
network_idle=network_idle,
block_images=block_images,
block_webrtc=block_webrtc,
os_randomize=os_randomize,
user_data_dir=user_data_dir,
wait_selector=wait_selector,
google_search=google_search,
extra_headers=extra_headers,
additional_args=additional_args,
selector_config=selector_config,
solve_cloudflare=solve_cloudflare,
disable_resources=disable_resources,
wait_selector_state=wait_selector_state,
)
super().__init__(max_pages=self.max_pages)
self.__validate__(**kwargs)
super().__init__(max_pages=self._max_pages)
def __create__(self):
"""Create a browser for this instance and context."""
self.playwright = sync_playwright().start()
self.context = self.playwright.firefox.launch_persistent_context(**self.launch_options)
if self.init_script: # pragma: no cover
self.context.add_init_script(path=self.init_script)
if self._init_script: # pragma: no cover
self.context.add_init_script(path=self._init_script)
if self.cookies: # pragma: no cover
self.context.add_cookies(self.cookies)
if self._cookies: # pragma: no cover
self.context.add_cookies(self._cookies)
def _solve_cloudflare(self, page: Page) -> None: # pragma: no cover
def _cloudflare_solver(self, page: Page) -> None: # pragma: no cover
"""Solve the cloudflare challenge displayed on the playwright page passed
:param page: The targeted page
@@ -247,59 +179,28 @@ class StealthySession(StealthySessionMixin, SyncSession):
log.info("Cloudflare captcha is solved")
return
def fetch(
self,
url: str,
google_search: bool = _UNSET,
timeout: int | float = _UNSET,
wait: int | float = _UNSET,
page_action: Optional[Callable] = _UNSET,
extra_headers: Optional[Dict[str, str]] = _UNSET,
disable_resources: bool = _UNSET,
wait_selector: Optional[str] = _UNSET,
wait_selector_state: SelectorWaitStates = _UNSET,
network_idle: bool = _UNSET,
load_dom: bool = _UNSET,
solve_cloudflare: bool = _UNSET,
selector_config: Optional[Dict] = _UNSET,
) -> Response:
def fetch(self, url: str, **kwargs: Unpack[CamoufoxFetchParams]) -> Response:
"""Opens up the browser and do your request based on your chosen options.
:param url: The Target url.
:param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name.
:param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
:param page_action: Added for automation. A function that takes the `page` object and does the automation you need.
:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
:param disable_resources: Drop requests of unnecessary resources for a speed boost. It depends, but it made requests ~25% faster in my tests for some websites.
Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
This can help save your proxy usage but be careful with this option as it makes some websites never finish loading.
:param wait_selector: Wait for a specific CSS selector to be in a specific state.
:param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.
:param network_idle: Wait for the page until there are no network connections for at least 500 ms.
:param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute.
:param solve_cloudflare: Solves all types of the Cloudflare's Turnstile/Interstitial challenges before returning the response to you.
:param selector_config: The arguments that will be passed in the end while creating the final Selector's class.
:param kwargs: Additional keyword arguments including:
- google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name.
- timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
- wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
- page_action: Added for automation. A function that takes the `page` object and does the automation you need.
- extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
- disable_resources: Drop requests of unnecessary resources for a speed boost. It depends, but it made requests ~25% faster in my tests for some websites.
Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
This can help save your proxy usage but be careful with this option as it makes some websites never finish loading.
- wait_selector: Wait for a specific CSS selector to be in a specific state.
- wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.
- network_idle: Wait for the page until there are no network connections for at least 500 ms.
- load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute.
- solve_cloudflare: Solves all types of the Cloudflare's Turnstile/Interstitial challenges before returning the response to you.
- selector_config: The arguments that will be passed in the end while creating the final Selector's class.
:return: A `Response` object.
"""
params = _validate(
[
("google_search", google_search, self.google_search),
("timeout", timeout, self.timeout),
("wait", wait, self.wait),
("page_action", page_action, self.page_action),
("extra_headers", extra_headers, self.extra_headers),
("disable_resources", disable_resources, self.disable_resources),
("wait_selector", wait_selector, self.wait_selector),
("wait_selector_state", wait_selector_state, self.wait_selector_state),
("network_idle", network_idle, self.network_idle),
("load_dom", load_dom, self.load_dom),
("solve_cloudflare", solve_cloudflare, self.solve_cloudflare),
("selector_config", selector_config, self.selector_config),
],
CamoufoxConfig,
_UNSET,
)
params = _validate(kwargs, self, CamoufoxConfig)
if self._closed: # pragma: no cover
raise RuntimeError("Context manager has been closed")
@@ -322,7 +223,7 @@ class StealthySession(StealthySessionMixin, SyncSession):
raise RuntimeError(f"Failed to get response for {url}")
if params.solve_cloudflare:
self._solve_cloudflare(page_info.page)
self._cloudflare_solver(page_info.page)
# Make sure the page is fully loaded after the captcha
self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle)
@@ -360,36 +261,7 @@ class StealthySession(StealthySessionMixin, SyncSession):
class AsyncStealthySession(StealthySessionMixin, AsyncSession):
"""A Stealthy session manager with page pooling."""
def __init__(
self,
max_pages: int = 1,
headless: bool = True, # noqa: F821
block_images: bool = False,
disable_resources: bool = False,
block_webrtc: bool = False,
allow_webgl: bool = True,
network_idle: bool = False,
load_dom: bool = True,
humanize: bool | float = True,
solve_cloudflare: bool = False,
wait: int | float = 0,
timeout: int | float = 30000,
page_action: Optional[Callable] = None,
wait_selector: Optional[str] = None,
init_script: Optional[str] = None,
addons: Optional[List[str]] = None,
wait_selector_state: SelectorWaitStates = "attached",
cookies: Optional[List[Dict]] = None,
google_search: bool = True,
extra_headers: Optional[Dict[str, str]] = None,
proxy: Optional[str | Dict[str, str]] = None,
os_randomize: bool = False,
disable_ads: bool = False,
geoip: bool = False,
user_data_dir: str = "",
selector_config: Optional[Dict] = None,
additional_args: Optional[Dict] = None,
):
def __init__(self, **kwargs: Unpack[CamoufoxSession]):
"""A Browser session manager with page pooling
:param headless: Run the browser in headless/hidden (default), or headful/visible mode.
@@ -424,36 +296,8 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):
:param selector_config: The arguments that will be passed in the end while creating the final Selector's class.
:param additional_args: Additional arguments to be passed to Camoufox as additional settings, and it takes higher priority than Scrapling's settings.
"""
self.__validate__(
wait=wait,
proxy=proxy,
geoip=geoip,
addons=addons,
timeout=timeout,
cookies=cookies,
headless=headless,
load_dom=load_dom,
humanize=humanize,
max_pages=max_pages,
disable_ads=disable_ads,
allow_webgl=allow_webgl,
page_action=page_action,
init_script=init_script,
network_idle=network_idle,
block_images=block_images,
block_webrtc=block_webrtc,
os_randomize=os_randomize,
wait_selector=wait_selector,
google_search=google_search,
extra_headers=extra_headers,
user_data_dir=user_data_dir,
additional_args=additional_args,
selector_config=selector_config,
solve_cloudflare=solve_cloudflare,
disable_resources=disable_resources,
wait_selector_state=wait_selector_state,
)
super().__init__(max_pages=self.max_pages)
self.__validate__(**kwargs)
super().__init__(max_pages=self._max_pages)
async def __create__(self):
"""Create a browser for this instance and context."""
@@ -462,13 +306,13 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):
**self.launch_options
)
if self.init_script: # pragma: no cover
await self.context.add_init_script(path=self.init_script)
if self._init_script: # pragma: no cover
await self.context.add_init_script(path=self._init_script)
if self.cookies:
await self.context.add_cookies(self.cookies) # pyright: ignore [reportArgumentType]
if self._cookies:
await self.context.add_cookies(self._cookies) # pyright: ignore [reportArgumentType]
async def _solve_cloudflare(self, page: async_Page): # pragma: no cover
async def _cloudflare_solver(self, page: async_Page): # pragma: no cover
"""Solve the cloudflare challenge displayed on the playwright page passed. The async version
:param page: The async targeted page
@@ -534,59 +378,28 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):
log.info("Cloudflare captcha is solved")
return
async def fetch(
self,
url: str,
google_search: bool = _UNSET,
timeout: int | float = _UNSET,
wait: int | float = _UNSET,
page_action: Optional[Callable] = _UNSET,
extra_headers: Optional[Dict[str, str]] = _UNSET,
disable_resources: bool = _UNSET,
wait_selector: Optional[str] = _UNSET,
wait_selector_state: SelectorWaitStates = _UNSET,
network_idle: bool = _UNSET,
load_dom: bool = _UNSET,
solve_cloudflare: bool = _UNSET,
selector_config: Optional[Dict] = _UNSET,
) -> Response:
async def fetch(self, url: str, **kwargs: Unpack[CamoufoxFetchParams]) -> Response:
"""Opens up the browser and do your request based on your chosen options.
:param url: The Target url.
:param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name.
:param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
:param page_action: Added for automation. A function that takes the `page` object and does the automation you need.
:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
:param disable_resources: Drop requests of unnecessary resources for a speed boost. It depends, but it made requests ~25% faster in my tests for some websites.
Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
This can help save your proxy usage but be careful with this option as it makes some websites never finish loading.
:param wait_selector: Wait for a specific CSS selector to be in a specific state.
:param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.
:param network_idle: Wait for the page until there are no network connections for at least 500 ms.
:param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute.
:param solve_cloudflare: Solves all types of the Cloudflare's Turnstile/Interstitial challenges before returning the response to you.
:param selector_config: The arguments that will be passed in the end while creating the final Selector's class.
:param kwargs: Additional keyword arguments including:
- google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name.
- timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
- wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
- page_action: Added for automation. A function that takes the `page` object and does the automation you need.
- extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
- disable_resources: Drop requests of unnecessary resources for a speed boost. It depends, but it made requests ~25% faster in my tests for some websites.
Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
This can help save your proxy usage but be careful with this option as it makes some websites never finish loading.
- wait_selector: Wait for a specific CSS selector to be in a specific state.
- wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.
- network_idle: Wait for the page until there are no network connections for at least 500 ms.
- load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute.
- solve_cloudflare: Solves all types of the Cloudflare's Turnstile/Interstitial challenges before returning the response to you.
- selector_config: The arguments that will be passed in the end while creating the final Selector's class.
:return: A `Response` object.
"""
params = _validate(
[
("google_search", google_search, self.google_search),
("timeout", timeout, self.timeout),
("wait", wait, self.wait),
("page_action", page_action, self.page_action),
("extra_headers", extra_headers, self.extra_headers),
("disable_resources", disable_resources, self.disable_resources),
("wait_selector", wait_selector, self.wait_selector),
("wait_selector_state", wait_selector_state, self.wait_selector_state),
("network_idle", network_idle, self.network_idle),
("load_dom", load_dom, self.load_dom),
("solve_cloudflare", solve_cloudflare, self.solve_cloudflare),
("selector_config", selector_config, self.selector_config),
],
CamoufoxConfig,
_UNSET,
)
params = _validate(kwargs, self, CamoufoxConfig)
if self._closed: # pragma: no cover
raise RuntimeError("Context manager has been closed")
@@ -613,7 +426,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):
raise RuntimeError(f"Failed to get response for {url}")
if params.solve_cloudflare:
await self._solve_cloudflare(page_info.page)
await self._cloudflare_solver(page_info.page)
# Make sure the page is fully loaded after the captcha
await self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle)
scrapling/engines/_browsers/_controllers.py
+81 -259
View File
@@ -13,92 +13,55 @@ from patchright.sync_api import sync_playwright as sync_patchright
from patchright.async_api import async_playwright as async_patchright
from scrapling.core.utils import log
from scrapling.core._types import Unpack, TYPE_CHECKING
from ._types import PlaywrightSession, PlaywrightFetchParams
from ._base import SyncSession, AsyncSession, DynamicSessionMixin
from ._validators import validate_fetch as _validate, PlaywrightConfig
from scrapling.core._types import (
Any,
Dict,
List,
Optional,
Callable,
TYPE_CHECKING,
SelectorWaitStates,
)
from scrapling.engines.toolbelt.convertor import (
Response,
ResponseFactory,
)
from scrapling.engines.toolbelt.convertor import Response, ResponseFactory
from scrapling.engines.toolbelt.fingerprints import generate_convincing_referer
_UNSET: Any = object()
class DynamicSession(DynamicSessionMixin, SyncSession):
"""A Browser session manager with page pooling."""
__slots__ = (
"max_pages",
"headless",
"hide_canvas",
"disable_webgl",
"real_chrome",
"stealth",
"google_search",
"proxy",
"locale",
"extra_headers",
"useragent",
"timeout",
"cookies",
"disable_resources",
"network_idle",
"load_dom",
"wait_selector",
"init_script",
"wait_selector_state",
"wait",
"_max_pages",
"_headless",
"_hide_canvas",
"_disable_webgl",
"_real_chrome",
"_stealth",
"_google_search",
"_proxy",
"_locale",
"_extra_headers",
"_useragent",
"_timeout",
"_cookies",
"_disable_resources",
"_network_idle",
"_load_dom",
"_wait_selector",
"_init_script",
"_wait_selector_state",
"_wait",
"playwright",
"browser",
"context",
"page_pool",
"_closed",
"selector_config",
"page_action",
"_selector_config",
"_page_action",
"launch_options",
"context_options",
"cdp_url",
"_cdp_url",
"_headers_keys",
"_extra_flags",
"_additional_args",
"_user_data_dir",
)
def __init__(
self,
__max_pages: int = 1,
headless: bool = True,
google_search: bool = True,
hide_canvas: bool = False,
disable_webgl: bool = False,
real_chrome: bool = False,
stealth: bool = False,
wait: int | float = 0,
page_action: Optional[Callable] = None,
proxy: Optional[str | Dict[str, str]] = None,
locale: str = "en-US",
extra_headers: Optional[Dict[str, str]] = None,
useragent: Optional[str] = None,
cdp_url: Optional[str] = None,
timeout: int | float = 30000,
disable_resources: bool = False,
wait_selector: Optional[str] = None,
init_script: Optional[str] = None,
cookies: Optional[List[Dict]] = None,
network_idle: bool = False,
load_dom: bool = True,
wait_selector_state: SelectorWaitStates = "attached",
user_data_dir: str = "",
extra_flags: Optional[List[str]] = None,
selector_config: Optional[Dict] = None,
additional_args: Optional[Dict] = None,
):
def __init__(self, **kwargs: Unpack[PlaywrightSession]):
"""A Browser session manager with page pooling, it's using a persistent browser Context by default with a temporary user profile directory.
:param headless: Run the browser in headless/hidden (default), or headful/visible mode.
@@ -129,105 +92,49 @@ class DynamicSession(DynamicSessionMixin, SyncSession):
:param selector_config: The arguments that will be passed in the end while creating the final Selector's class.
:param additional_args: Additional arguments to be passed to Playwright's context as additional settings, and it takes higher priority than Scrapling's settings.
"""
self.__validate__(
wait=wait,
proxy=proxy,
locale=locale,
timeout=timeout,
stealth=stealth,
cdp_url=cdp_url,
cookies=cookies,
load_dom=load_dom,
headless=headless,
useragent=useragent,
max_pages=__max_pages,
real_chrome=real_chrome,
page_action=page_action,
hide_canvas=hide_canvas,
init_script=init_script,
network_idle=network_idle,
user_data_dir=user_data_dir,
google_search=google_search,
extra_headers=extra_headers,
wait_selector=wait_selector,
disable_webgl=disable_webgl,
extra_flags=extra_flags,
selector_config=selector_config,
additional_args=additional_args,
disable_resources=disable_resources,
wait_selector_state=wait_selector_state,
)
super().__init__(max_pages=self.max_pages)
self.__validate__(**kwargs)
super().__init__(max_pages=self._max_pages)
def __create__(self):
"""Create a browser for this instance and context."""
sync_context = sync_patchright if self.stealth else sync_playwright
sync_context = sync_patchright if self._stealth else sync_playwright
self.playwright: Playwright = sync_context().start() # pyright: ignore [reportAttributeAccessIssue]
if self.cdp_url: # pragma: no cover
self.context = self.playwright.chromium.connect_over_cdp(endpoint_url=self.cdp_url).new_context(
if self._cdp_url: # pragma: no cover
self.context = self.playwright.chromium.connect_over_cdp(endpoint_url=self._cdp_url).new_context(
**self.context_options
)
else:
self.context = self.playwright.chromium.launch_persistent_context(**self.launch_options)
if self.init_script: # pragma: no cover
self.context.add_init_script(path=self.init_script)
if self._init_script: # pragma: no cover
self.context.add_init_script(path=self._init_script)
if self.cookies: # pragma: no cover
self.context.add_cookies(self.cookies)
if self._cookies: # pragma: no cover
self.context.add_cookies(self._cookies)
def fetch(
self,
url: str,
google_search: bool = _UNSET,
timeout: int | float = _UNSET,
wait: int | float = _UNSET,
page_action: Optional[Callable] = _UNSET,
extra_headers: Optional[Dict[str, str]] = _UNSET,
disable_resources: bool = _UNSET,
wait_selector: Optional[str] = _UNSET,
wait_selector_state: SelectorWaitStates = _UNSET,
network_idle: bool = _UNSET,
load_dom: bool = _UNSET,
selector_config: Optional[Dict] = _UNSET,
) -> Response:
def fetch(self, url: str, **kwargs: Unpack[PlaywrightFetchParams]) -> Response:
"""Opens up the browser and do your request based on your chosen options.
:param url: The Target url.
:param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name.
:param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
:param page_action: Added for automation. A function that takes the `page` object and does the automation you need.
:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
:param disable_resources: Drop requests of unnecessary resources for a speed boost. It depends, but it made requests ~25% faster in my tests for some websites.
Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
This can help save your proxy usage but be careful with this option as it makes some websites never finish loading.
:param wait_selector: Wait for a specific CSS selector to be in a specific state.
:param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.
:param network_idle: Wait for the page until there are no network connections for at least 500 ms.
:param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute.
:param selector_config: The arguments that will be passed in the end while creating the final Selector's class.
:param kwargs: Additional keyword arguments including:
- google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name.
- timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
- wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
- page_action: Added for automation. A function that takes the `page` object and does the automation you need.
- extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
- disable_resources: Drop requests of unnecessary resources for a speed boost. It depends, but it made requests ~25% faster in my tests for some websites.
Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
This can help save your proxy usage but be careful with this option as it makes some websites never finish loading.
- wait_selector: Wait for a specific CSS selector to be in a specific state.
- wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.
- network_idle: Wait for the page until there are no network connections for at least 500 ms.
- load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute.
- selector_config: The arguments that will be passed in the end while creating the final Selector's class.
:return: A `Response` object.
"""
params = _validate(
[
("google_search", google_search, self.google_search),
("timeout", timeout, self.timeout),
("wait", wait, self.wait),
("page_action", page_action, self.page_action),
("extra_headers", extra_headers, self.extra_headers),
("disable_resources", disable_resources, self.disable_resources),
("wait_selector", wait_selector, self.wait_selector),
("wait_selector_state", wait_selector_state, self.wait_selector_state),
("network_idle", network_idle, self.network_idle),
("load_dom", load_dom, self.load_dom),
("selector_config", selector_config, self.selector_config),
],
PlaywrightConfig,
_UNSET,
)
params = _validate(kwargs, self, PlaywrightConfig)
if self._closed: # pragma: no cover
raise RuntimeError("Context manager has been closed")
@@ -285,35 +192,7 @@ class DynamicSession(DynamicSessionMixin, SyncSession):
class AsyncDynamicSession(DynamicSessionMixin, AsyncSession):
"""An async Browser session manager with page pooling, it's using a persistent browser Context by default with a temporary user profile directory."""
def __init__(
self,
max_pages: int = 1,
headless: bool = True,
google_search: bool = True,
hide_canvas: bool = False,
disable_webgl: bool = False,
real_chrome: bool = False,
stealth: bool = False,
wait: int | float = 0,
page_action: Optional[Callable] = None,
proxy: Optional[str | Dict[str, str]] = None,
locale: str = "en-US",
extra_headers: Optional[Dict[str, str]] = None,
useragent: Optional[str] = None,
cdp_url: Optional[str] = None,
timeout: int | float = 30000,
disable_resources: bool = False,
wait_selector: Optional[str] = None,
init_script: Optional[str] = None,
cookies: Optional[List[Dict]] = None,
network_idle: bool = False,
load_dom: bool = True,
wait_selector_state: SelectorWaitStates = "attached",
user_data_dir: str = "",
extra_flags: Optional[List[str]] = None,
selector_config: Optional[Dict] = None,
additional_args: Optional[Dict] = None,
):
def __init__(self, **kwargs: Unpack[PlaywrightSession]):
"""A Browser session manager with page pooling
:param headless: Run the browser in headless/hidden (default), or headful/visible mode.
@@ -345,107 +224,50 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession):
:param selector_config: The arguments that will be passed in the end while creating the final Selector's class.
:param additional_args: Additional arguments to be passed to Playwright's context as additional settings, and it takes higher priority than Scrapling's settings.
"""
self.__validate__(
wait=wait,
proxy=proxy,
locale=locale,
timeout=timeout,
stealth=stealth,
cdp_url=cdp_url,
cookies=cookies,
load_dom=load_dom,
headless=headless,
useragent=useragent,
max_pages=max_pages,
real_chrome=real_chrome,
page_action=page_action,
hide_canvas=hide_canvas,
init_script=init_script,
network_idle=network_idle,
user_data_dir=user_data_dir,
google_search=google_search,
extra_headers=extra_headers,
wait_selector=wait_selector,
disable_webgl=disable_webgl,
extra_flags=extra_flags,
selector_config=selector_config,
additional_args=additional_args,
disable_resources=disable_resources,
wait_selector_state=wait_selector_state,
)
super().__init__(max_pages=self.max_pages)
self.__validate__(**kwargs)
super().__init__(max_pages=self._max_pages)
async def __create__(self):
"""Create a browser for this instance and context."""
async_context = async_patchright if self.stealth else async_playwright
async_context = async_patchright if self._stealth else async_playwright
self.playwright: AsyncPlaywright = await async_context().start() # pyright: ignore [reportAttributeAccessIssue]
if self.cdp_url:
browser = await self.playwright.chromium.connect_over_cdp(endpoint_url=self.cdp_url)
if self._cdp_url:
browser = await self.playwright.chromium.connect_over_cdp(endpoint_url=self._cdp_url)
self.context: AsyncBrowserContext = await browser.new_context(**self.context_options)
else:
self.context: AsyncBrowserContext = await self.playwright.chromium.launch_persistent_context(
**self.launch_options
)
if self.init_script: # pragma: no cover
await self.context.add_init_script(path=self.init_script)
if self._init_script: # pragma: no cover
await self.context.add_init_script(path=self._init_script)
if self.cookies:
await self.context.add_cookies(self.cookies) # pyright: ignore
if self._cookies:
await self.context.add_cookies(self._cookies) # pyright: ignore
async def fetch(
self,
url: str,
google_search: bool = _UNSET,
timeout: int | float = _UNSET,
wait: int | float = _UNSET,
page_action: Optional[Callable] = _UNSET,
extra_headers: Optional[Dict[str, str]] = _UNSET,
disable_resources: bool = _UNSET,
wait_selector: Optional[str] = _UNSET,
wait_selector_state: SelectorWaitStates = _UNSET,
network_idle: bool = _UNSET,
load_dom: bool = _UNSET,
selector_config: Optional[Dict] = _UNSET,
) -> Response:
async def fetch(self, url: str, **kwargs: Unpack[PlaywrightFetchParams]) -> Response:
"""Opens up the browser and do your request based on your chosen options.
:param url: The Target url.
:param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name.
:param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
:param page_action: Added for automation. A function that takes the `page` object and does the automation you need.
:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
:param disable_resources: Drop requests of unnecessary resources for a speed boost. It depends, but it made requests ~25% faster in my tests for some websites.
Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
This can help save your proxy usage but be careful with this option as it makes some websites never finish loading.
:param wait_selector: Wait for a specific CSS selector to be in a specific state.
:param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.
:param network_idle: Wait for the page until there are no network connections for at least 500 ms.
:param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute.
:param selector_config: The arguments that will be passed in the end while creating the final Selector's class.
:param kwargs: Additional keyword arguments including:
- google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name.
- timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
- wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
- page_action: Added for automation. A function that takes the `page` object and does the automation you need.
- extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
- disable_resources: Drop requests of unnecessary resources for a speed boost. It depends, but it made requests ~25% faster in my tests for some websites.
Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
This can help save your proxy usage but be careful with this option as it makes some websites never finish loading.
- wait_selector: Wait for a specific CSS selector to be in a specific state.
- wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.
- network_idle: Wait for the page until there are no network connections for at least 500 ms.
- load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute.
- selector_config: The arguments that will be passed in the end while creating the final Selector's class.
:return: A `Response` object.
"""
params = _validate(
[
("google_search", google_search, self.google_search),
("timeout", timeout, self.timeout),
("wait", wait, self.wait),
("page_action", page_action, self.page_action),
("extra_headers", extra_headers, self.extra_headers),
("disable_resources", disable_resources, self.disable_resources),
("wait_selector", wait_selector, self.wait_selector),
("wait_selector_state", wait_selector_state, self.wait_selector_state),
("network_idle", network_idle, self.network_idle),
("load_dom", load_dom, self.load_dom),
("selector_config", selector_config, self.selector_config),
],
PlaywrightConfig,
_UNSET,
)
params = _validate(kwargs, self, PlaywrightConfig)
if self._closed: # pragma: no cover
raise RuntimeError("Context manager has been closed")
scrapling/engines/_browsers/_page.py
+19 -8
View File
@@ -1,20 +1,21 @@
from threading import RLock
from dataclasses import dataclass
from playwright.sync_api import Page as SyncPage
from playwright.async_api import Page as AsyncPage
from playwright.sync_api._generated import Page as SyncPage
from playwright.async_api._generated import Page as AsyncPage
from scrapling.core._types import Optional, List, Literal
from scrapling.core._types import Optional, List, Literal, overload, TypeVar, Generic, cast
PageState = Literal["ready", "busy", "error"] # States that a page can be in
PageType = TypeVar("PageType", SyncPage, AsyncPage)
@dataclass
class PageInfo:
class PageInfo(Generic[PageType]):
"""Information about the page and its current state"""
__slots__ = ("page", "state", "url")
page: SyncPage | AsyncPage
page: PageType
state: PageState
url: Optional[str]
@@ -44,16 +45,26 @@ class PagePool:
def __init__(self, max_pages: int = 5):
self.max_pages = max_pages
self.pages: List[PageInfo] = []
self.pages: List[PageInfo[SyncPage] | PageInfo[AsyncPage]] = []
self._lock = RLock()
def add_page(self, page: SyncPage | AsyncPage) -> PageInfo:
@overload
def add_page(self, page: SyncPage) -> PageInfo[SyncPage]: ...
@overload
def add_page(self, page: AsyncPage) -> PageInfo[AsyncPage]: ...
def add_page(self, page: SyncPage | AsyncPage) -> PageInfo[SyncPage] | PageInfo[AsyncPage]:
"""Add a new page to the pool"""
with self._lock:
if len(self.pages) >= self.max_pages:
raise RuntimeError(f"Maximum page limit ({self.max_pages}) reached")
page_info = PageInfo(page, "ready", "")
if isinstance(page, AsyncPage):
page_info = cast(PageInfo[AsyncPage], PageInfo(page, "ready", ""))
else:
page_info = cast(PageInfo[SyncPage], PageInfo(page, "ready", ""))
self.pages.append(page_info)
return page_info
scrapling/engines/_browsers/_types.py
+65
View File
@@ -10,8 +10,11 @@ from scrapling.core._types import (
Tuple,
Mapping,
Optional,
Callable,
Iterable,
TypedDict,
TypeAlias,
SelectorWaitStates,
TYPE_CHECKING,
)
@@ -49,7 +52,69 @@ if TYPE_CHECKING: # pragma: no cover
data: Optional[Dict | str]
json: Optional[Dict | List]
# Types for browser session
class BrowserSession(TypedDict, total=False):
max_pages: int
headless: bool
disable_resources: bool
network_idle: bool
load_dom: bool
wait_selector: Optional[str]
wait_selector_state: SelectorWaitStates
cookies: Optional[Iterable[Dict]]
google_search: bool
wait: int | float
page_action: Optional[Callable]
proxy: Optional[str | Dict[str, str] | Tuple]
extra_headers: Optional[Dict[str, str]]
timeout: int | float
init_script: Optional[str]
user_data_dir: str
selector_config: Optional[Dict]
additional_args: Optional[Dict]
class PlaywrightSession(BrowserSession, total=False):
cdp_url: Optional[str]
hide_canvas: bool
disable_webgl: bool
real_chrome: bool
stealth: bool
locale: str
useragent: Optional[str]
extra_flags: Optional[List[str]]
class PlaywrightFetchParams(TypedDict, total=False):
google_search: bool
timeout: int | float
wait: int | float
page_action: Optional[Callable]
extra_headers: Optional[Dict[str, str]]
disable_resources: bool
wait_selector: Optional[str]
wait_selector_state: SelectorWaitStates
network_idle: bool
load_dom: bool
selector_config: Optional[Dict]
class CamoufoxSession(BrowserSession, total=False):
block_images: bool
block_webrtc: bool
allow_webgl: bool
humanize: bool | float
solve_cloudflare: bool
addons: Optional[List[str]]
os_randomize: bool
disable_ads: bool
geoip: bool
class CamoufoxFetchParams(PlaywrightFetchParams, total=False):
solve_cloudflare: bool
else: # pragma: no cover
RequestsSession = TypedDict
GetRequestParams = TypedDict
DataRequestParams = TypedDict
PlaywrightSession = TypedDict
PlaywrightFetchParams = TypedDict
CamoufoxSession = TypedDict
CamoufoxFetchParams = TypedDict
scrapling/engines/_browsers/_validators.py
+15 -5
View File
@@ -7,6 +7,7 @@ from dataclasses import dataclass, fields
from msgspec import Struct, Meta, convert, ValidationError
from scrapling.core._types import (
Any,
Dict,
List,
Tuple,
@@ -17,6 +18,7 @@ from scrapling.core._types import (
overload,
)
from scrapling.engines.toolbelt.navigation import construct_proxy_dict
from scrapling.engines._browsers._types import PlaywrightFetchParams, CamoufoxFetchParams
# Custom validators for msgspec
@@ -194,16 +196,24 @@ class _fetch_params:
def validate_fetch(
params: List[Tuple], model: type[PlaywrightConfig] | type[CamoufoxConfig], sentinel=None
method_kwargs: Dict | PlaywrightFetchParams | CamoufoxFetchParams,
session: Any,
model: type[PlaywrightConfig] | type[CamoufoxConfig],
) -> _fetch_params: # pragma: no cover
result = {}
overrides = {}
for arg, request_value, session_value in params:
if request_value is not sentinel:
overrides[arg] = request_value
# Get all field names that _fetch_params needs
fetch_param_fields = {f.name for f in fields(_fetch_params)}
for key in fetch_param_fields:
if key in method_kwargs:
overrides[key] = method_kwargs[key]
else:
result[arg] = session_value
# Check for underscore-prefixed attribute (private)
attr_name = f"_{key}"
if hasattr(session, attr_name):
result[key] = getattr(session, attr_name)
if overrides:
validated_config = validate(overrides, model)