diff --git a/scrapling/core/_types.py b/scrapling/core/_types.py index bd83f49..ac19a91 100644 --- a/scrapling/core/_types.py +++ b/scrapling/core/_types.py @@ -12,6 +12,7 @@ from typing import ( Callable, Dict, Generator, + Generic, Iterable, List, Set, diff --git a/scrapling/engines/_browsers/_base.py b/scrapling/engines/_browsers/_base.py index f670e4c..f329fa4 100644 --- a/scrapling/engines/_browsers/_base.py +++ b/scrapling/engines/_browsers/_base.py @@ -2,15 +2,15 @@ from time import time from asyncio import sleep as asyncio_sleep, Lock from camoufox import DefaultAddons +from playwright.sync_api._generated import Page from playwright.sync_api import ( - Page, Frame, BrowserContext, Playwright, Response as SyncPlaywrightResponse, ) +from playwright.async_api._generated import Page as AsyncPage from playwright.async_api import ( - Page as AsyncPage, Frame as AsyncFrame, Playwright as AsyncPlaywright, Response as AsyncPlaywrightResponse, @@ -70,7 +70,7 @@ class SyncSession: timeout: int | float, extra_headers: Optional[Dict[str, str]], disable_resources: bool, - ) -> PageInfo: # pragma: no cover + ) -> PageInfo[Page]: # pragma: no cover """Get a new page to use""" # No need to check if a page is available or not in sync code because the code blocked before reaching here till the page closed, ofc. @@ -116,7 +116,7 @@ class SyncSession: self._wait_for_networkidle(page) @staticmethod - def _create_response_handler(page_info: PageInfo, response_container: List) -> Callable: + def _create_response_handler(page_info: PageInfo[Page], response_container: List) -> Callable: """Create a response handler that captures the final navigation response. :param page_info: The PageInfo object containing the page @@ -175,7 +175,7 @@ class AsyncSession: timeout: int | float, extra_headers: Optional[Dict[str, str]], disable_resources: bool, - ) -> PageInfo: # pragma: no cover + ) -> PageInfo[AsyncPage]: # pragma: no cover """Get a new page to use""" if TYPE_CHECKING: assert self.context is not None, "Browser context not initialized" @@ -232,7 +232,7 @@ class AsyncSession: await self._wait_for_networkidle(page) @staticmethod - def _create_response_handler(page_info: PageInfo, response_container: List) -> Callable: + def _create_response_handler(page_info: PageInfo[AsyncPage], response_container: List) -> Callable: """Create an async response handler that captures the final navigation response. :param page_info: The PageInfo object containing the page @@ -253,130 +253,135 @@ class AsyncSession: class DynamicSessionMixin: def __validate__(self, **params): + if "__max_pages" in params: + params["max_pages"] = params.pop("__max_pages") + config = validate(params, model=PlaywrightConfig) - self.max_pages = config.max_pages - self.headless = config.headless - self.hide_canvas = config.hide_canvas - self.disable_webgl = config.disable_webgl - self.real_chrome = config.real_chrome - self.stealth = config.stealth - self.google_search = config.google_search - self.wait = config.wait - self.proxy = config.proxy - self.locale = config.locale - self.extra_headers = config.extra_headers - self.useragent = config.useragent - self.timeout = config.timeout - self.cookies = config.cookies - self.disable_resources = config.disable_resources - self.cdp_url = config.cdp_url - self.network_idle = config.network_idle - self.load_dom = config.load_dom - self.wait_selector = config.wait_selector - self.init_script = config.init_script - self.wait_selector_state = config.wait_selector_state - self.extra_flags = config.extra_flags - self.selector_config = config.selector_config - self.additional_args = config.additional_args - self.page_action = config.page_action - self.user_data_dir = config.user_data_dir - self._headers_keys = {header.lower() for header in self.extra_headers.keys()} if self.extra_headers else set() + self._max_pages = config.max_pages + self._headless = config.headless + self._hide_canvas = config.hide_canvas + self._disable_webgl = config.disable_webgl + self._real_chrome = config.real_chrome + self._stealth = config.stealth + self._google_search = config.google_search + self._wait = config.wait + self._proxy = config.proxy + self._locale = config.locale + self._extra_headers = config.extra_headers + self._useragent = config.useragent + self._timeout = config.timeout + self._cookies = config.cookies + self._disable_resources = config.disable_resources + self._cdp_url = config.cdp_url + self._network_idle = config.network_idle + self._load_dom = config.load_dom + self._wait_selector = config.wait_selector + self._init_script = config.init_script + self._wait_selector_state = config.wait_selector_state + self._extra_flags = config.extra_flags + self._selector_config = config.selector_config + self._additional_args = config.additional_args + self._page_action = config.page_action + self._user_data_dir = config.user_data_dir + self._headers_keys = {header.lower() for header in self._extra_headers.keys()} if self._extra_headers else set() self.__initiate_browser_options__() def __initiate_browser_options__(self): if TYPE_CHECKING: - assert isinstance(self.proxy, tuple) + assert isinstance(self._proxy, tuple) - if not self.cdp_url: + if not self._cdp_url: # `launch_options` is used with persistent context self.launch_options = dict( _launch_kwargs( - self.headless, - self.proxy, - self.locale, - tuple(self.extra_headers.items()) if self.extra_headers else tuple(), - self.useragent, - self.real_chrome, - self.stealth, - self.hide_canvas, - self.disable_webgl, - tuple(self.extra_flags) if self.extra_flags else tuple(), + self._headless, + self._proxy, + self._locale, + tuple(self._extra_headers.items()) if self._extra_headers else tuple(), + self._useragent, + self._real_chrome, + self._stealth, + self._hide_canvas, + self._disable_webgl, + tuple(self._extra_flags) if self._extra_flags else tuple(), ) ) self.launch_options["extra_http_headers"] = dict(self.launch_options["extra_http_headers"]) self.launch_options["proxy"] = dict(self.launch_options["proxy"]) or None - self.launch_options["user_data_dir"] = self.user_data_dir - self.launch_options.update(cast(Dict, self.additional_args)) + self.launch_options["user_data_dir"] = self._user_data_dir + self.launch_options.update(cast(Dict, self._additional_args)) self.context_options = dict() else: # while `context_options` is left to be used when cdp mode is enabled self.launch_options = dict() self.context_options = dict( _context_kwargs( - self.proxy, - self.locale, - tuple(self.extra_headers.items()) if self.extra_headers else tuple(), - self.useragent, - self.stealth, + self._proxy, + self._locale, + tuple(self._extra_headers.items()) if self._extra_headers else tuple(), + self._useragent, + self._stealth, ) ) self.context_options["extra_http_headers"] = dict(self.context_options["extra_http_headers"]) self.context_options["proxy"] = dict(self.context_options["proxy"]) or None - self.context_options.update(cast(Dict, self.additional_args)) + self.context_options.update(cast(Dict, self._additional_args)) class StealthySessionMixin: def __validate__(self, **params): + if "__max_pages" in params: + params["max_pages"] = params.pop("__max_pages") + config: CamoufoxConfig = validate(params, model=CamoufoxConfig) - self.max_pages = config.max_pages - self.headless = config.headless - self.block_images = config.block_images - self.disable_resources = config.disable_resources - self.block_webrtc = config.block_webrtc - self.allow_webgl = config.allow_webgl - self.network_idle = config.network_idle - self.load_dom = config.load_dom - self.humanize = config.humanize - self.solve_cloudflare = config.solve_cloudflare - self.wait = config.wait - self.timeout = config.timeout - self.page_action = config.page_action - self.wait_selector = config.wait_selector - self.init_script = config.init_script - self.addons = config.addons - self.wait_selector_state = config.wait_selector_state - self.cookies = config.cookies - self.google_search = config.google_search - self.extra_headers = config.extra_headers - self.proxy = config.proxy - self.os_randomize = config.os_randomize - self.disable_ads = config.disable_ads - self.geoip = config.geoip - self.selector_config = config.selector_config - self.additional_args = config.additional_args - self.page_action = config.page_action - self.user_data_dir = config.user_data_dir - self._headers_keys = {header.lower() for header in self.extra_headers.keys()} if self.extra_headers else set() + self._max_pages = config.max_pages + self._headless = config.headless + self._block_images = config.block_images + self._disable_resources = config.disable_resources + self._block_webrtc = config.block_webrtc + self._allow_webgl = config.allow_webgl + self._network_idle = config.network_idle + self._load_dom = config.load_dom + self._humanize = config.humanize + self._solve_cloudflare = config.solve_cloudflare + self._wait = config.wait + self._timeout = config.timeout + self._page_action = config.page_action + self._wait_selector = config.wait_selector + self._init_script = config.init_script + self._addons = config.addons + self._wait_selector_state = config.wait_selector_state + self._cookies = config.cookies + self._google_search = config.google_search + self._extra_headers = config.extra_headers + self._proxy = config.proxy + self._os_randomize = config.os_randomize + self._disable_ads = config.disable_ads + self._geoip = config.geoip + self._selector_config = config.selector_config + self._additional_args = config.additional_args + self._user_data_dir = config.user_data_dir + self._headers_keys = {header.lower() for header in self._extra_headers.keys()} if self._extra_headers else set() self.__initiate_browser_options__() def __initiate_browser_options__(self): """Initiate browser options.""" self.launch_options: Dict[str, Any] = generate_launch_options( **{ - "geoip": self.geoip, - "proxy": dict(self.proxy) if self.proxy and isinstance(self.proxy, tuple) else self.proxy, - "addons": self.addons, - "exclude_addons": [] if self.disable_ads else [DefaultAddons.UBO], - "headless": self.headless, - "humanize": True if self.solve_cloudflare else self.humanize, + "geoip": self._geoip, + "proxy": dict(self._proxy) if self._proxy and isinstance(self._proxy, tuple) else self._proxy, + "addons": self._addons, + "exclude_addons": [] if self._disable_ads else [DefaultAddons.UBO], + "headless": self._headless, + "humanize": True if self._solve_cloudflare else self._humanize, "i_know_what_im_doing": True, # To turn warnings off with the user configurations - "allow_webgl": self.allow_webgl, - "block_webrtc": self.block_webrtc, - "block_images": self.block_images, # Careful! it makes some websites don't finish loading at all like stackoverflow even in headful mode. - "os": None if self.os_randomize else get_os_name(), - "user_data_dir": self.user_data_dir, + "allow_webgl": self._allow_webgl, + "block_webrtc": self._block_webrtc, + "block_images": self._block_images, # Careful! it makes some websites don't finish loading at all like stackoverflow even in headful mode. + "os": None if self._os_randomize else get_os_name(), + "user_data_dir": self._user_data_dir, "ff_version": __ff_version_str__, "firefox_user_prefs": { # This is what enabling `enable_cache` does internally, so we do it from here instead @@ -386,7 +391,7 @@ class StealthySessionMixin: "browser.cache.disk_cache_ssl": True, "browser.cache.disk.smart_size.enabled": True, }, - **cast(Dict, self.additional_args), + **cast(Dict, self._additional_args), } ) diff --git a/scrapling/engines/_browsers/_camoufox.py b/scrapling/engines/_browsers/_camoufox.py index 8d70dbd..436a006 100644 --- a/scrapling/engines/_browsers/_camoufox.py +++ b/scrapling/engines/_browsers/_camoufox.py @@ -14,58 +14,47 @@ from playwright.async_api import ( BrowserContext as AsyncBrowserContext, ) -from ._validators import validate_fetch as _validate, CamoufoxConfig -from ._base import SyncSession, AsyncSession, StealthySessionMixin from scrapling.core.utils import log -from scrapling.core._types import ( - Any, - Dict, - List, - Optional, - Callable, - TYPE_CHECKING, - SelectorWaitStates, -) -from scrapling.engines.toolbelt.convertor import ( - Response, - ResponseFactory, -) +from ._types import CamoufoxSession, CamoufoxFetchParams +from scrapling.core._types import Any, Unpack, TYPE_CHECKING +from ._base import SyncSession, AsyncSession, StealthySessionMixin +from ._validators import validate_fetch as _validate, CamoufoxConfig +from scrapling.engines.toolbelt.convertor import Response, ResponseFactory from scrapling.engines.toolbelt.fingerprints import generate_convincing_referer __CF_PATTERN__ = re_compile("challenges.cloudflare.com/cdn-cgi/challenge-platform/.*") -_UNSET: Any = object() class StealthySession(StealthySessionMixin, SyncSession): """A Stealthy session manager with page pooling.""" __slots__ = ( - "max_pages", - "headless", - "block_images", - "disable_resources", - "block_webrtc", - "allow_webgl", - "network_idle", - "load_dom", - "humanize", - "solve_cloudflare", - "wait", - "timeout", - "page_action", - "wait_selector", - "init_script", - "addons", - "wait_selector_state", - "cookies", - "google_search", - "extra_headers", - "proxy", - "os_randomize", - "disable_ads", - "geoip", - "selector_config", - "additional_args", + "_max_pages", + "_headless", + "_block_images", + "_disable_resources", + "_block_webrtc", + "_allow_webgl", + "_network_idle", + "_load_dom", + "_humanize", + "_solve_cloudflare", + "_wait", + "_timeout", + "_page_action", + "_wait_selector", + "_init_script", + "_addons", + "_wait_selector_state", + "_cookies", + "_google_search", + "_extra_headers", + "_proxy", + "_os_randomize", + "_disable_ads", + "_geoip", + "_selector_config", + "_additional_args", "playwright", "browser", "context", @@ -73,38 +62,10 @@ class StealthySession(StealthySessionMixin, SyncSession): "_closed", "launch_options", "_headers_keys", + "_user_data_dir", ) - def __init__( - self, - __max_pages: int = 1, - headless: bool = True, # noqa: F821 - block_images: bool = False, - disable_resources: bool = False, - block_webrtc: bool = False, - allow_webgl: bool = True, - network_idle: bool = False, - load_dom: bool = True, - humanize: bool | float = True, - solve_cloudflare: bool = False, - wait: int | float = 0, - timeout: int | float = 30000, - page_action: Optional[Callable] = None, - wait_selector: Optional[str] = None, - init_script: Optional[str] = None, - addons: Optional[List[str]] = None, - wait_selector_state: SelectorWaitStates = "attached", - cookies: Optional[List[Dict]] = None, - google_search: bool = True, - extra_headers: Optional[Dict[str, str]] = None, - proxy: Optional[str | Dict[str, str]] = None, - os_randomize: bool = False, - disable_ads: bool = False, - geoip: bool = False, - user_data_dir: str = "", - selector_config: Optional[Dict] = None, - additional_args: Optional[Dict] = None, - ): + def __init__(self, **kwargs: Unpack[CamoufoxSession]): """A Browser session manager with page pooling :param headless: Run the browser in headless/hidden (default), or headful/visible mode. @@ -138,50 +99,21 @@ class StealthySession(StealthySessionMixin, SyncSession): :param selector_config: The arguments that will be passed in the end while creating the final Selector's class. :param additional_args: Additional arguments to be passed to Camoufox as additional settings, and it takes higher priority than Scrapling's settings. """ - - self.__validate__( - wait=wait, - proxy=proxy, - geoip=geoip, - addons=addons, - timeout=timeout, - cookies=cookies, - headless=headless, - humanize=humanize, - load_dom=load_dom, - max_pages=__max_pages, - disable_ads=disable_ads, - allow_webgl=allow_webgl, - page_action=page_action, - init_script=init_script, - network_idle=network_idle, - block_images=block_images, - block_webrtc=block_webrtc, - os_randomize=os_randomize, - user_data_dir=user_data_dir, - wait_selector=wait_selector, - google_search=google_search, - extra_headers=extra_headers, - additional_args=additional_args, - selector_config=selector_config, - solve_cloudflare=solve_cloudflare, - disable_resources=disable_resources, - wait_selector_state=wait_selector_state, - ) - super().__init__(max_pages=self.max_pages) + self.__validate__(**kwargs) + super().__init__(max_pages=self._max_pages) def __create__(self): """Create a browser for this instance and context.""" self.playwright = sync_playwright().start() self.context = self.playwright.firefox.launch_persistent_context(**self.launch_options) - if self.init_script: # pragma: no cover - self.context.add_init_script(path=self.init_script) + if self._init_script: # pragma: no cover + self.context.add_init_script(path=self._init_script) - if self.cookies: # pragma: no cover - self.context.add_cookies(self.cookies) + if self._cookies: # pragma: no cover + self.context.add_cookies(self._cookies) - def _solve_cloudflare(self, page: Page) -> None: # pragma: no cover + def _cloudflare_solver(self, page: Page) -> None: # pragma: no cover """Solve the cloudflare challenge displayed on the playwright page passed :param page: The targeted page @@ -247,59 +179,28 @@ class StealthySession(StealthySessionMixin, SyncSession): log.info("Cloudflare captcha is solved") return - def fetch( - self, - url: str, - google_search: bool = _UNSET, - timeout: int | float = _UNSET, - wait: int | float = _UNSET, - page_action: Optional[Callable] = _UNSET, - extra_headers: Optional[Dict[str, str]] = _UNSET, - disable_resources: bool = _UNSET, - wait_selector: Optional[str] = _UNSET, - wait_selector_state: SelectorWaitStates = _UNSET, - network_idle: bool = _UNSET, - load_dom: bool = _UNSET, - solve_cloudflare: bool = _UNSET, - selector_config: Optional[Dict] = _UNSET, - ) -> Response: + def fetch(self, url: str, **kwargs: Unpack[CamoufoxFetchParams]) -> Response: """Opens up the browser and do your request based on your chosen options. :param url: The Target url. - :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name. - :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000 - :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object. - :param page_action: Added for automation. A function that takes the `page` object and does the automation you need. - :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ - :param disable_resources: Drop requests of unnecessary resources for a speed boost. It depends, but it made requests ~25% faster in my tests for some websites. - Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`. - This can help save your proxy usage but be careful with this option as it makes some websites never finish loading. - :param wait_selector: Wait for a specific CSS selector to be in a specific state. - :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`. - :param network_idle: Wait for the page until there are no network connections for at least 500 ms. - :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute. - :param solve_cloudflare: Solves all types of the Cloudflare's Turnstile/Interstitial challenges before returning the response to you. - :param selector_config: The arguments that will be passed in the end while creating the final Selector's class. + :param kwargs: Additional keyword arguments including: + - google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name. + - timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000 + - wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object. + - page_action: Added for automation. A function that takes the `page` object and does the automation you need. + - extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ + - disable_resources: Drop requests of unnecessary resources for a speed boost. It depends, but it made requests ~25% faster in my tests for some websites. + Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`. + This can help save your proxy usage but be careful with this option as it makes some websites never finish loading. + - wait_selector: Wait for a specific CSS selector to be in a specific state. + - wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`. + - network_idle: Wait for the page until there are no network connections for at least 500 ms. + - load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute. + - solve_cloudflare: Solves all types of the Cloudflare's Turnstile/Interstitial challenges before returning the response to you. + - selector_config: The arguments that will be passed in the end while creating the final Selector's class. :return: A `Response` object. """ - params = _validate( - [ - ("google_search", google_search, self.google_search), - ("timeout", timeout, self.timeout), - ("wait", wait, self.wait), - ("page_action", page_action, self.page_action), - ("extra_headers", extra_headers, self.extra_headers), - ("disable_resources", disable_resources, self.disable_resources), - ("wait_selector", wait_selector, self.wait_selector), - ("wait_selector_state", wait_selector_state, self.wait_selector_state), - ("network_idle", network_idle, self.network_idle), - ("load_dom", load_dom, self.load_dom), - ("solve_cloudflare", solve_cloudflare, self.solve_cloudflare), - ("selector_config", selector_config, self.selector_config), - ], - CamoufoxConfig, - _UNSET, - ) + params = _validate(kwargs, self, CamoufoxConfig) if self._closed: # pragma: no cover raise RuntimeError("Context manager has been closed") @@ -322,7 +223,7 @@ class StealthySession(StealthySessionMixin, SyncSession): raise RuntimeError(f"Failed to get response for {url}") if params.solve_cloudflare: - self._solve_cloudflare(page_info.page) + self._cloudflare_solver(page_info.page) # Make sure the page is fully loaded after the captcha self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle) @@ -360,36 +261,7 @@ class StealthySession(StealthySessionMixin, SyncSession): class AsyncStealthySession(StealthySessionMixin, AsyncSession): """A Stealthy session manager with page pooling.""" - def __init__( - self, - max_pages: int = 1, - headless: bool = True, # noqa: F821 - block_images: bool = False, - disable_resources: bool = False, - block_webrtc: bool = False, - allow_webgl: bool = True, - network_idle: bool = False, - load_dom: bool = True, - humanize: bool | float = True, - solve_cloudflare: bool = False, - wait: int | float = 0, - timeout: int | float = 30000, - page_action: Optional[Callable] = None, - wait_selector: Optional[str] = None, - init_script: Optional[str] = None, - addons: Optional[List[str]] = None, - wait_selector_state: SelectorWaitStates = "attached", - cookies: Optional[List[Dict]] = None, - google_search: bool = True, - extra_headers: Optional[Dict[str, str]] = None, - proxy: Optional[str | Dict[str, str]] = None, - os_randomize: bool = False, - disable_ads: bool = False, - geoip: bool = False, - user_data_dir: str = "", - selector_config: Optional[Dict] = None, - additional_args: Optional[Dict] = None, - ): + def __init__(self, **kwargs: Unpack[CamoufoxSession]): """A Browser session manager with page pooling :param headless: Run the browser in headless/hidden (default), or headful/visible mode. @@ -424,36 +296,8 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession): :param selector_config: The arguments that will be passed in the end while creating the final Selector's class. :param additional_args: Additional arguments to be passed to Camoufox as additional settings, and it takes higher priority than Scrapling's settings. """ - self.__validate__( - wait=wait, - proxy=proxy, - geoip=geoip, - addons=addons, - timeout=timeout, - cookies=cookies, - headless=headless, - load_dom=load_dom, - humanize=humanize, - max_pages=max_pages, - disable_ads=disable_ads, - allow_webgl=allow_webgl, - page_action=page_action, - init_script=init_script, - network_idle=network_idle, - block_images=block_images, - block_webrtc=block_webrtc, - os_randomize=os_randomize, - wait_selector=wait_selector, - google_search=google_search, - extra_headers=extra_headers, - user_data_dir=user_data_dir, - additional_args=additional_args, - selector_config=selector_config, - solve_cloudflare=solve_cloudflare, - disable_resources=disable_resources, - wait_selector_state=wait_selector_state, - ) - super().__init__(max_pages=self.max_pages) + self.__validate__(**kwargs) + super().__init__(max_pages=self._max_pages) async def __create__(self): """Create a browser for this instance and context.""" @@ -462,13 +306,13 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession): **self.launch_options ) - if self.init_script: # pragma: no cover - await self.context.add_init_script(path=self.init_script) + if self._init_script: # pragma: no cover + await self.context.add_init_script(path=self._init_script) - if self.cookies: - await self.context.add_cookies(self.cookies) # pyright: ignore [reportArgumentType] + if self._cookies: + await self.context.add_cookies(self._cookies) # pyright: ignore [reportArgumentType] - async def _solve_cloudflare(self, page: async_Page): # pragma: no cover + async def _cloudflare_solver(self, page: async_Page): # pragma: no cover """Solve the cloudflare challenge displayed on the playwright page passed. The async version :param page: The async targeted page @@ -534,59 +378,28 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession): log.info("Cloudflare captcha is solved") return - async def fetch( - self, - url: str, - google_search: bool = _UNSET, - timeout: int | float = _UNSET, - wait: int | float = _UNSET, - page_action: Optional[Callable] = _UNSET, - extra_headers: Optional[Dict[str, str]] = _UNSET, - disable_resources: bool = _UNSET, - wait_selector: Optional[str] = _UNSET, - wait_selector_state: SelectorWaitStates = _UNSET, - network_idle: bool = _UNSET, - load_dom: bool = _UNSET, - solve_cloudflare: bool = _UNSET, - selector_config: Optional[Dict] = _UNSET, - ) -> Response: + async def fetch(self, url: str, **kwargs: Unpack[CamoufoxFetchParams]) -> Response: """Opens up the browser and do your request based on your chosen options. :param url: The Target url. - :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name. - :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000 - :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object. - :param page_action: Added for automation. A function that takes the `page` object and does the automation you need. - :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ - :param disable_resources: Drop requests of unnecessary resources for a speed boost. It depends, but it made requests ~25% faster in my tests for some websites. - Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`. - This can help save your proxy usage but be careful with this option as it makes some websites never finish loading. - :param wait_selector: Wait for a specific CSS selector to be in a specific state. - :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`. - :param network_idle: Wait for the page until there are no network connections for at least 500 ms. - :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute. - :param solve_cloudflare: Solves all types of the Cloudflare's Turnstile/Interstitial challenges before returning the response to you. - :param selector_config: The arguments that will be passed in the end while creating the final Selector's class. + :param kwargs: Additional keyword arguments including: + - google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name. + - timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000 + - wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object. + - page_action: Added for automation. A function that takes the `page` object and does the automation you need. + - extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ + - disable_resources: Drop requests of unnecessary resources for a speed boost. It depends, but it made requests ~25% faster in my tests for some websites. + Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`. + This can help save your proxy usage but be careful with this option as it makes some websites never finish loading. + - wait_selector: Wait for a specific CSS selector to be in a specific state. + - wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`. + - network_idle: Wait for the page until there are no network connections for at least 500 ms. + - load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute. + - solve_cloudflare: Solves all types of the Cloudflare's Turnstile/Interstitial challenges before returning the response to you. + - selector_config: The arguments that will be passed in the end while creating the final Selector's class. :return: A `Response` object. """ - params = _validate( - [ - ("google_search", google_search, self.google_search), - ("timeout", timeout, self.timeout), - ("wait", wait, self.wait), - ("page_action", page_action, self.page_action), - ("extra_headers", extra_headers, self.extra_headers), - ("disable_resources", disable_resources, self.disable_resources), - ("wait_selector", wait_selector, self.wait_selector), - ("wait_selector_state", wait_selector_state, self.wait_selector_state), - ("network_idle", network_idle, self.network_idle), - ("load_dom", load_dom, self.load_dom), - ("solve_cloudflare", solve_cloudflare, self.solve_cloudflare), - ("selector_config", selector_config, self.selector_config), - ], - CamoufoxConfig, - _UNSET, - ) + params = _validate(kwargs, self, CamoufoxConfig) if self._closed: # pragma: no cover raise RuntimeError("Context manager has been closed") @@ -613,7 +426,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession): raise RuntimeError(f"Failed to get response for {url}") if params.solve_cloudflare: - await self._solve_cloudflare(page_info.page) + await self._cloudflare_solver(page_info.page) # Make sure the page is fully loaded after the captcha await self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle) diff --git a/scrapling/engines/_browsers/_controllers.py b/scrapling/engines/_browsers/_controllers.py index 9fc3490..9ab0e3c 100644 --- a/scrapling/engines/_browsers/_controllers.py +++ b/scrapling/engines/_browsers/_controllers.py @@ -13,92 +13,55 @@ from patchright.sync_api import sync_playwright as sync_patchright from patchright.async_api import async_playwright as async_patchright from scrapling.core.utils import log +from scrapling.core._types import Unpack, TYPE_CHECKING +from ._types import PlaywrightSession, PlaywrightFetchParams from ._base import SyncSession, AsyncSession, DynamicSessionMixin from ._validators import validate_fetch as _validate, PlaywrightConfig -from scrapling.core._types import ( - Any, - Dict, - List, - Optional, - Callable, - TYPE_CHECKING, - SelectorWaitStates, -) -from scrapling.engines.toolbelt.convertor import ( - Response, - ResponseFactory, -) +from scrapling.engines.toolbelt.convertor import Response, ResponseFactory from scrapling.engines.toolbelt.fingerprints import generate_convincing_referer -_UNSET: Any = object() - class DynamicSession(DynamicSessionMixin, SyncSession): """A Browser session manager with page pooling.""" __slots__ = ( - "max_pages", - "headless", - "hide_canvas", - "disable_webgl", - "real_chrome", - "stealth", - "google_search", - "proxy", - "locale", - "extra_headers", - "useragent", - "timeout", - "cookies", - "disable_resources", - "network_idle", - "load_dom", - "wait_selector", - "init_script", - "wait_selector_state", - "wait", + "_max_pages", + "_headless", + "_hide_canvas", + "_disable_webgl", + "_real_chrome", + "_stealth", + "_google_search", + "_proxy", + "_locale", + "_extra_headers", + "_useragent", + "_timeout", + "_cookies", + "_disable_resources", + "_network_idle", + "_load_dom", + "_wait_selector", + "_init_script", + "_wait_selector_state", + "_wait", "playwright", "browser", "context", "page_pool", "_closed", - "selector_config", - "page_action", + "_selector_config", + "_page_action", "launch_options", "context_options", - "cdp_url", + "_cdp_url", "_headers_keys", + "_extra_flags", + "_additional_args", + "_user_data_dir", ) - def __init__( - self, - __max_pages: int = 1, - headless: bool = True, - google_search: bool = True, - hide_canvas: bool = False, - disable_webgl: bool = False, - real_chrome: bool = False, - stealth: bool = False, - wait: int | float = 0, - page_action: Optional[Callable] = None, - proxy: Optional[str | Dict[str, str]] = None, - locale: str = "en-US", - extra_headers: Optional[Dict[str, str]] = None, - useragent: Optional[str] = None, - cdp_url: Optional[str] = None, - timeout: int | float = 30000, - disable_resources: bool = False, - wait_selector: Optional[str] = None, - init_script: Optional[str] = None, - cookies: Optional[List[Dict]] = None, - network_idle: bool = False, - load_dom: bool = True, - wait_selector_state: SelectorWaitStates = "attached", - user_data_dir: str = "", - extra_flags: Optional[List[str]] = None, - selector_config: Optional[Dict] = None, - additional_args: Optional[Dict] = None, - ): + def __init__(self, **kwargs: Unpack[PlaywrightSession]): """A Browser session manager with page pooling, it's using a persistent browser Context by default with a temporary user profile directory. :param headless: Run the browser in headless/hidden (default), or headful/visible mode. @@ -129,105 +92,49 @@ class DynamicSession(DynamicSessionMixin, SyncSession): :param selector_config: The arguments that will be passed in the end while creating the final Selector's class. :param additional_args: Additional arguments to be passed to Playwright's context as additional settings, and it takes higher priority than Scrapling's settings. """ - self.__validate__( - wait=wait, - proxy=proxy, - locale=locale, - timeout=timeout, - stealth=stealth, - cdp_url=cdp_url, - cookies=cookies, - load_dom=load_dom, - headless=headless, - useragent=useragent, - max_pages=__max_pages, - real_chrome=real_chrome, - page_action=page_action, - hide_canvas=hide_canvas, - init_script=init_script, - network_idle=network_idle, - user_data_dir=user_data_dir, - google_search=google_search, - extra_headers=extra_headers, - wait_selector=wait_selector, - disable_webgl=disable_webgl, - extra_flags=extra_flags, - selector_config=selector_config, - additional_args=additional_args, - disable_resources=disable_resources, - wait_selector_state=wait_selector_state, - ) - super().__init__(max_pages=self.max_pages) + self.__validate__(**kwargs) + super().__init__(max_pages=self._max_pages) def __create__(self): """Create a browser for this instance and context.""" - sync_context = sync_patchright if self.stealth else sync_playwright + sync_context = sync_patchright if self._stealth else sync_playwright self.playwright: Playwright = sync_context().start() # pyright: ignore [reportAttributeAccessIssue] - if self.cdp_url: # pragma: no cover - self.context = self.playwright.chromium.connect_over_cdp(endpoint_url=self.cdp_url).new_context( + if self._cdp_url: # pragma: no cover + self.context = self.playwright.chromium.connect_over_cdp(endpoint_url=self._cdp_url).new_context( **self.context_options ) else: self.context = self.playwright.chromium.launch_persistent_context(**self.launch_options) - if self.init_script: # pragma: no cover - self.context.add_init_script(path=self.init_script) + if self._init_script: # pragma: no cover + self.context.add_init_script(path=self._init_script) - if self.cookies: # pragma: no cover - self.context.add_cookies(self.cookies) + if self._cookies: # pragma: no cover + self.context.add_cookies(self._cookies) - def fetch( - self, - url: str, - google_search: bool = _UNSET, - timeout: int | float = _UNSET, - wait: int | float = _UNSET, - page_action: Optional[Callable] = _UNSET, - extra_headers: Optional[Dict[str, str]] = _UNSET, - disable_resources: bool = _UNSET, - wait_selector: Optional[str] = _UNSET, - wait_selector_state: SelectorWaitStates = _UNSET, - network_idle: bool = _UNSET, - load_dom: bool = _UNSET, - selector_config: Optional[Dict] = _UNSET, - ) -> Response: + def fetch(self, url: str, **kwargs: Unpack[PlaywrightFetchParams]) -> Response: """Opens up the browser and do your request based on your chosen options. :param url: The Target url. - :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name. - :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000 - :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object. - :param page_action: Added for automation. A function that takes the `page` object and does the automation you need. - :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ - :param disable_resources: Drop requests of unnecessary resources for a speed boost. It depends, but it made requests ~25% faster in my tests for some websites. - Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`. - This can help save your proxy usage but be careful with this option as it makes some websites never finish loading. - :param wait_selector: Wait for a specific CSS selector to be in a specific state. - :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`. - :param network_idle: Wait for the page until there are no network connections for at least 500 ms. - :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute. - :param selector_config: The arguments that will be passed in the end while creating the final Selector's class. + :param kwargs: Additional keyword arguments including: + - google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name. + - timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000 + - wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object. + - page_action: Added for automation. A function that takes the `page` object and does the automation you need. + - extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ + - disable_resources: Drop requests of unnecessary resources for a speed boost. It depends, but it made requests ~25% faster in my tests for some websites. + Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`. + This can help save your proxy usage but be careful with this option as it makes some websites never finish loading. + - wait_selector: Wait for a specific CSS selector to be in a specific state. + - wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`. + - network_idle: Wait for the page until there are no network connections for at least 500 ms. + - load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute. + - selector_config: The arguments that will be passed in the end while creating the final Selector's class. :return: A `Response` object. """ - params = _validate( - [ - ("google_search", google_search, self.google_search), - ("timeout", timeout, self.timeout), - ("wait", wait, self.wait), - ("page_action", page_action, self.page_action), - ("extra_headers", extra_headers, self.extra_headers), - ("disable_resources", disable_resources, self.disable_resources), - ("wait_selector", wait_selector, self.wait_selector), - ("wait_selector_state", wait_selector_state, self.wait_selector_state), - ("network_idle", network_idle, self.network_idle), - ("load_dom", load_dom, self.load_dom), - ("selector_config", selector_config, self.selector_config), - ], - PlaywrightConfig, - _UNSET, - ) + params = _validate(kwargs, self, PlaywrightConfig) if self._closed: # pragma: no cover raise RuntimeError("Context manager has been closed") @@ -285,35 +192,7 @@ class DynamicSession(DynamicSessionMixin, SyncSession): class AsyncDynamicSession(DynamicSessionMixin, AsyncSession): """An async Browser session manager with page pooling, it's using a persistent browser Context by default with a temporary user profile directory.""" - def __init__( - self, - max_pages: int = 1, - headless: bool = True, - google_search: bool = True, - hide_canvas: bool = False, - disable_webgl: bool = False, - real_chrome: bool = False, - stealth: bool = False, - wait: int | float = 0, - page_action: Optional[Callable] = None, - proxy: Optional[str | Dict[str, str]] = None, - locale: str = "en-US", - extra_headers: Optional[Dict[str, str]] = None, - useragent: Optional[str] = None, - cdp_url: Optional[str] = None, - timeout: int | float = 30000, - disable_resources: bool = False, - wait_selector: Optional[str] = None, - init_script: Optional[str] = None, - cookies: Optional[List[Dict]] = None, - network_idle: bool = False, - load_dom: bool = True, - wait_selector_state: SelectorWaitStates = "attached", - user_data_dir: str = "", - extra_flags: Optional[List[str]] = None, - selector_config: Optional[Dict] = None, - additional_args: Optional[Dict] = None, - ): + def __init__(self, **kwargs: Unpack[PlaywrightSession]): """A Browser session manager with page pooling :param headless: Run the browser in headless/hidden (default), or headful/visible mode. @@ -345,107 +224,50 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession): :param selector_config: The arguments that will be passed in the end while creating the final Selector's class. :param additional_args: Additional arguments to be passed to Playwright's context as additional settings, and it takes higher priority than Scrapling's settings. """ - - self.__validate__( - wait=wait, - proxy=proxy, - locale=locale, - timeout=timeout, - stealth=stealth, - cdp_url=cdp_url, - cookies=cookies, - load_dom=load_dom, - headless=headless, - useragent=useragent, - max_pages=max_pages, - real_chrome=real_chrome, - page_action=page_action, - hide_canvas=hide_canvas, - init_script=init_script, - network_idle=network_idle, - user_data_dir=user_data_dir, - google_search=google_search, - extra_headers=extra_headers, - wait_selector=wait_selector, - disable_webgl=disable_webgl, - extra_flags=extra_flags, - selector_config=selector_config, - additional_args=additional_args, - disable_resources=disable_resources, - wait_selector_state=wait_selector_state, - ) - super().__init__(max_pages=self.max_pages) + self.__validate__(**kwargs) + super().__init__(max_pages=self._max_pages) async def __create__(self): """Create a browser for this instance and context.""" - async_context = async_patchright if self.stealth else async_playwright + async_context = async_patchright if self._stealth else async_playwright self.playwright: AsyncPlaywright = await async_context().start() # pyright: ignore [reportAttributeAccessIssue] - if self.cdp_url: - browser = await self.playwright.chromium.connect_over_cdp(endpoint_url=self.cdp_url) + if self._cdp_url: + browser = await self.playwright.chromium.connect_over_cdp(endpoint_url=self._cdp_url) self.context: AsyncBrowserContext = await browser.new_context(**self.context_options) else: self.context: AsyncBrowserContext = await self.playwright.chromium.launch_persistent_context( **self.launch_options ) - if self.init_script: # pragma: no cover - await self.context.add_init_script(path=self.init_script) + if self._init_script: # pragma: no cover + await self.context.add_init_script(path=self._init_script) - if self.cookies: - await self.context.add_cookies(self.cookies) # pyright: ignore + if self._cookies: + await self.context.add_cookies(self._cookies) # pyright: ignore - async def fetch( - self, - url: str, - google_search: bool = _UNSET, - timeout: int | float = _UNSET, - wait: int | float = _UNSET, - page_action: Optional[Callable] = _UNSET, - extra_headers: Optional[Dict[str, str]] = _UNSET, - disable_resources: bool = _UNSET, - wait_selector: Optional[str] = _UNSET, - wait_selector_state: SelectorWaitStates = _UNSET, - network_idle: bool = _UNSET, - load_dom: bool = _UNSET, - selector_config: Optional[Dict] = _UNSET, - ) -> Response: + async def fetch(self, url: str, **kwargs: Unpack[PlaywrightFetchParams]) -> Response: """Opens up the browser and do your request based on your chosen options. :param url: The Target url. - :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name. - :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000 - :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object. - :param page_action: Added for automation. A function that takes the `page` object and does the automation you need. - :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ - :param disable_resources: Drop requests of unnecessary resources for a speed boost. It depends, but it made requests ~25% faster in my tests for some websites. - Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`. - This can help save your proxy usage but be careful with this option as it makes some websites never finish loading. - :param wait_selector: Wait for a specific CSS selector to be in a specific state. - :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`. - :param network_idle: Wait for the page until there are no network connections for at least 500 ms. - :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute. - :param selector_config: The arguments that will be passed in the end while creating the final Selector's class. + :param kwargs: Additional keyword arguments including: + - google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name. + - timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000 + - wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object. + - page_action: Added for automation. A function that takes the `page` object and does the automation you need. + - extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ + - disable_resources: Drop requests of unnecessary resources for a speed boost. It depends, but it made requests ~25% faster in my tests for some websites. + Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`. + This can help save your proxy usage but be careful with this option as it makes some websites never finish loading. + - wait_selector: Wait for a specific CSS selector to be in a specific state. + - wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`. + - network_idle: Wait for the page until there are no network connections for at least 500 ms. + - load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute. + - selector_config: The arguments that will be passed in the end while creating the final Selector's class. :return: A `Response` object. """ - params = _validate( - [ - ("google_search", google_search, self.google_search), - ("timeout", timeout, self.timeout), - ("wait", wait, self.wait), - ("page_action", page_action, self.page_action), - ("extra_headers", extra_headers, self.extra_headers), - ("disable_resources", disable_resources, self.disable_resources), - ("wait_selector", wait_selector, self.wait_selector), - ("wait_selector_state", wait_selector_state, self.wait_selector_state), - ("network_idle", network_idle, self.network_idle), - ("load_dom", load_dom, self.load_dom), - ("selector_config", selector_config, self.selector_config), - ], - PlaywrightConfig, - _UNSET, - ) + params = _validate(kwargs, self, PlaywrightConfig) if self._closed: # pragma: no cover raise RuntimeError("Context manager has been closed") diff --git a/scrapling/engines/_browsers/_page.py b/scrapling/engines/_browsers/_page.py index 821fbbb..655d3d1 100644 --- a/scrapling/engines/_browsers/_page.py +++ b/scrapling/engines/_browsers/_page.py @@ -1,20 +1,21 @@ from threading import RLock from dataclasses import dataclass -from playwright.sync_api import Page as SyncPage -from playwright.async_api import Page as AsyncPage +from playwright.sync_api._generated import Page as SyncPage +from playwright.async_api._generated import Page as AsyncPage -from scrapling.core._types import Optional, List, Literal +from scrapling.core._types import Optional, List, Literal, overload, TypeVar, Generic, cast PageState = Literal["ready", "busy", "error"] # States that a page can be in +PageType = TypeVar("PageType", SyncPage, AsyncPage) @dataclass -class PageInfo: +class PageInfo(Generic[PageType]): """Information about the page and its current state""" __slots__ = ("page", "state", "url") - page: SyncPage | AsyncPage + page: PageType state: PageState url: Optional[str] @@ -44,16 +45,26 @@ class PagePool: def __init__(self, max_pages: int = 5): self.max_pages = max_pages - self.pages: List[PageInfo] = [] + self.pages: List[PageInfo[SyncPage] | PageInfo[AsyncPage]] = [] self._lock = RLock() - def add_page(self, page: SyncPage | AsyncPage) -> PageInfo: + @overload + def add_page(self, page: SyncPage) -> PageInfo[SyncPage]: ... + + @overload + def add_page(self, page: AsyncPage) -> PageInfo[AsyncPage]: ... + + def add_page(self, page: SyncPage | AsyncPage) -> PageInfo[SyncPage] | PageInfo[AsyncPage]: """Add a new page to the pool""" with self._lock: if len(self.pages) >= self.max_pages: raise RuntimeError(f"Maximum page limit ({self.max_pages}) reached") - page_info = PageInfo(page, "ready", "") + if isinstance(page, AsyncPage): + page_info = cast(PageInfo[AsyncPage], PageInfo(page, "ready", "")) + else: + page_info = cast(PageInfo[SyncPage], PageInfo(page, "ready", "")) + self.pages.append(page_info) return page_info diff --git a/scrapling/engines/_browsers/_types.py b/scrapling/engines/_browsers/_types.py index 0dd0c91..d1edca5 100644 --- a/scrapling/engines/_browsers/_types.py +++ b/scrapling/engines/_browsers/_types.py @@ -10,8 +10,11 @@ from scrapling.core._types import ( Tuple, Mapping, Optional, + Callable, + Iterable, TypedDict, TypeAlias, + SelectorWaitStates, TYPE_CHECKING, ) @@ -49,7 +52,69 @@ if TYPE_CHECKING: # pragma: no cover data: Optional[Dict | str] json: Optional[Dict | List] + # Types for browser session + class BrowserSession(TypedDict, total=False): + max_pages: int + headless: bool + disable_resources: bool + network_idle: bool + load_dom: bool + wait_selector: Optional[str] + wait_selector_state: SelectorWaitStates + cookies: Optional[Iterable[Dict]] + google_search: bool + wait: int | float + page_action: Optional[Callable] + proxy: Optional[str | Dict[str, str] | Tuple] + extra_headers: Optional[Dict[str, str]] + timeout: int | float + init_script: Optional[str] + user_data_dir: str + selector_config: Optional[Dict] + additional_args: Optional[Dict] + + class PlaywrightSession(BrowserSession, total=False): + cdp_url: Optional[str] + hide_canvas: bool + disable_webgl: bool + real_chrome: bool + stealth: bool + locale: str + useragent: Optional[str] + extra_flags: Optional[List[str]] + + class PlaywrightFetchParams(TypedDict, total=False): + google_search: bool + timeout: int | float + wait: int | float + page_action: Optional[Callable] + extra_headers: Optional[Dict[str, str]] + disable_resources: bool + wait_selector: Optional[str] + wait_selector_state: SelectorWaitStates + network_idle: bool + load_dom: bool + selector_config: Optional[Dict] + + class CamoufoxSession(BrowserSession, total=False): + block_images: bool + block_webrtc: bool + allow_webgl: bool + humanize: bool | float + solve_cloudflare: bool + addons: Optional[List[str]] + os_randomize: bool + disable_ads: bool + geoip: bool + + class CamoufoxFetchParams(PlaywrightFetchParams, total=False): + solve_cloudflare: bool + else: # pragma: no cover RequestsSession = TypedDict GetRequestParams = TypedDict DataRequestParams = TypedDict + PlaywrightSession = TypedDict + PlaywrightFetchParams = TypedDict + CamoufoxSession = TypedDict + CamoufoxFetchParams = TypedDict diff --git a/scrapling/engines/_browsers/_validators.py b/scrapling/engines/_browsers/_validators.py index 5442102..a867a37 100644 --- a/scrapling/engines/_browsers/_validators.py +++ b/scrapling/engines/_browsers/_validators.py @@ -7,6 +7,7 @@ from dataclasses import dataclass, fields from msgspec import Struct, Meta, convert, ValidationError from scrapling.core._types import ( + Any, Dict, List, Tuple, @@ -17,6 +18,7 @@ from scrapling.core._types import ( overload, ) from scrapling.engines.toolbelt.navigation import construct_proxy_dict +from scrapling.engines._browsers._types import PlaywrightFetchParams, CamoufoxFetchParams # Custom validators for msgspec @@ -194,16 +196,24 @@ class _fetch_params: def validate_fetch( - params: List[Tuple], model: type[PlaywrightConfig] | type[CamoufoxConfig], sentinel=None + method_kwargs: Dict | PlaywrightFetchParams | CamoufoxFetchParams, + session: Any, + model: type[PlaywrightConfig] | type[CamoufoxConfig], ) -> _fetch_params: # pragma: no cover result = {} overrides = {} - for arg, request_value, session_value in params: - if request_value is not sentinel: - overrides[arg] = request_value + # Get all field names that _fetch_params needs + fetch_param_fields = {f.name for f in fields(_fetch_params)} + + for key in fetch_param_fields: + if key in method_kwargs: + overrides[key] = method_kwargs[key] else: - result[arg] = session_value + # Check for underscore-prefixed attribute (private) + attr_name = f"_{key}" + if hasattr(session, attr_name): + result[key] = getattr(session, attr_name) if overrides: validated_config = validate(overrides, model)