From 170205599d45eb1407c9c1a450fc9274d0e88838 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Sat, 22 Nov 2025 20:24:55 +0200 Subject: [PATCH 01/21] fix(parser): Better approach for web pages where the encoding is not always correctly declared Fixes #110 and avoids defaulting to a specific encoding like #111 --- scrapling/parser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapling/parser.py b/scrapling/parser.py index ac97b7d..95b88ee 100644 --- a/scrapling/parser.py +++ b/scrapling/parser.py @@ -121,7 +121,7 @@ class Selector(SelectorsGeneration): self.__text = None if root is None: if isinstance(content, str): - body = content.strip().replace("\x00", "").encode(encoding) or b"" + body = content.strip().replace("\x00", "") or "" elif isinstance(content, bytes): body = content.replace(b"\x00", b"") else: From 2bae3261a04f60cf82fae59bbb364d0e102750d2 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Sat, 22 Nov 2025 21:41:47 +0200 Subject: [PATCH 02/21] build: Pumping version and deps up This also updates the Maxmind database for camoufox --- pyproject.toml | 8 ++++---- scrapling/__init__.py | 2 +- setup.cfg | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 558bf55..0855739 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta" [project] name = "scrapling" # Static version instead of a dynamic version so we can get better layer caching while building docker, check the docker file to understand -version = "0.3.9" +version = "0.3.10" description = "Scrapling is an undetectable, powerful, flexible, high-performance Python library that makes Web Scraping easy and effortless as it should be!" readme = {file = "docs/README.md", content-type = "text/markdown"} license = {file = "LICENSE"} @@ -67,10 +67,10 @@ dependencies = [ fetchers = [ "click>=8.3.0", "curl_cffi>=0.13.0", - "playwright>=1.55.0", - "patchright>=1.55.2", + "playwright>=1.56.0", + "patchright>=1.56.0", "camoufox>=0.4.11", - "geoip2>=5.1.0", + "geoip2>=5.2.0", "msgspec>=0.19.0", ] ai = [ diff --git a/scrapling/__init__.py b/scrapling/__init__.py index bbbb02b..214a0f9 100644 --- a/scrapling/__init__.py +++ b/scrapling/__init__.py @@ -1,5 +1,5 @@ __author__ = "Karim Shoair (karim.shoair@pm.me)" -__version__ = "0.3.9" +__version__ = "0.3.10" __copyright__ = "Copyright (c) 2024 Karim Shoair" from typing import Any, TYPE_CHECKING diff --git a/setup.cfg b/setup.cfg index 54350cc..8a55f35 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = scrapling -version = 0.3.9 +version = 0.3.10 author = Karim Shoair author_email = karim.shoair@pm.me description = Scrapling is an undetectable, powerful, flexible, high-performance Python library that makes Web Scraping easy and effortless as it should be! From 357b170e1c54fd5d06b417430da1cd114d7f32ec Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Sun, 23 Nov 2025 20:48:51 +0200 Subject: [PATCH 03/21] refactor(requests): Make all the type hints dynamic Made the code shorter by about ~500 lines and easier to maintain in return for making the arguments autocompletion bad for shells that don't check for dynamic type hints like IPython. --- scrapling/core/_types.py | 13 + scrapling/engines/_browsers/_types.py | 55 ++ scrapling/engines/static.py | 983 +++++++------------------- 3 files changed, 340 insertions(+), 711 deletions(-) create mode 100644 scrapling/engines/_browsers/_types.py diff --git a/scrapling/core/_types.py b/scrapling/core/_types.py index 51016d4..bd83f49 100644 --- a/scrapling/core/_types.py +++ b/scrapling/core/_types.py @@ -4,6 +4,8 @@ Type definitions for type checking purposes. from typing import ( TYPE_CHECKING, + TypedDict, + TypeAlias, cast, overload, Any, @@ -34,6 +36,17 @@ PageLoadStates = Literal["commit", "domcontentloaded", "load", "networkidle"] extraction_types = Literal["text", "html", "markdown"] StrOrBytes = Union[str, bytes] +if TYPE_CHECKING: # pragma: no cover + from typing_extensions import Unpack +else: # pragma: no cover + + class _Unpack: + @staticmethod + def __getitem__(*args, **kwargs): + pass + + Unpack = _Unpack() + try: # Python 3.11+ diff --git a/scrapling/engines/_browsers/_types.py b/scrapling/engines/_browsers/_types.py new file mode 100644 index 0000000..0dd0c91 --- /dev/null +++ b/scrapling/engines/_browsers/_types.py @@ -0,0 +1,55 @@ +from curl_cffi.requests import ( + ProxySpec, + CookieTypes, + BrowserTypeLiteral, +) + +from scrapling.core._types import ( + Dict, + List, + Tuple, + Mapping, + Optional, + TypedDict, + TypeAlias, + TYPE_CHECKING, +) + +# Type alias for `impersonate` parameter - accepts a single browser or list of browsers +ImpersonateType: TypeAlias = BrowserTypeLiteral | List[BrowserTypeLiteral] | None + + +if TYPE_CHECKING: # pragma: no cover + # Types for session initialization + class RequestsSession(TypedDict, total=False): + impersonate: ImpersonateType + http3: Optional[bool] + stealthy_headers: Optional[bool] + proxies: Optional[ProxySpec] + proxy: Optional[str] + proxy_auth: Optional[Tuple[str, str]] + timeout: Optional[int | float] + headers: Optional[Mapping[str, Optional[str]]] + retries: Optional[int] + retry_delay: Optional[int] + follow_redirects: Optional[bool] + max_redirects: Optional[int] + verify: Optional[bool] + cert: Optional[str | Tuple[str, str]] + selector_config: Optional[Dict] + + # Types for GET request method parameters + class GetRequestParams(RequestsSession, total=False): + params: Optional[Dict | List | Tuple] + cookies: Optional[CookieTypes] + auth: Optional[Tuple[str, str]] + + # Types for POST/PUT/DELETE request method parameters + class DataRequestParams(GetRequestParams, total=False): + data: Optional[Dict | str] + json: Optional[Dict | List] + +else: # pragma: no cover + RequestsSession = TypedDict + GetRequestParams = TypedDict + DataRequestParams = TypedDict diff --git a/scrapling/engines/static.py b/scrapling/engines/static.py index 6f68737..7251209 100644 --- a/scrapling/engines/static.py +++ b/scrapling/engines/static.py @@ -6,8 +6,6 @@ from asyncio import sleep as asyncio_sleep from curl_cffi.curl import CurlError from curl_cffi import CurlHttpVersion from curl_cffi.requests import ( - ProxySpec, - CookieTypes, BrowserTypeLiteral, Session as CurlSession, AsyncSession as AsyncCurlSession, @@ -15,26 +13,22 @@ from curl_cffi.requests import ( from scrapling.core.utils import log from scrapling.core._types import ( - Dict, - Optional, - Tuple, - Mapping, - SUPPORTED_HTTP_METHODS, - Awaitable, - List, Any, + Dict, + Tuple, + Unpack, + Optional, + Awaitable, + SUPPORTED_HTTP_METHODS, ) +from ._browsers._types import RequestsSession, GetRequestParams, DataRequestParams, ImpersonateType from .toolbelt.custom import Response from .toolbelt.convertor import ResponseFactory from .toolbelt.fingerprints import generate_convincing_referer, generate_headers, __default_useragent__ -_UNSET: Any = object() _NO_SESSION: Any = object() -# Type alias for `impersonate` parameter - accepts a single browser or list of browsers -ImpersonateType = BrowserTypeLiteral | List[BrowserTypeLiteral] | None - def _select_random_browser(impersonate: ImpersonateType) -> Optional[BrowserTypeLiteral]: """ @@ -52,82 +46,81 @@ def _select_random_browser(impersonate: ImpersonateType) -> Optional[BrowserType class _ConfigurationLogic(ABC): # Core Logic Handler (Internal Engine) - def __init__( - self, - impersonate: ImpersonateType = "chrome", - http3: Optional[bool] = False, - stealthy_headers: Optional[bool] = True, - proxies: Optional[Dict[str, str]] = None, - proxy: Optional[str] = None, - proxy_auth: Optional[Tuple[str, str]] = None, - timeout: Optional[int | float] = 30, - headers: Optional[Dict[str, str]] = None, - retries: Optional[int] = 3, - retry_delay: Optional[int] = 1, - follow_redirects: bool = True, - max_redirects: int = 30, - verify: bool = True, - cert: Optional[str | Tuple[str, str]] = None, - selector_config: Optional[Dict] = None, - ): - self._default_impersonate = impersonate - self._stealth = stealthy_headers - self._default_proxies = proxies or {} - self._default_proxy = proxy or None - self._default_proxy_auth = proxy_auth or None - self._default_timeout = timeout - self._default_headers = headers or {} - self._default_retries = retries - self._default_retry_delay = retry_delay - self._default_follow_redirects = follow_redirects - self._default_max_redirects = max_redirects - self._default_verify = verify - self._default_cert = cert - self._default_http3 = http3 - self.selector_config = selector_config or {} + def __init__(self, **kwargs: Unpack[RequestsSession]): + self._default_impersonate = kwargs.get("impersonate", "chrome") + self._stealth = kwargs.get("stealthy_headers", True) + self._default_proxies = kwargs.get("proxies") or {} + self._default_proxy = kwargs.get("proxy") or None + self._default_proxy_auth = kwargs.get("proxy_auth") or None + self._default_timeout = kwargs.get("timeout", 30) + self._default_headers = kwargs.get("headers") or {} + self._default_retries = kwargs.get("retries", 3) + self._default_retry_delay = kwargs.get("retry_delay", 1) + self._default_follow_redirects = kwargs.get("follow_redirects", True) + self._default_max_redirects = kwargs.get("max_redirects", 30) + self._default_verify = kwargs.get("verify", True) + self._default_cert = kwargs.get("cert") or None + self._default_http3 = kwargs.get("http3", False) + self.selector_config = kwargs.get("selector_config") or {} @staticmethod - def _get_with_precedence(request_val: Any, default_val: Any) -> Any: - """Get value with request-level priority over session-level""" - return request_val if request_val is not _UNSET else default_val + def _get_param(kwargs: Dict, key: str, default: Any) -> Any: + """Get parameter from kwargs if present, otherwise return default.""" + return kwargs[key] if key in kwargs else default def _merge_request_args(self, **method_kwargs) -> Dict[str, Any]: """Merge request-specific arguments with default session arguments.""" url = method_kwargs.pop("url") - impersonate = _select_random_browser( - self._get_with_precedence(method_kwargs.pop("impersonate"), self._default_impersonate) - ) - http3_enabled = self._get_with_precedence(method_kwargs.pop("http3"), self._default_http3) + + # Get parameters from kwargs or use defaults + impersonate = self._get_param(method_kwargs, "impersonate", self._default_impersonate) + impersonate = _select_random_browser(impersonate) + http3_enabled = self._get_param(method_kwargs, "http3", self._default_http3) + stealth = self._get_param(method_kwargs, "stealth", self._stealth) + final_args = { "url": url, # Curl automatically generates the suitable browser headers when you use `impersonate` "headers": self._headers_job( url, - self._get_with_precedence(method_kwargs.pop("headers"), self._default_headers), - self._get_with_precedence(method_kwargs.pop("stealth"), self._stealth), + self._get_param(method_kwargs, "headers", self._default_headers), + stealth, bool(impersonate), ), - "proxies": self._get_with_precedence(method_kwargs.pop("proxies"), self._default_proxies), - "proxy": self._get_with_precedence(method_kwargs.pop("proxy"), self._default_proxy), - "proxy_auth": self._get_with_precedence(method_kwargs.pop("proxy_auth"), self._default_proxy_auth), - "timeout": self._get_with_precedence(method_kwargs.pop("timeout"), self._default_timeout), - "allow_redirects": self._get_with_precedence( - method_kwargs.pop("follow_redirects"), self._default_follow_redirects - ), - "max_redirects": self._get_with_precedence(method_kwargs.pop("max_redirects"), self._default_max_redirects), - "verify": self._get_with_precedence(method_kwargs.pop("verify"), self._default_verify), - "cert": self._get_with_precedence(method_kwargs.pop("cert"), self._default_cert), + "proxies": self._get_param(method_kwargs, "proxies", self._default_proxies), + "proxy": self._get_param(method_kwargs, "proxy", self._default_proxy), + "proxy_auth": self._get_param(method_kwargs, "proxy_auth", self._default_proxy_auth), + "timeout": self._get_param(method_kwargs, "timeout", self._default_timeout), + "allow_redirects": self._get_param(method_kwargs, "follow_redirects", self._default_follow_redirects), + "max_redirects": self._get_param(method_kwargs, "max_redirects", self._default_max_redirects), + "verify": self._get_param(method_kwargs, "verify", self._default_verify), + "cert": self._get_param(method_kwargs, "cert", self._default_cert), "impersonate": impersonate, - **{ - k: v - for k, v in method_kwargs.items() - if v - not in ( - _UNSET, - None, - ) - }, # Add any remaining parameters (after all known ones are popped) } + + # Add any remaining parameters that weren't explicitly handled above + # Skip the ones we already processed plus internal params + skip_keys = { + "impersonate", + "http3", + "stealth", + "headers", + "proxies", + "proxy", + "proxy_auth", + "timeout", + "follow_redirects", + "max_redirects", + "verify", + "cert", + "retries", + "retry_delay", + "selector_config", + } + for k, v in method_kwargs.items(): + if k not in skip_keys and v is not None: + final_args[k] = v + if http3_enabled: # pragma: no cover final_args["http_version"] = CurlHttpVersion.V3ONLY if impersonate: @@ -144,7 +137,7 @@ class _ConfigurationLogic(ABC): 3. Generates a referer header that looks like as if this request came from a Google's search of the current URL's domain. """ # Merge session headers with request headers, request takes precedence (if it was set) - final_headers = {**self._default_headers, **(headers if headers and headers is not _UNSET else {})} + final_headers = {**self._default_headers, **(headers if headers else {})} headers_keys = {k.lower() for k in final_headers} if stealth: if "referer" not in headers_keys: @@ -156,7 +149,7 @@ class _ConfigurationLogic(ABC): {k: v for k, v in extra_headers.items() if k.lower() not in headers_keys} ) # Don't overwrite user-supplied headers - elif "user-agent" not in headers_keys and not impersonate_enabled: + elif "user-agent" not in headers_keys and not impersonate_enabled: # pragma: no cover final_headers["User-Agent"] = __default_useragent__ log.debug(f"Can't find useragent in headers so '{final_headers['User-Agent']}' was used.") @@ -164,41 +157,8 @@ class _ConfigurationLogic(ABC): class _SyncSessionLogic(_ConfigurationLogic): - def __init__( - self, - impersonate: ImpersonateType = "chrome", - http3: Optional[bool] = False, - stealthy_headers: Optional[bool] = True, - proxies: Optional[Dict[str, str]] = None, - proxy: Optional[str] = None, - proxy_auth: Optional[Tuple[str, str]] = None, - timeout: Optional[int | float] = 30, - headers: Optional[Dict[str, str]] = None, - retries: Optional[int] = 3, - retry_delay: Optional[int] = 1, - follow_redirects: bool = True, - max_redirects: int = 30, - verify: bool = True, - cert: Optional[str | Tuple[str, str]] = None, - selector_config: Optional[Dict] = None, - ): - super().__init__( - impersonate, - http3, - stealthy_headers, - proxies, - proxy, - proxy_auth, - timeout, - headers, - retries, - retry_delay, - follow_redirects, - max_redirects, - verify, - cert, - selector_config, - ) + def __init__(self, **kwargs: Unpack[RequestsSession]): + super().__init__(**kwargs) self._curl_session: Optional[CurlSession] = None def __enter__(self): @@ -221,20 +181,15 @@ class _SyncSessionLogic(_ConfigurationLogic): self._curl_session.close() self._curl_session = None - def __make_request( - self, - method: SUPPORTED_HTTP_METHODS, - stealth: Optional[bool] = None, - **kwargs, - ) -> Response: + def __make_request(self, method: SUPPORTED_HTTP_METHODS, stealth: Optional[bool] = None, **kwargs) -> Response: """ Perform an HTTP request using the configured session. """ stealth = self._stealth if stealth is None else stealth - selector_config = kwargs.pop("selector_config", {}) or self.selector_config - max_retries = self._get_with_precedence(kwargs.pop("retries"), self._default_retries) - retry_delay = self._get_with_precedence(kwargs.pop("retry_delay"), self._default_retry_delay) + selector_config = self._get_param(kwargs, "selector_config", self.selector_config) or self.selector_config + max_retries = self._get_param(kwargs, "retries", self._default_retries) + retry_delay = self._get_param(kwargs, "retry_delay", self._default_retry_delay) request_args = self._merge_request_args(stealth=stealth, **kwargs) session = self._curl_session @@ -264,350 +219,135 @@ class _SyncSessionLogic(_ConfigurationLogic): raise RuntimeError("No active session available.") # pragma: no cover - def get( - self, - url: str, - params: Optional[Dict | List | Tuple] = None, - headers: Optional[Mapping[str, Optional[str]]] = _UNSET, - cookies: Optional[CookieTypes] = None, - timeout: Optional[int | float] = _UNSET, - follow_redirects: Optional[bool] = _UNSET, - max_redirects: Optional[int] = _UNSET, - retries: Optional[int] = _UNSET, - retry_delay: Optional[int] = _UNSET, - proxies: Optional[ProxySpec] = _UNSET, - proxy: Optional[str] = _UNSET, - proxy_auth: Optional[Tuple[str, str]] = _UNSET, - auth: Optional[Tuple[str, str]] = None, - verify: Optional[bool] = _UNSET, - cert: Optional[str | Tuple[str, str]] = _UNSET, - impersonate: ImpersonateType = _UNSET, - http3: Optional[bool] = _UNSET, - stealthy_headers: Optional[bool] = _UNSET, - **kwargs, - ) -> Response: + def get(self, url: str, **kwargs: Unpack[GetRequestParams]) -> Response: """ Perform a GET request. + Any additional keyword arguments are passed to the [`curl_cffi.requests.Session().request()`, `curl_cffi.requests.AsyncSession().request()`] method. + :param url: Target URL for the request. - :param params: Query string parameters for the request. - :param headers: Headers to include in the request. - :param cookies: Cookies to use in the request. - :param timeout: Number of seconds to wait before timing out. - :param follow_redirects: Whether to follow redirects. Defaults to True. - :param max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited. - :param retries: Number of retry attempts. Defaults to 3. - :param retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second. - :param proxies: Dict of proxies to use. - :param proxy: Proxy URL to use. Format: "http://username:password@localhost:8030". - Cannot be used together with the `proxies` parameter. - :param proxy_auth: HTTP basic auth for proxy, tuple of (username, password). - :param auth: HTTP basic auth tuple of (username, password). Only basic auth is supported. - :param verify: Whether to verify HTTPS certificates. - :param cert: Tuple of (cert, key) filenames for the client certificate. - :param impersonate: Browser version to impersonate. Automatically defaults to the latest available Chrome version. - :param http3: Whether to use HTTP3. Defaults to False. It might be problematic if used it with `impersonate`. - :param stealthy_headers: If enabled (default), it creates and adds real browser headers. It also sets the referer header as if this request came from a Google search of URL's domain. - :param kwargs: Additional keyword arguments to pass to the [`curl_cffi.requests.Session().request()`, `curl_cffi.requests.AsyncSession().request()`] method. + :param kwargs: Additional keyword arguments including: + - params: Query string parameters for the request. + - headers: Headers to include in the request. + - cookies: Cookies to use in the request. + - timeout: Number of seconds to wait before timing out. + - follow_redirects: Whether to follow redirects. Defaults to True. + - max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited. + - retries: Number of retry attempts. Defaults to 3. + - retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second. + - proxies: Dict of proxies to use. + - proxy: Proxy URL to use. Format: "http://username:password@localhost:8030". + - proxy_auth: HTTP basic auth for proxy, tuple of (username, password). + - auth: HTTP basic auth tuple of (username, password). Only basic auth is supported. + - verify: Whether to verify HTTPS certificates. + - cert: Tuple of (cert, key) filenames for the client certificate. + - impersonate: Browser version to impersonate. Automatically defaults to the latest available Chrome version. + - http3: Whether to use HTTP3. Defaults to False. It might be problematic if used it with `impersonate`. + - stealthy_headers: If enabled (default), it creates and adds real browser headers. :return: A `Response` object. """ - method_args = {k: v for k, v in locals().items() if k not in ("self", "stealthy_headers", "kwargs")} - method_args.update(kwargs) - # For type checking (not accessed error) - _ = ( - url, - params, - headers, - cookies, - timeout, - follow_redirects, - max_redirects, - retries, - retry_delay, - proxies, - proxy, - proxy_auth, - auth, - verify, - cert, - impersonate, - http3, - ) - return self.__make_request("GET", stealth=stealthy_headers, **method_args) + stealthy_headers = kwargs.pop("stealthy_headers", None) + return self.__make_request("GET", stealth=stealthy_headers, url=url, **kwargs) - def post( - self, - url: str, - data: Optional[Dict | str] = None, - json: Optional[Dict | List] = None, - headers: Optional[Mapping[str, Optional[str]]] = _UNSET, - params: Optional[Dict | List | Tuple] = None, - cookies: Optional[CookieTypes] = None, - timeout: Optional[int | float] = _UNSET, - follow_redirects: Optional[bool] = _UNSET, - max_redirects: Optional[int] = _UNSET, - retries: Optional[int] = _UNSET, - retry_delay: Optional[int] = _UNSET, - proxies: Optional[ProxySpec] = _UNSET, - proxy: Optional[str] = _UNSET, - proxy_auth: Optional[Tuple[str, str]] = _UNSET, - auth: Optional[Tuple[str, str]] = None, - verify: Optional[bool] = _UNSET, - cert: Optional[str | Tuple[str, str]] = _UNSET, - impersonate: ImpersonateType = _UNSET, - http3: Optional[bool] = _UNSET, - stealthy_headers: Optional[bool] = _UNSET, - **kwargs, - ) -> Response: + def post(self, url: str, **kwargs: Unpack[DataRequestParams]) -> Response: """ Perform a POST request. :param url: Target URL for the request. - :param data: Form data to include in the request body. - :param json: A JSON serializable object to include in the body of the request. - :param params: Query string parameters for the request. - :param headers: Headers to include in the request. - :param cookies: Cookies to use in the request. - :param timeout: Number of seconds to wait before timing out. - :param follow_redirects: Whether to follow redirects. Defaults to True. - :param max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited. - :param retries: Number of retry attempts. Defaults to 3. - :param retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second. - :param proxies: Dict of proxies to use. - :param proxy: Proxy URL to use. Format: "http://username:password@localhost:8030". - Cannot be used together with the `proxies` parameter. - :param proxy_auth: HTTP basic auth for proxy, tuple of (username, password). - :param auth: HTTP basic auth tuple of (username, password). Only basic auth is supported. - :param verify: Whether to verify HTTPS certificates. - :param cert: Tuple of (cert, key) filenames for the client certificate. - :param impersonate: Browser version to impersonate. Automatically defaults to the latest available Chrome version. - :param http3: Whether to use HTTP3. Defaults to False. It might be problematic if used it with `impersonate`. - :param stealthy_headers: If enabled (default), it creates and adds real browser headers. It also sets the referer header as if this request came from a Google search of URL's domain. - :param kwargs: Additional keyword arguments to pass to the [`curl_cffi.requests.Session().request()`, `curl_cffi.requests.AsyncSession().request()`] method. + :param kwargs: Additional keyword arguments including: + - data: Form data to include in the request body. + - json: A JSON serializable object to include in the body of the request. + - params: Query string parameters for the request. + - headers: Headers to include in the request. + - cookies: Cookies to use in the request. + - timeout: Number of seconds to wait before timing out. + - follow_redirects: Whether to follow redirects. Defaults to True. + - max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited. + - retries: Number of retry attempts. Defaults to 3. + - retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second. + - proxies: Dict of proxies to use. + - proxy: Proxy URL to use. Format: "http://username:password@localhost:8030". + - proxy_auth: HTTP basic auth for proxy, tuple of (username, password). + - auth: HTTP basic auth tuple of (username, password). Only basic auth is supported. + - verify: Whether to verify HTTPS certificates. + - cert: Tuple of (cert, key) filenames for the client certificate. + - impersonate: Browser version to impersonate. Automatically defaults to the latest available Chrome version. + - http3: Whether to use HTTP3. Defaults to False. It might be problematic if used it with `impersonate`. + - stealthy_headers: If enabled (default), it creates and adds real browser headers. :return: A `Response` object. """ - method_args = {k: v for k, v in locals().items() if k not in ("self", "stealthy_headers", "kwargs")} - method_args.update(kwargs) - # For type checking (not accessed error) - _ = ( - url, - params, - headers, - data, - json, - cookies, - timeout, - follow_redirects, - max_redirects, - retries, - retry_delay, - proxies, - proxy, - proxy_auth, - auth, - verify, - cert, - impersonate, - http3, - ) - return self.__make_request("POST", stealth=stealthy_headers, **method_args) + stealthy_headers = kwargs.pop("stealthy_headers", None) + return self.__make_request("POST", stealth=stealthy_headers, url=url, **kwargs) - def put( - self, - url: str, - data: Optional[Dict | str] = None, - json: Optional[Dict | List] = None, - headers: Optional[Mapping[str, Optional[str]]] = _UNSET, - params: Optional[Dict | List | Tuple] = None, - cookies: Optional[CookieTypes] = None, - timeout: Optional[int | float] = _UNSET, - follow_redirects: Optional[bool] = _UNSET, - max_redirects: Optional[int] = _UNSET, - retries: Optional[int] = _UNSET, - retry_delay: Optional[int] = _UNSET, - proxies: Optional[ProxySpec] = _UNSET, - proxy: Optional[str] = _UNSET, - proxy_auth: Optional[Tuple[str, str]] = _UNSET, - auth: Optional[Tuple[str, str]] = None, - verify: Optional[bool] = _UNSET, - cert: Optional[str | Tuple[str, str]] = _UNSET, - impersonate: ImpersonateType = _UNSET, - http3: Optional[bool] = _UNSET, - stealthy_headers: Optional[bool] = _UNSET, - **kwargs, - ) -> Response: + def put(self, url: str, **kwargs: Unpack[DataRequestParams]) -> Response: """ Perform a PUT request. :param url: Target URL for the request. - :param data: Form data to include in the request body. - :param json: A JSON serializable object to include in the body of the request. - :param params: Query string parameters for the request. - :param headers: Headers to include in the request. - :param cookies: Cookies to use in the request. - :param timeout: Number of seconds to wait before timing out. - :param follow_redirects: Whether to follow redirects. Defaults to True. - :param max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited. - :param retries: Number of retry attempts. Defaults to 3. - :param retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second. - :param proxies: Dict of proxies to use. - :param proxy: Proxy URL to use. Format: "http://username:password@localhost:8030". - Cannot be used together with the `proxies` parameter. - :param proxy_auth: HTTP basic auth for proxy, tuple of (username, password). - :param auth: HTTP basic auth tuple of (username, password). Only basic auth is supported. - :param verify: Whether to verify HTTPS certificates. - :param cert: Tuple of (cert, key) filenames for the client certificate. - :param impersonate: Browser version to impersonate. Automatically defaults to the latest available Chrome version. - :param http3: Whether to use HTTP3. Defaults to False. It might be problematic if used it with `impersonate`. - :param stealthy_headers: If enabled (default), it creates and adds real browser headers. It also sets the referer header as if this request came from a Google search of URL's domain. - :param kwargs: Additional keyword arguments to pass to the [`curl_cffi.requests.Session().request()`, `curl_cffi.requests.AsyncSession().request()`] method. + :param kwargs: Additional keyword arguments including: + - data: Form data to include in the request body. + - json: A JSON serializable object to include in the body of the request. + - params: Query string parameters for the request. + - headers: Headers to include in the request. + - cookies: Cookies to use in the request. + - timeout: Number of seconds to wait before timing out. + - follow_redirects: Whether to follow redirects. Defaults to True. + - max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited. + - retries: Number of retry attempts. Defaults to 3. + - retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second. + - proxies: Dict of proxies to use. + - proxy: Proxy URL to use. Format: "http://username:password@localhost:8030". + - proxy_auth: HTTP basic auth for proxy, tuple of (username, password). + - auth: HTTP basic auth tuple of (username, password). Only basic auth is supported. + - verify: Whether to verify HTTPS certificates. + - cert: Tuple of (cert, key) filenames for the client certificate. + - impersonate: Browser version to impersonate. Automatically defaults to the latest available Chrome version. + - http3: Whether to use HTTP3. Defaults to False. It might be problematic if used it with `impersonate`. + - stealthy_headers: If enabled (default), it creates and adds real browser headers. :return: A `Response` object. """ - method_args = {k: v for k, v in locals().items() if k not in ("self", "stealthy_headers", "kwargs")} - method_args.update(kwargs) - # For type checking (not accessed error) - _ = ( - url, - params, - headers, - data, - json, - cookies, - timeout, - follow_redirects, - max_redirects, - retries, - retry_delay, - proxies, - proxy, - proxy_auth, - auth, - verify, - cert, - impersonate, - http3, - ) - return self.__make_request("PUT", stealth=stealthy_headers, **method_args) + stealthy_headers = kwargs.pop("stealthy_headers", None) + return self.__make_request("PUT", stealth=stealthy_headers, url=url, **kwargs) - def delete( - self, - url: str, - data: Optional[Dict | str] = None, - json: Optional[Dict | List] = None, - headers: Optional[Mapping[str, Optional[str]]] = _UNSET, - params: Optional[Dict | List | Tuple] = None, - cookies: Optional[CookieTypes] = None, - timeout: Optional[int | float] = _UNSET, - follow_redirects: Optional[bool] = _UNSET, - max_redirects: Optional[int] = _UNSET, - retries: Optional[int] = _UNSET, - retry_delay: Optional[int] = _UNSET, - proxies: Optional[ProxySpec] = _UNSET, - proxy: Optional[str] = _UNSET, - proxy_auth: Optional[Tuple[str, str]] = _UNSET, - auth: Optional[Tuple[str, str]] = None, - verify: Optional[bool] = _UNSET, - cert: Optional[str | Tuple[str, str]] = _UNSET, - impersonate: ImpersonateType = _UNSET, - http3: Optional[bool] = _UNSET, - stealthy_headers: Optional[bool] = _UNSET, - **kwargs, - ) -> Response: + def delete(self, url: str, **kwargs: Unpack[DataRequestParams]) -> Response: """ Perform a DELETE request. :param url: Target URL for the request. - :param data: Form data to include in the request body. - :param json: A JSON serializable object to include in the body of the request. - :param params: Query string parameters for the request. - :param headers: Headers to include in the request. - :param cookies: Cookies to use in the request. - :param timeout: Number of seconds to wait before timing out. - :param follow_redirects: Whether to follow redirects. Defaults to True. - :param max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited. - :param retries: Number of retry attempts. Defaults to 3. - :param retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second. - :param proxies: Dict of proxies to use. - :param proxy: Proxy URL to use. Format: "http://username:password@localhost:8030". - Cannot be used together with the `proxies` parameter. - :param proxy_auth: HTTP basic auth for proxy, tuple of (username, password). - :param auth: HTTP basic auth tuple of (username, password). Only basic auth is supported. - :param verify: Whether to verify HTTPS certificates. - :param cert: Tuple of (cert, key) filenames for the client certificate. - :param impersonate: Browser version to impersonate. Automatically defaults to the latest available Chrome version. - :param http3: Whether to use HTTP3. Defaults to False. It might be problematic if used it with `impersonate`. - :param stealthy_headers: If enabled (default), it creates and adds real browser headers. It also sets the referer header as if this request came from a Google search of URL's domain. - :param kwargs: Additional keyword arguments to pass to the [`curl_cffi.requests.Session().request()`, `curl_cffi.requests.AsyncSession().request()`] method. + :param kwargs: Additional keyword arguments including: + - data: Form data to include in the request body. + - json: A JSON serializable object to include in the body of the request. + - params: Query string parameters for the request. + - headers: Headers to include in the request. + - cookies: Cookies to use in the request. + - timeout: Number of seconds to wait before timing out. + - follow_redirects: Whether to follow redirects. Defaults to True. + - max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited. + - retries: Number of retry attempts. Defaults to 3. + - retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second. + - proxies: Dict of proxies to use. + - proxy: Proxy URL to use. Format: "http://username:password@localhost:8030". + - proxy_auth: HTTP basic auth for proxy, tuple of (username, password). + - auth: HTTP basic auth tuple of (username, password). Only basic auth is supported. + - verify: Whether to verify HTTPS certificates. + - cert: Tuple of (cert, key) filenames for the client certificate. + - impersonate: Browser version to impersonate. Automatically defaults to the latest available Chrome version. + - http3: Whether to use HTTP3. Defaults to False. It might be problematic if used it with `impersonate`. + - stealthy_headers: If enabled (default), it creates and adds real browser headers. :return: A `Response` object. """ # Careful of sending a body in a DELETE request, it might cause some websites to reject the request as per https://www.rfc-editor.org/rfc/rfc7231#section-4.3.5, # But some websites accept it, it depends on the implementation used. - method_args = {k: v for k, v in locals().items() if k not in ("self", "stealthy_headers", "kwargs")} - method_args.update(kwargs) - # For type checking (not accessed error) - _ = ( - url, - params, - headers, - data, - json, - cookies, - timeout, - follow_redirects, - max_redirects, - retries, - retry_delay, - proxies, - proxy, - proxy_auth, - auth, - verify, - cert, - impersonate, - http3, - ) - return self.__make_request("DELETE", stealth=stealthy_headers, **method_args) + stealthy_headers = kwargs.pop("stealthy_headers", None) + return self.__make_request("DELETE", stealth=stealthy_headers, url=url, **kwargs) class _ASyncSessionLogic(_ConfigurationLogic): - def __init__( - self, - impersonate: ImpersonateType = "chrome", - http3: Optional[bool] = False, - stealthy_headers: Optional[bool] = True, - proxies: Optional[Dict[str, str]] = None, - proxy: Optional[str] = None, - proxy_auth: Optional[Tuple[str, str]] = None, - timeout: Optional[int | float] = 30, - headers: Optional[Dict[str, str]] = None, - retries: Optional[int] = 3, - retry_delay: Optional[int] = 1, - follow_redirects: bool = True, - max_redirects: int = 30, - verify: bool = True, - cert: Optional[str | Tuple[str, str]] = None, - selector_config: Optional[Dict] = None, - ): - super().__init__( - impersonate, - http3, - stealthy_headers, - proxies, - proxy, - proxy_auth, - timeout, - headers, - retries, - retry_delay, - follow_redirects, - max_redirects, - verify, - cert, - selector_config, - ) + def __init__(self, **kwargs: Unpack[RequestsSession]): + super().__init__(**kwargs) self._async_curl_session: Optional[AsyncCurlSession] = None - async def __aenter__(self): + async def __aenter__(self): # pragma: no cover """Creates and returns a new asynchronous Session.""" if self._async_curl_session: raise RuntimeError("This FetcherSession instance already has an active asynchronous session.") @@ -628,19 +368,16 @@ class _ASyncSessionLogic(_ConfigurationLogic): self._async_curl_session = None async def __make_request( - self, - method: SUPPORTED_HTTP_METHODS, - stealth: Optional[bool] = None, - **kwargs, + self, method: SUPPORTED_HTTP_METHODS, stealth: Optional[bool] = None, **kwargs ) -> Response: """ Perform an HTTP request using the configured session. """ stealth = self._stealth if stealth is None else stealth - selector_config = kwargs.pop("selector_config", {}) or self.selector_config - max_retries = self._get_with_precedence(kwargs.pop("retries"), self._default_retries) - retry_delay = self._get_with_precedence(kwargs.pop("retry_delay"), self._default_retry_delay) + selector_config = self._get_param(kwargs, "selector_config", self.selector_config) or self.selector_config + max_retries = self._get_param(kwargs, "retries", self._default_retries) + retry_delay = self._get_param(kwargs, "retry_delay", self._default_retry_delay) request_args = self._merge_request_args(stealth=stealth, **kwargs) session = self._async_curl_session @@ -672,309 +409,133 @@ class _ASyncSessionLogic(_ConfigurationLogic): raise RuntimeError("No active session available.") # pragma: no cover - def get( - self, - url: str, - params: Optional[Dict | List | Tuple] = None, - headers: Optional[Mapping[str, Optional[str]]] = _UNSET, - cookies: Optional[CookieTypes] = None, - timeout: Optional[int | float] = _UNSET, - follow_redirects: Optional[bool] = _UNSET, - max_redirects: Optional[int] = _UNSET, - retries: Optional[int] = _UNSET, - retry_delay: Optional[int] = _UNSET, - proxies: Optional[ProxySpec] = _UNSET, - proxy: Optional[str] = _UNSET, - proxy_auth: Optional[Tuple[str, str]] = _UNSET, - auth: Optional[Tuple[str, str]] = None, - verify: Optional[bool] = _UNSET, - cert: Optional[str | Tuple[str, str]] = _UNSET, - impersonate: ImpersonateType = _UNSET, - http3: Optional[bool] = _UNSET, - stealthy_headers: Optional[bool] = _UNSET, - **kwargs, - ) -> Awaitable[Response]: + def get(self, url: str, **kwargs: Unpack[GetRequestParams]) -> Awaitable[Response]: """ Perform a GET request. + Any additional keyword arguments are passed to the [`curl_cffi.requests.Session().request()`, `curl_cffi.requests.AsyncSession().request()`] method. + :param url: Target URL for the request. - :param params: Query string parameters for the request. - :param headers: Headers to include in the request. - :param cookies: Cookies to use in the request. - :param timeout: Number of seconds to wait before timing out. - :param follow_redirects: Whether to follow redirects. Defaults to True. - :param max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited. - :param retries: Number of retry attempts. Defaults to 3. - :param retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second. - :param proxies: Dict of proxies to use. - :param proxy: Proxy URL to use. Format: "http://username:password@localhost:8030". - Cannot be used together with the `proxies` parameter. - :param proxy_auth: HTTP basic auth for proxy, tuple of (username, password). - :param auth: HTTP basic auth tuple of (username, password). Only basic auth is supported. - :param verify: Whether to verify HTTPS certificates. - :param cert: Tuple of (cert, key) filenames for the client certificate. - :param impersonate: Browser version to impersonate. Automatically defaults to the latest available Chrome version. - :param http3: Whether to use HTTP3. Defaults to False. It might be problematic if used it with `impersonate`. - :param stealthy_headers: If enabled (default), it creates and adds real browser headers. It also sets the referer header as if this request came from a Google search of URL's domain. - :param kwargs: Additional keyword arguments to pass to the [`curl_cffi.requests.Session().request()`, `curl_cffi.requests.AsyncSession().request()`] method. + :param kwargs: Additional keyword arguments including: + - params: Query string parameters for the request. + - headers: Headers to include in the request. + - cookies: Cookies to use in the request. + - timeout: Number of seconds to wait before timing out. + - follow_redirects: Whether to follow redirects. Defaults to True. + - max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited. + - retries: Number of retry attempts. Defaults to 3. + - retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second. + - proxies: Dict of proxies to use. + - proxy: Proxy URL to use. Format: "http://username:password@localhost:8030". + - proxy_auth: HTTP basic auth for proxy, tuple of (username, password). + - auth: HTTP basic auth tuple of (username, password). Only basic auth is supported. + - verify: Whether to verify HTTPS certificates. + - cert: Tuple of (cert, key) filenames for the client certificate. + - impersonate: Browser version to impersonate. Automatically defaults to the latest available Chrome version. + - http3: Whether to use HTTP3. Defaults to False. It might be problematic if used it with `impersonate`. + - stealthy_headers: If enabled (default), it creates and adds real browser headers. :return: A `Response` object. """ - method_args = {k: v for k, v in locals().items() if k not in ("self", "stealthy_headers", "kwargs")} - method_args.update(kwargs) - # For type checking (not accessed error) - _ = ( - url, - params, - headers, - cookies, - timeout, - follow_redirects, - max_redirects, - retries, - retry_delay, - proxies, - proxy, - proxy_auth, - auth, - verify, - cert, - impersonate, - http3, - ) - return self.__make_request("GET", stealth=stealthy_headers, **method_args) + stealthy_headers = kwargs.pop("stealthy_headers", None) + return self.__make_request("GET", stealth=stealthy_headers, url=url, **kwargs) - def post( - self, - url: str, - data: Optional[Dict | str] = None, - json: Optional[Dict | List] = None, - headers: Optional[Mapping[str, Optional[str]]] = _UNSET, - params: Optional[Dict | List | Tuple] = None, - cookies: Optional[CookieTypes] = None, - timeout: Optional[int | float] = _UNSET, - follow_redirects: Optional[bool] = _UNSET, - max_redirects: Optional[int] = _UNSET, - retries: Optional[int] = _UNSET, - retry_delay: Optional[int] = _UNSET, - proxies: Optional[ProxySpec] = _UNSET, - proxy: Optional[str] = _UNSET, - proxy_auth: Optional[Tuple[str, str]] = _UNSET, - auth: Optional[Tuple[str, str]] = None, - verify: Optional[bool] = _UNSET, - cert: Optional[str | Tuple[str, str]] = _UNSET, - impersonate: ImpersonateType = _UNSET, - http3: Optional[bool] = _UNSET, - stealthy_headers: Optional[bool] = _UNSET, - **kwargs, - ) -> Awaitable[Response]: + def post(self, url: str, **kwargs: Unpack[DataRequestParams]) -> Awaitable[Response]: """ Perform a POST request. + Any additional keyword arguments are passed to the [`curl_cffi.requests.Session().request()`, `curl_cffi.requests.AsyncSession().request()`] method. + :param url: Target URL for the request. - :param data: Form data to include in the request body. - :param json: A JSON serializable object to include in the body of the request. - :param params: Query string parameters for the request. - :param headers: Headers to include in the request. - :param cookies: Cookies to use in the request. - :param timeout: Number of seconds to wait before timing out. - :param follow_redirects: Whether to follow redirects. Defaults to True. - :param max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited. - :param retries: Number of retry attempts. Defaults to 3. - :param retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second. - :param proxies: Dict of proxies to use. - :param proxy: Proxy URL to use. Format: "http://username:password@localhost:8030". - Cannot be used together with the `proxies` parameter. - :param proxy_auth: HTTP basic auth for proxy, tuple of (username, password). - :param auth: HTTP basic auth tuple of (username, password). Only basic auth is supported. - :param verify: Whether to verify HTTPS certificates. - :param cert: Tuple of (cert, key) filenames for the client certificate. - :param impersonate: Browser version to impersonate. Automatically defaults to the latest available Chrome version. - :param http3: Whether to use HTTP3. Defaults to False. It might be problematic if used it with `impersonate`. - :param stealthy_headers: If enabled (default), it creates and adds real browser headers. It also sets the referer header as if this request came from a Google search of URL's domain. - :param kwargs: Additional keyword arguments to pass to the [`curl_cffi.requests.Session().request()`, `curl_cffi.requests.AsyncSession().request()`] method. + :param kwargs: Additional keyword arguments including: + - data: Form data to include in the request body. + - json: A JSON serializable object to include in the body of the request. + - params: Query string parameters for the request. + - headers: Headers to include in the request. + - cookies: Cookies to use in the request. + - timeout: Number of seconds to wait before timing out. + - follow_redirects: Whether to follow redirects. Defaults to True. + - max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited. + - retries: Number of retry attempts. Defaults to 3. + - retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second. + - proxies: Dict of proxies to use. + - proxy: Proxy URL to use. Format: "http://username:password@localhost:8030". + - proxy_auth: HTTP basic auth for proxy, tuple of (username, password). + - auth: HTTP basic auth tuple of (username, password). Only basic auth is supported. + - verify: Whether to verify HTTPS certificates. + - cert: Tuple of (cert, key) filenames for the client certificate. + - impersonate: Browser version to impersonate. Automatically defaults to the latest available Chrome version. + - http3: Whether to use HTTP3. Defaults to False. It might be problematic if used it with `impersonate`. + - stealthy_headers: If enabled (default), it creates and adds real browser headers. :return: A `Response` object. """ - method_args = {k: v for k, v in locals().items() if k not in ("self", "stealthy_headers", "kwargs")} - method_args.update(kwargs) - # For type checking (not accessed error) - _ = ( - url, - params, - headers, - data, - json, - cookies, - timeout, - follow_redirects, - max_redirects, - retries, - retry_delay, - proxies, - proxy, - proxy_auth, - auth, - verify, - cert, - impersonate, - http3, - ) - return self.__make_request("POST", stealth=stealthy_headers, **method_args) + stealthy_headers = kwargs.pop("stealthy_headers", None) + return self.__make_request("POST", stealth=stealthy_headers, url=url, **kwargs) - def put( - self, - url: str, - data: Optional[Dict | str] = None, - json: Optional[Dict | List] = None, - headers: Optional[Mapping[str, Optional[str]]] = _UNSET, - params: Optional[Dict | List | Tuple] = None, - cookies: Optional[CookieTypes] = None, - timeout: Optional[int | float] = _UNSET, - follow_redirects: Optional[bool] = _UNSET, - max_redirects: Optional[int] = _UNSET, - retries: Optional[int] = _UNSET, - retry_delay: Optional[int] = _UNSET, - proxies: Optional[ProxySpec] = _UNSET, - proxy: Optional[str] = _UNSET, - proxy_auth: Optional[Tuple[str, str]] = _UNSET, - auth: Optional[Tuple[str, str]] = None, - verify: Optional[bool] = _UNSET, - cert: Optional[str | Tuple[str, str]] = _UNSET, - impersonate: ImpersonateType = _UNSET, - http3: Optional[bool] = _UNSET, - stealthy_headers: Optional[bool] = _UNSET, - **kwargs, - ) -> Awaitable[Response]: + def put(self, url: str, **kwargs: Unpack[DataRequestParams]) -> Awaitable[Response]: """ Perform a PUT request. + Any additional keyword arguments are passed to the [`curl_cffi.requests.Session().request()`, `curl_cffi.requests.AsyncSession().request()`] method. + :param url: Target URL for the request. - :param data: Form data to include in the request body. - :param json: A JSON serializable object to include in the body of the request. - :param params: Query string parameters for the request. - :param headers: Headers to include in the request. - :param cookies: Cookies to use in the request. - :param timeout: Number of seconds to wait before timing out. - :param follow_redirects: Whether to follow redirects. Defaults to True. - :param max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited. - :param retries: Number of retry attempts. Defaults to 3. - :param retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second. - :param proxies: Dict of proxies to use. - :param proxy: Proxy URL to use. Format: "http://username:password@localhost:8030". - Cannot be used together with the `proxies` parameter. - :param proxy_auth: HTTP basic auth for proxy, tuple of (username, password). - :param auth: HTTP basic auth tuple of (username, password). Only basic auth is supported. - :param verify: Whether to verify HTTPS certificates. - :param cert: Tuple of (cert, key) filenames for the client certificate. - :param impersonate: Browser version to impersonate. Automatically defaults to the latest available Chrome version. - :param http3: Whether to use HTTP3. Defaults to False. It might be problematic if used it with `impersonate`. - :param stealthy_headers: If enabled (default), it creates and adds real browser headers. It also sets the referer header as if this request came from a Google search of URL's domain. - :param kwargs: Additional keyword arguments to pass to the [`curl_cffi.requests.Session().request()`, `curl_cffi.requests.AsyncSession().request()`] method. + :param kwargs: Additional keyword arguments including: + - data: Form data to include in the request body. + - json: A JSON serializable object to include in the body of the request. + - params: Query string parameters for the request. + - headers: Headers to include in the request. + - cookies: Cookies to use in the request. + - timeout: Number of seconds to wait before timing out. + - follow_redirects: Whether to follow redirects. Defaults to True. + - max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited. + - retries: Number of retry attempts. Defaults to 3. + - retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second. + - proxies: Dict of proxies to use. + - proxy: Proxy URL to use. Format: "http://username:password@localhost:8030". + - proxy_auth: HTTP basic auth for proxy, tuple of (username, password). + - auth: HTTP basic auth tuple of (username, password). Only basic auth is supported. + - verify: Whether to verify HTTPS certificates. + - cert: Tuple of (cert, key) filenames for the client certificate. + - impersonate: Browser version to impersonate. Automatically defaults to the latest available Chrome version. + - http3: Whether to use HTTP3. Defaults to False. It might be problematic if used it with `impersonate`. + - stealthy_headers: If enabled (default), it creates and adds real browser headers. :return: A `Response` object. """ - method_args = {k: v for k, v in locals().items() if k not in ("self", "stealthy_headers", "kwargs")} - method_args.update(kwargs) - # For type checking (not accessed error) - _ = ( - url, - params, - headers, - data, - json, - cookies, - timeout, - follow_redirects, - max_redirects, - retries, - retry_delay, - proxies, - proxy, - proxy_auth, - auth, - verify, - cert, - impersonate, - http3, - ) - return self.__make_request("PUT", stealth=stealthy_headers, **method_args) + stealthy_headers = kwargs.pop("stealthy_headers", None) + return self.__make_request("PUT", stealth=stealthy_headers, url=url, **kwargs) - def delete( - self, - url: str, - data: Optional[Dict | str] = None, - json: Optional[Dict | List] = None, - headers: Optional[Mapping[str, Optional[str]]] = _UNSET, - params: Optional[Dict | List | Tuple] = None, - cookies: Optional[CookieTypes] = None, - timeout: Optional[int | float] = _UNSET, - follow_redirects: Optional[bool] = _UNSET, - max_redirects: Optional[int] = _UNSET, - retries: Optional[int] = _UNSET, - retry_delay: Optional[int] = _UNSET, - proxies: Optional[ProxySpec] = _UNSET, - proxy: Optional[str] = _UNSET, - proxy_auth: Optional[Tuple[str, str]] = _UNSET, - auth: Optional[Tuple[str, str]] = None, - verify: Optional[bool] = _UNSET, - cert: Optional[str | Tuple[str, str]] = _UNSET, - impersonate: ImpersonateType = _UNSET, - http3: Optional[bool] = _UNSET, - stealthy_headers: Optional[bool] = _UNSET, - **kwargs, - ) -> Awaitable[Response]: + def delete(self, url: str, **kwargs: Unpack[DataRequestParams]) -> Awaitable[Response]: """ Perform a DELETE request. + Any additional keyword arguments are passed to the [`curl_cffi.requests.Session().request()`, `curl_cffi.requests.AsyncSession().request()`] method. + :param url: Target URL for the request. - :param data: Form data to include in the request body. - :param json: A JSON serializable object to include in the body of the request. - :param params: Query string parameters for the request. - :param headers: Headers to include in the request. - :param cookies: Cookies to use in the request. - :param timeout: Number of seconds to wait before timing out. - :param follow_redirects: Whether to follow redirects. Defaults to True. - :param max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited. - :param retries: Number of retry attempts. Defaults to 3. - :param retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second. - :param proxies: Dict of proxies to use. - :param proxy: Proxy URL to use. Format: "http://username:password@localhost:8030". - Cannot be used together with the `proxies` parameter. - :param proxy_auth: HTTP basic auth for proxy, tuple of (username, password). - :param auth: HTTP basic auth tuple of (username, password). Only basic auth is supported. - :param verify: Whether to verify HTTPS certificates. - :param cert: Tuple of (cert, key) filenames for the client certificate. - :param impersonate: Browser version to impersonate. Automatically defaults to the latest available Chrome version. - :param http3: Whether to use HTTP3. Defaults to False. It might be problematic if used it with `impersonate`. - :param stealthy_headers: If enabled (default), it creates and adds real browser headers. It also sets the referer header as if this request came from a Google search of URL's domain. - :param kwargs: Additional keyword arguments to pass to the [`curl_cffi.requests.Session().request()`, `curl_cffi.requests.AsyncSession().request()`] method. + :param kwargs: Additional keyword arguments including: + - data: Form data to include in the request body. + - json: A JSON serializable object to include in the body of the request. + - params: Query string parameters for the request. + - headers: Headers to include in the request. + - cookies: Cookies to use in the request. + - timeout: Number of seconds to wait before timing out. + - follow_redirects: Whether to follow redirects. Defaults to True. + - max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited. + - retries: Number of retry attempts. Defaults to 3. + - retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second. + - proxies: Dict of proxies to use. + - proxy: Proxy URL to use. Format: "http://username:password@localhost:8030". + - proxy_auth: HTTP basic auth for proxy, tuple of (username, password). + - auth: HTTP basic auth tuple of (username, password). Only basic auth is supported. + - verify: Whether to verify HTTPS certificates. + - cert: Tuple of (cert, key) filenames for the client certificate. + - impersonate: Browser version to impersonate. Automatically defaults to the latest available Chrome version. + - http3: Whether to use HTTP3. Defaults to False. It might be problematic if used it with `impersonate`. + - stealthy_headers: If enabled (default), it creates and adds real browser headers. :return: A `Response` object. """ # Careful of sending a body in a DELETE request, it might cause some websites to reject the request as per https://www.rfc-editor.org/rfc/rfc7231#section-4.3.5, # But some websites accept it, it depends on the implementation used. - method_args = {k: v for k, v in locals().items() if k not in ("self", "stealthy_headers", "kwargs")} - method_args.update(kwargs) - # For type checking (not accessed error) - _ = ( - url, - params, - headers, - data, - json, - cookies, - timeout, - follow_redirects, - max_redirects, - retries, - retry_delay, - proxies, - proxy, - proxy_auth, - auth, - verify, - cert, - impersonate, - http3, - ) - return self.__make_request("DELETE", stealth=stealthy_headers, **method_args) + stealthy_headers = kwargs.pop("stealthy_headers", None) + return self.__make_request("DELETE", stealth=stealthy_headers, url=url, **kwargs) class FetcherSession: From c487faf4fdd32fb7576d22286a9654c7040ffa09 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Sun, 23 Nov 2025 21:50:54 +0200 Subject: [PATCH 04/21] pref(requests): Optimizing memory by specifiying slots --- scrapling/engines/_browsers/_validators.py | 10 ++--- scrapling/engines/static.py | 45 ++++++++++++++++++++++ 2 files changed, 50 insertions(+), 5 deletions(-) diff --git a/scrapling/engines/_browsers/_validators.py b/scrapling/engines/_browsers/_validators.py index edeaa2f..2b5ba0a 100644 --- a/scrapling/engines/_browsers/_validators.py +++ b/scrapling/engines/_browsers/_validators.py @@ -21,7 +21,7 @@ from scrapling.engines.toolbelt.navigation import construct_proxy_dict # Custom validators for msgspec @lru_cache(8) -def _is_invalid_file_path(value: str) -> bool | str: +def _is_invalid_file_path(value: str) -> bool | str: # pragma: no cover """Fast file path validation""" path = Path(value) if not path.exists(): @@ -33,7 +33,7 @@ def _is_invalid_file_path(value: str) -> bool | str: return False -def _validate_addon_path(value: str) -> None: +def _validate_addon_path(value: str) -> None: # pragma: no cover """Fast addon path validation""" path = Path(value) if not path.exists(): @@ -49,7 +49,7 @@ def _is_invalid_cdp_url(cdp_url: str) -> bool | str: return "CDP URL must use 'ws://' or 'wss://' scheme" netloc = urlparse(cdp_url).netloc - if not netloc: + if not netloc: # pragma: no cover return "Invalid hostname for the CDP URL" return False @@ -89,7 +89,7 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False, weakref=True): selector_config: Optional[Dict] = {} additional_args: Optional[Dict] = {} - def __post_init__(self): + def __post_init__(self): # pragma: no cover """Custom validation after msgspec validation""" if self.page_action and not callable(self.page_action): raise TypeError(f"page_action must be callable, got {type(self.page_action).__name__}") @@ -195,7 +195,7 @@ class _fetch_params: def validate_fetch( params: List[Tuple], model: type[PlaywrightConfig] | type[CamoufoxConfig], sentinel=None -) -> _fetch_params: +) -> _fetch_params: # pragma: no cover result = {} overrides = {} diff --git a/scrapling/engines/static.py b/scrapling/engines/static.py index 7251209..796df13 100644 --- a/scrapling/engines/static.py +++ b/scrapling/engines/static.py @@ -46,6 +46,24 @@ def _select_random_browser(impersonate: ImpersonateType) -> Optional[BrowserType class _ConfigurationLogic(ABC): # Core Logic Handler (Internal Engine) + __slots__ = ( + "_default_impersonate", + "_stealth", + "_default_proxies", + "_default_proxy", + "_default_proxy_auth", + "_default_timeout", + "_default_headers", + "_default_retries", + "_default_retry_delay", + "_default_follow_redirects", + "_default_max_redirects", + "_default_verify", + "_default_cert", + "_default_http3", + "selector_config", + ) + def __init__(self, **kwargs: Unpack[RequestsSession]): self._default_impersonate = kwargs.get("impersonate", "chrome") self._stealth = kwargs.get("stealthy_headers", True) @@ -157,6 +175,8 @@ class _ConfigurationLogic(ABC): class _SyncSessionLogic(_ConfigurationLogic): + __slots__ = ("_curl_session",) + def __init__(self, **kwargs: Unpack[RequestsSession]): super().__init__(**kwargs) self._curl_session: Optional[CurlSession] = None @@ -343,6 +363,8 @@ class _SyncSessionLogic(_ConfigurationLogic): class _ASyncSessionLogic(_ConfigurationLogic): + __slots__ = ("_async_curl_session",) + def __init__(self, **kwargs: Unpack[RequestsSession]): super().__init__(**kwargs) self._async_curl_session: Optional[AsyncCurlSession] = None @@ -549,6 +571,25 @@ class FetcherSession: same manager instance while a session is already active is disallowed. """ + __slots__ = ( + "_default_impersonate", + "_stealth", + "_default_proxies", + "_default_proxy", + "_default_proxy_auth", + "_default_timeout", + "_default_headers", + "_default_retries", + "_default_retry_delay", + "_default_follow_redirects", + "_default_max_redirects", + "_default_verify", + "_default_cert", + "_default_http3", + "selector_config", + "_client", + ) + def __init__( self, impersonate: ImpersonateType = "chrome", @@ -640,6 +681,8 @@ class FetcherSession: class FetcherClient(_SyncSessionLogic): + __slots__ = ("__enter__", "__exit__") + def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.__enter__: Any = None @@ -648,6 +691,8 @@ class FetcherClient(_SyncSessionLogic): class AsyncFetcherClient(_ASyncSessionLogic): + __slots__ = ("__aenter__", "__aexit__") + def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.__aenter__: Any = None From b5cb235c53096b3be3c38e05a967e9ffe18624e4 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Mon, 24 Nov 2025 03:30:14 +0200 Subject: [PATCH 05/21] ops: fix tests workflow caching --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index f1a7343..62faf0c 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -116,7 +116,7 @@ jobs: with: path: .tox # Include python version and os in the cache key - key: tox-v1-${{ runner.os }}-py${{ matrix.python-version }}-${{ hashFiles('tox.ini', 'pyproject.toml') }} + key: tox-v1-${{ runner.os }}-py${{ matrix.python-version }}-${{ hashFiles('pyproject.toml') }} restore-keys: | tox-v1-${{ runner.os }}-py${{ matrix.python-version }}- tox-v1-${{ runner.os }}- From abaf877eff03d5519a1e24654c3f5b64db83e77b Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Mon, 24 Nov 2025 03:55:13 +0200 Subject: [PATCH 06/21] ops: fix tests workflow caching --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 62faf0c..4e034ee 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -116,7 +116,7 @@ jobs: with: path: .tox # Include python version and os in the cache key - key: tox-v1-${{ runner.os }}-py${{ matrix.python-version }}-${{ hashFiles('pyproject.toml') }} + key: tox-v1-${{ runner.os }}-py${{ matrix.python-version }}-${{ hashFiles(/Users/runner/work/Scrapling/pyproject.toml') }} restore-keys: | tox-v1-${{ runner.os }}-py${{ matrix.python-version }}- tox-v1-${{ runner.os }}- From 088d1912d6b5b37a9bae5d5d82da53494cecf912 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Mon, 24 Nov 2025 12:52:28 +0200 Subject: [PATCH 07/21] ops: fix tests workflow caching --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 4e034ee..368c9a8 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -116,7 +116,7 @@ jobs: with: path: .tox # Include python version and os in the cache key - key: tox-v1-${{ runner.os }}-py${{ matrix.python-version }}-${{ hashFiles(/Users/runner/work/Scrapling/pyproject.toml') }} + key: tox-v1-${{ runner.os }}-py${{ matrix.python-version }}-${{ hashFiles('/Users/runner/work/Scrapling/pyproject.toml') }} restore-keys: | tox-v1-${{ runner.os }}-py${{ matrix.python-version }}- tox-v1-${{ runner.os }}- From abc9aaebfe628d6dd81a521b9a3f9cd962f318b6 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Mon, 24 Nov 2025 13:03:18 +0200 Subject: [PATCH 08/21] build: Pump deps up msgspec now supports py14 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 0855739..cd2145b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -71,7 +71,7 @@ fetchers = [ "patchright>=1.56.0", "camoufox>=0.4.11", "geoip2>=5.2.0", - "msgspec>=0.19.0", + "msgspec>=0.20.0", ] ai = [ "mcp>=1.19.0", From 74f24d60f6a2c3fd1835d4c6934312c99a324395 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Mon, 24 Nov 2025 13:04:12 +0200 Subject: [PATCH 09/21] fix(fetcher): Replace vars logic with one that doesn't require dict attribute --- scrapling/engines/static.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapling/engines/static.py b/scrapling/engines/static.py index 796df13..a442002 100644 --- a/scrapling/engines/static.py +++ b/scrapling/engines/static.py @@ -647,7 +647,7 @@ class FetcherSession: """Creates and returns a new synchronous Fetcher Session""" if self._client is None: # Use **vars(self) to avoid repeating all parameters - config = {k.replace("_default_", ""): v for k, v in vars(self).items() if k.startswith("_default")} + config = {k.replace("_default_", ""): getattr(self, k) for k in self.__slots__ if k.startswith("_default")} config["stealthy_headers"] = self._stealth config["selector_config"] = self.selector_config self._client = _SyncSessionLogic(**config) @@ -665,7 +665,7 @@ class FetcherSession: """Creates and returns a new asynchronous Session.""" if self._client is None: # Use **vars(self) to avoid repeating all parameters - config = {k.replace("_default_", ""): v for k, v in vars(self).items() if k.startswith("_default")} + config = {k.replace("_default_", ""): getattr(self, k) for k in self.__slots__ if k.startswith("_default")} config["stealthy_headers"] = self._stealth config["selector_config"] = self.selector_config self._client = _ASyncSessionLogic(**config) From 9e85311ff61afeb00c4e9468264b697cfb616ae3 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Tue, 25 Nov 2025 19:48:09 +0200 Subject: [PATCH 10/21] fix(validators): Fix overriding session values with request ones --- scrapling/engines/_browsers/_validators.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scrapling/engines/_browsers/_validators.py b/scrapling/engines/_browsers/_validators.py index 2b5ba0a..5442102 100644 --- a/scrapling/engines/_browsers/_validators.py +++ b/scrapling/engines/_browsers/_validators.py @@ -213,12 +213,12 @@ def validate_fetch( for f in fields(_fetch_params) if hasattr(validated_config, f.name) } - # solve_cloudflare defaults to False for models that don't have it (PlaywrightConfig) validated_dict.setdefault("solve_cloudflare", False) - validated_dict.update(result) - return _fetch_params(**validated_dict) + # Start with session defaults, then overwrite with validated overrides + result.update(validated_dict) + # solve_cloudflare defaults to False for models that don't have it (PlaywrightConfig) result.setdefault("solve_cloudflare", False) return _fetch_params(**result) From f0b01fc25333c8a450e823ce547c2ccddf6937db Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Tue, 25 Nov 2025 20:18:36 +0200 Subject: [PATCH 11/21] refactor(browser fetchers): Make all the type hints dynamic + Faster validation Made the code shorter by an additional ~200 lines and easier to maintain in return for making the arguments autocompletion bad for shells that don't check for dynamic type hints like IPython. --- scrapling/core/_types.py | 1 + scrapling/engines/_browsers/_base.py | 193 +++++------ scrapling/engines/_browsers/_camoufox.py | 355 +++++--------------- scrapling/engines/_browsers/_controllers.py | 340 +++++-------------- scrapling/engines/_browsers/_page.py | 27 +- scrapling/engines/_browsers/_types.py | 65 ++++ scrapling/engines/_browsers/_validators.py | 20 +- 7 files changed, 364 insertions(+), 637 deletions(-) diff --git a/scrapling/core/_types.py b/scrapling/core/_types.py index bd83f49..ac19a91 100644 --- a/scrapling/core/_types.py +++ b/scrapling/core/_types.py @@ -12,6 +12,7 @@ from typing import ( Callable, Dict, Generator, + Generic, Iterable, List, Set, diff --git a/scrapling/engines/_browsers/_base.py b/scrapling/engines/_browsers/_base.py index f670e4c..f329fa4 100644 --- a/scrapling/engines/_browsers/_base.py +++ b/scrapling/engines/_browsers/_base.py @@ -2,15 +2,15 @@ from time import time from asyncio import sleep as asyncio_sleep, Lock from camoufox import DefaultAddons +from playwright.sync_api._generated import Page from playwright.sync_api import ( - Page, Frame, BrowserContext, Playwright, Response as SyncPlaywrightResponse, ) +from playwright.async_api._generated import Page as AsyncPage from playwright.async_api import ( - Page as AsyncPage, Frame as AsyncFrame, Playwright as AsyncPlaywright, Response as AsyncPlaywrightResponse, @@ -70,7 +70,7 @@ class SyncSession: timeout: int | float, extra_headers: Optional[Dict[str, str]], disable_resources: bool, - ) -> PageInfo: # pragma: no cover + ) -> PageInfo[Page]: # pragma: no cover """Get a new page to use""" # No need to check if a page is available or not in sync code because the code blocked before reaching here till the page closed, ofc. @@ -116,7 +116,7 @@ class SyncSession: self._wait_for_networkidle(page) @staticmethod - def _create_response_handler(page_info: PageInfo, response_container: List) -> Callable: + def _create_response_handler(page_info: PageInfo[Page], response_container: List) -> Callable: """Create a response handler that captures the final navigation response. :param page_info: The PageInfo object containing the page @@ -175,7 +175,7 @@ class AsyncSession: timeout: int | float, extra_headers: Optional[Dict[str, str]], disable_resources: bool, - ) -> PageInfo: # pragma: no cover + ) -> PageInfo[AsyncPage]: # pragma: no cover """Get a new page to use""" if TYPE_CHECKING: assert self.context is not None, "Browser context not initialized" @@ -232,7 +232,7 @@ class AsyncSession: await self._wait_for_networkidle(page) @staticmethod - def _create_response_handler(page_info: PageInfo, response_container: List) -> Callable: + def _create_response_handler(page_info: PageInfo[AsyncPage], response_container: List) -> Callable: """Create an async response handler that captures the final navigation response. :param page_info: The PageInfo object containing the page @@ -253,130 +253,135 @@ class AsyncSession: class DynamicSessionMixin: def __validate__(self, **params): + if "__max_pages" in params: + params["max_pages"] = params.pop("__max_pages") + config = validate(params, model=PlaywrightConfig) - self.max_pages = config.max_pages - self.headless = config.headless - self.hide_canvas = config.hide_canvas - self.disable_webgl = config.disable_webgl - self.real_chrome = config.real_chrome - self.stealth = config.stealth - self.google_search = config.google_search - self.wait = config.wait - self.proxy = config.proxy - self.locale = config.locale - self.extra_headers = config.extra_headers - self.useragent = config.useragent - self.timeout = config.timeout - self.cookies = config.cookies - self.disable_resources = config.disable_resources - self.cdp_url = config.cdp_url - self.network_idle = config.network_idle - self.load_dom = config.load_dom - self.wait_selector = config.wait_selector - self.init_script = config.init_script - self.wait_selector_state = config.wait_selector_state - self.extra_flags = config.extra_flags - self.selector_config = config.selector_config - self.additional_args = config.additional_args - self.page_action = config.page_action - self.user_data_dir = config.user_data_dir - self._headers_keys = {header.lower() for header in self.extra_headers.keys()} if self.extra_headers else set() + self._max_pages = config.max_pages + self._headless = config.headless + self._hide_canvas = config.hide_canvas + self._disable_webgl = config.disable_webgl + self._real_chrome = config.real_chrome + self._stealth = config.stealth + self._google_search = config.google_search + self._wait = config.wait + self._proxy = config.proxy + self._locale = config.locale + self._extra_headers = config.extra_headers + self._useragent = config.useragent + self._timeout = config.timeout + self._cookies = config.cookies + self._disable_resources = config.disable_resources + self._cdp_url = config.cdp_url + self._network_idle = config.network_idle + self._load_dom = config.load_dom + self._wait_selector = config.wait_selector + self._init_script = config.init_script + self._wait_selector_state = config.wait_selector_state + self._extra_flags = config.extra_flags + self._selector_config = config.selector_config + self._additional_args = config.additional_args + self._page_action = config.page_action + self._user_data_dir = config.user_data_dir + self._headers_keys = {header.lower() for header in self._extra_headers.keys()} if self._extra_headers else set() self.__initiate_browser_options__() def __initiate_browser_options__(self): if TYPE_CHECKING: - assert isinstance(self.proxy, tuple) + assert isinstance(self._proxy, tuple) - if not self.cdp_url: + if not self._cdp_url: # `launch_options` is used with persistent context self.launch_options = dict( _launch_kwargs( - self.headless, - self.proxy, - self.locale, - tuple(self.extra_headers.items()) if self.extra_headers else tuple(), - self.useragent, - self.real_chrome, - self.stealth, - self.hide_canvas, - self.disable_webgl, - tuple(self.extra_flags) if self.extra_flags else tuple(), + self._headless, + self._proxy, + self._locale, + tuple(self._extra_headers.items()) if self._extra_headers else tuple(), + self._useragent, + self._real_chrome, + self._stealth, + self._hide_canvas, + self._disable_webgl, + tuple(self._extra_flags) if self._extra_flags else tuple(), ) ) self.launch_options["extra_http_headers"] = dict(self.launch_options["extra_http_headers"]) self.launch_options["proxy"] = dict(self.launch_options["proxy"]) or None - self.launch_options["user_data_dir"] = self.user_data_dir - self.launch_options.update(cast(Dict, self.additional_args)) + self.launch_options["user_data_dir"] = self._user_data_dir + self.launch_options.update(cast(Dict, self._additional_args)) self.context_options = dict() else: # while `context_options` is left to be used when cdp mode is enabled self.launch_options = dict() self.context_options = dict( _context_kwargs( - self.proxy, - self.locale, - tuple(self.extra_headers.items()) if self.extra_headers else tuple(), - self.useragent, - self.stealth, + self._proxy, + self._locale, + tuple(self._extra_headers.items()) if self._extra_headers else tuple(), + self._useragent, + self._stealth, ) ) self.context_options["extra_http_headers"] = dict(self.context_options["extra_http_headers"]) self.context_options["proxy"] = dict(self.context_options["proxy"]) or None - self.context_options.update(cast(Dict, self.additional_args)) + self.context_options.update(cast(Dict, self._additional_args)) class StealthySessionMixin: def __validate__(self, **params): + if "__max_pages" in params: + params["max_pages"] = params.pop("__max_pages") + config: CamoufoxConfig = validate(params, model=CamoufoxConfig) - self.max_pages = config.max_pages - self.headless = config.headless - self.block_images = config.block_images - self.disable_resources = config.disable_resources - self.block_webrtc = config.block_webrtc - self.allow_webgl = config.allow_webgl - self.network_idle = config.network_idle - self.load_dom = config.load_dom - self.humanize = config.humanize - self.solve_cloudflare = config.solve_cloudflare - self.wait = config.wait - self.timeout = config.timeout - self.page_action = config.page_action - self.wait_selector = config.wait_selector - self.init_script = config.init_script - self.addons = config.addons - self.wait_selector_state = config.wait_selector_state - self.cookies = config.cookies - self.google_search = config.google_search - self.extra_headers = config.extra_headers - self.proxy = config.proxy - self.os_randomize = config.os_randomize - self.disable_ads = config.disable_ads - self.geoip = config.geoip - self.selector_config = config.selector_config - self.additional_args = config.additional_args - self.page_action = config.page_action - self.user_data_dir = config.user_data_dir - self._headers_keys = {header.lower() for header in self.extra_headers.keys()} if self.extra_headers else set() + self._max_pages = config.max_pages + self._headless = config.headless + self._block_images = config.block_images + self._disable_resources = config.disable_resources + self._block_webrtc = config.block_webrtc + self._allow_webgl = config.allow_webgl + self._network_idle = config.network_idle + self._load_dom = config.load_dom + self._humanize = config.humanize + self._solve_cloudflare = config.solve_cloudflare + self._wait = config.wait + self._timeout = config.timeout + self._page_action = config.page_action + self._wait_selector = config.wait_selector + self._init_script = config.init_script + self._addons = config.addons + self._wait_selector_state = config.wait_selector_state + self._cookies = config.cookies + self._google_search = config.google_search + self._extra_headers = config.extra_headers + self._proxy = config.proxy + self._os_randomize = config.os_randomize + self._disable_ads = config.disable_ads + self._geoip = config.geoip + self._selector_config = config.selector_config + self._additional_args = config.additional_args + self._user_data_dir = config.user_data_dir + self._headers_keys = {header.lower() for header in self._extra_headers.keys()} if self._extra_headers else set() self.__initiate_browser_options__() def __initiate_browser_options__(self): """Initiate browser options.""" self.launch_options: Dict[str, Any] = generate_launch_options( **{ - "geoip": self.geoip, - "proxy": dict(self.proxy) if self.proxy and isinstance(self.proxy, tuple) else self.proxy, - "addons": self.addons, - "exclude_addons": [] if self.disable_ads else [DefaultAddons.UBO], - "headless": self.headless, - "humanize": True if self.solve_cloudflare else self.humanize, + "geoip": self._geoip, + "proxy": dict(self._proxy) if self._proxy and isinstance(self._proxy, tuple) else self._proxy, + "addons": self._addons, + "exclude_addons": [] if self._disable_ads else [DefaultAddons.UBO], + "headless": self._headless, + "humanize": True if self._solve_cloudflare else self._humanize, "i_know_what_im_doing": True, # To turn warnings off with the user configurations - "allow_webgl": self.allow_webgl, - "block_webrtc": self.block_webrtc, - "block_images": self.block_images, # Careful! it makes some websites don't finish loading at all like stackoverflow even in headful mode. - "os": None if self.os_randomize else get_os_name(), - "user_data_dir": self.user_data_dir, + "allow_webgl": self._allow_webgl, + "block_webrtc": self._block_webrtc, + "block_images": self._block_images, # Careful! it makes some websites don't finish loading at all like stackoverflow even in headful mode. + "os": None if self._os_randomize else get_os_name(), + "user_data_dir": self._user_data_dir, "ff_version": __ff_version_str__, "firefox_user_prefs": { # This is what enabling `enable_cache` does internally, so we do it from here instead @@ -386,7 +391,7 @@ class StealthySessionMixin: "browser.cache.disk_cache_ssl": True, "browser.cache.disk.smart_size.enabled": True, }, - **cast(Dict, self.additional_args), + **cast(Dict, self._additional_args), } ) diff --git a/scrapling/engines/_browsers/_camoufox.py b/scrapling/engines/_browsers/_camoufox.py index 8d70dbd..436a006 100644 --- a/scrapling/engines/_browsers/_camoufox.py +++ b/scrapling/engines/_browsers/_camoufox.py @@ -14,58 +14,47 @@ from playwright.async_api import ( BrowserContext as AsyncBrowserContext, ) -from ._validators import validate_fetch as _validate, CamoufoxConfig -from ._base import SyncSession, AsyncSession, StealthySessionMixin from scrapling.core.utils import log -from scrapling.core._types import ( - Any, - Dict, - List, - Optional, - Callable, - TYPE_CHECKING, - SelectorWaitStates, -) -from scrapling.engines.toolbelt.convertor import ( - Response, - ResponseFactory, -) +from ._types import CamoufoxSession, CamoufoxFetchParams +from scrapling.core._types import Any, Unpack, TYPE_CHECKING +from ._base import SyncSession, AsyncSession, StealthySessionMixin +from ._validators import validate_fetch as _validate, CamoufoxConfig +from scrapling.engines.toolbelt.convertor import Response, ResponseFactory from scrapling.engines.toolbelt.fingerprints import generate_convincing_referer __CF_PATTERN__ = re_compile("challenges.cloudflare.com/cdn-cgi/challenge-platform/.*") -_UNSET: Any = object() class StealthySession(StealthySessionMixin, SyncSession): """A Stealthy session manager with page pooling.""" __slots__ = ( - "max_pages", - "headless", - "block_images", - "disable_resources", - "block_webrtc", - "allow_webgl", - "network_idle", - "load_dom", - "humanize", - "solve_cloudflare", - "wait", - "timeout", - "page_action", - "wait_selector", - "init_script", - "addons", - "wait_selector_state", - "cookies", - "google_search", - "extra_headers", - "proxy", - "os_randomize", - "disable_ads", - "geoip", - "selector_config", - "additional_args", + "_max_pages", + "_headless", + "_block_images", + "_disable_resources", + "_block_webrtc", + "_allow_webgl", + "_network_idle", + "_load_dom", + "_humanize", + "_solve_cloudflare", + "_wait", + "_timeout", + "_page_action", + "_wait_selector", + "_init_script", + "_addons", + "_wait_selector_state", + "_cookies", + "_google_search", + "_extra_headers", + "_proxy", + "_os_randomize", + "_disable_ads", + "_geoip", + "_selector_config", + "_additional_args", "playwright", "browser", "context", @@ -73,38 +62,10 @@ class StealthySession(StealthySessionMixin, SyncSession): "_closed", "launch_options", "_headers_keys", + "_user_data_dir", ) - def __init__( - self, - __max_pages: int = 1, - headless: bool = True, # noqa: F821 - block_images: bool = False, - disable_resources: bool = False, - block_webrtc: bool = False, - allow_webgl: bool = True, - network_idle: bool = False, - load_dom: bool = True, - humanize: bool | float = True, - solve_cloudflare: bool = False, - wait: int | float = 0, - timeout: int | float = 30000, - page_action: Optional[Callable] = None, - wait_selector: Optional[str] = None, - init_script: Optional[str] = None, - addons: Optional[List[str]] = None, - wait_selector_state: SelectorWaitStates = "attached", - cookies: Optional[List[Dict]] = None, - google_search: bool = True, - extra_headers: Optional[Dict[str, str]] = None, - proxy: Optional[str | Dict[str, str]] = None, - os_randomize: bool = False, - disable_ads: bool = False, - geoip: bool = False, - user_data_dir: str = "", - selector_config: Optional[Dict] = None, - additional_args: Optional[Dict] = None, - ): + def __init__(self, **kwargs: Unpack[CamoufoxSession]): """A Browser session manager with page pooling :param headless: Run the browser in headless/hidden (default), or headful/visible mode. @@ -138,50 +99,21 @@ class StealthySession(StealthySessionMixin, SyncSession): :param selector_config: The arguments that will be passed in the end while creating the final Selector's class. :param additional_args: Additional arguments to be passed to Camoufox as additional settings, and it takes higher priority than Scrapling's settings. """ - - self.__validate__( - wait=wait, - proxy=proxy, - geoip=geoip, - addons=addons, - timeout=timeout, - cookies=cookies, - headless=headless, - humanize=humanize, - load_dom=load_dom, - max_pages=__max_pages, - disable_ads=disable_ads, - allow_webgl=allow_webgl, - page_action=page_action, - init_script=init_script, - network_idle=network_idle, - block_images=block_images, - block_webrtc=block_webrtc, - os_randomize=os_randomize, - user_data_dir=user_data_dir, - wait_selector=wait_selector, - google_search=google_search, - extra_headers=extra_headers, - additional_args=additional_args, - selector_config=selector_config, - solve_cloudflare=solve_cloudflare, - disable_resources=disable_resources, - wait_selector_state=wait_selector_state, - ) - super().__init__(max_pages=self.max_pages) + self.__validate__(**kwargs) + super().__init__(max_pages=self._max_pages) def __create__(self): """Create a browser for this instance and context.""" self.playwright = sync_playwright().start() self.context = self.playwright.firefox.launch_persistent_context(**self.launch_options) - if self.init_script: # pragma: no cover - self.context.add_init_script(path=self.init_script) + if self._init_script: # pragma: no cover + self.context.add_init_script(path=self._init_script) - if self.cookies: # pragma: no cover - self.context.add_cookies(self.cookies) + if self._cookies: # pragma: no cover + self.context.add_cookies(self._cookies) - def _solve_cloudflare(self, page: Page) -> None: # pragma: no cover + def _cloudflare_solver(self, page: Page) -> None: # pragma: no cover """Solve the cloudflare challenge displayed on the playwright page passed :param page: The targeted page @@ -247,59 +179,28 @@ class StealthySession(StealthySessionMixin, SyncSession): log.info("Cloudflare captcha is solved") return - def fetch( - self, - url: str, - google_search: bool = _UNSET, - timeout: int | float = _UNSET, - wait: int | float = _UNSET, - page_action: Optional[Callable] = _UNSET, - extra_headers: Optional[Dict[str, str]] = _UNSET, - disable_resources: bool = _UNSET, - wait_selector: Optional[str] = _UNSET, - wait_selector_state: SelectorWaitStates = _UNSET, - network_idle: bool = _UNSET, - load_dom: bool = _UNSET, - solve_cloudflare: bool = _UNSET, - selector_config: Optional[Dict] = _UNSET, - ) -> Response: + def fetch(self, url: str, **kwargs: Unpack[CamoufoxFetchParams]) -> Response: """Opens up the browser and do your request based on your chosen options. :param url: The Target url. - :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name. - :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000 - :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object. - :param page_action: Added for automation. A function that takes the `page` object and does the automation you need. - :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ - :param disable_resources: Drop requests of unnecessary resources for a speed boost. It depends, but it made requests ~25% faster in my tests for some websites. - Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`. - This can help save your proxy usage but be careful with this option as it makes some websites never finish loading. - :param wait_selector: Wait for a specific CSS selector to be in a specific state. - :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`. - :param network_idle: Wait for the page until there are no network connections for at least 500 ms. - :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute. - :param solve_cloudflare: Solves all types of the Cloudflare's Turnstile/Interstitial challenges before returning the response to you. - :param selector_config: The arguments that will be passed in the end while creating the final Selector's class. + :param kwargs: Additional keyword arguments including: + - google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name. + - timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000 + - wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object. + - page_action: Added for automation. A function that takes the `page` object and does the automation you need. + - extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ + - disable_resources: Drop requests of unnecessary resources for a speed boost. It depends, but it made requests ~25% faster in my tests for some websites. + Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`. + This can help save your proxy usage but be careful with this option as it makes some websites never finish loading. + - wait_selector: Wait for a specific CSS selector to be in a specific state. + - wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`. + - network_idle: Wait for the page until there are no network connections for at least 500 ms. + - load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute. + - solve_cloudflare: Solves all types of the Cloudflare's Turnstile/Interstitial challenges before returning the response to you. + - selector_config: The arguments that will be passed in the end while creating the final Selector's class. :return: A `Response` object. """ - params = _validate( - [ - ("google_search", google_search, self.google_search), - ("timeout", timeout, self.timeout), - ("wait", wait, self.wait), - ("page_action", page_action, self.page_action), - ("extra_headers", extra_headers, self.extra_headers), - ("disable_resources", disable_resources, self.disable_resources), - ("wait_selector", wait_selector, self.wait_selector), - ("wait_selector_state", wait_selector_state, self.wait_selector_state), - ("network_idle", network_idle, self.network_idle), - ("load_dom", load_dom, self.load_dom), - ("solve_cloudflare", solve_cloudflare, self.solve_cloudflare), - ("selector_config", selector_config, self.selector_config), - ], - CamoufoxConfig, - _UNSET, - ) + params = _validate(kwargs, self, CamoufoxConfig) if self._closed: # pragma: no cover raise RuntimeError("Context manager has been closed") @@ -322,7 +223,7 @@ class StealthySession(StealthySessionMixin, SyncSession): raise RuntimeError(f"Failed to get response for {url}") if params.solve_cloudflare: - self._solve_cloudflare(page_info.page) + self._cloudflare_solver(page_info.page) # Make sure the page is fully loaded after the captcha self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle) @@ -360,36 +261,7 @@ class StealthySession(StealthySessionMixin, SyncSession): class AsyncStealthySession(StealthySessionMixin, AsyncSession): """A Stealthy session manager with page pooling.""" - def __init__( - self, - max_pages: int = 1, - headless: bool = True, # noqa: F821 - block_images: bool = False, - disable_resources: bool = False, - block_webrtc: bool = False, - allow_webgl: bool = True, - network_idle: bool = False, - load_dom: bool = True, - humanize: bool | float = True, - solve_cloudflare: bool = False, - wait: int | float = 0, - timeout: int | float = 30000, - page_action: Optional[Callable] = None, - wait_selector: Optional[str] = None, - init_script: Optional[str] = None, - addons: Optional[List[str]] = None, - wait_selector_state: SelectorWaitStates = "attached", - cookies: Optional[List[Dict]] = None, - google_search: bool = True, - extra_headers: Optional[Dict[str, str]] = None, - proxy: Optional[str | Dict[str, str]] = None, - os_randomize: bool = False, - disable_ads: bool = False, - geoip: bool = False, - user_data_dir: str = "", - selector_config: Optional[Dict] = None, - additional_args: Optional[Dict] = None, - ): + def __init__(self, **kwargs: Unpack[CamoufoxSession]): """A Browser session manager with page pooling :param headless: Run the browser in headless/hidden (default), or headful/visible mode. @@ -424,36 +296,8 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession): :param selector_config: The arguments that will be passed in the end while creating the final Selector's class. :param additional_args: Additional arguments to be passed to Camoufox as additional settings, and it takes higher priority than Scrapling's settings. """ - self.__validate__( - wait=wait, - proxy=proxy, - geoip=geoip, - addons=addons, - timeout=timeout, - cookies=cookies, - headless=headless, - load_dom=load_dom, - humanize=humanize, - max_pages=max_pages, - disable_ads=disable_ads, - allow_webgl=allow_webgl, - page_action=page_action, - init_script=init_script, - network_idle=network_idle, - block_images=block_images, - block_webrtc=block_webrtc, - os_randomize=os_randomize, - wait_selector=wait_selector, - google_search=google_search, - extra_headers=extra_headers, - user_data_dir=user_data_dir, - additional_args=additional_args, - selector_config=selector_config, - solve_cloudflare=solve_cloudflare, - disable_resources=disable_resources, - wait_selector_state=wait_selector_state, - ) - super().__init__(max_pages=self.max_pages) + self.__validate__(**kwargs) + super().__init__(max_pages=self._max_pages) async def __create__(self): """Create a browser for this instance and context.""" @@ -462,13 +306,13 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession): **self.launch_options ) - if self.init_script: # pragma: no cover - await self.context.add_init_script(path=self.init_script) + if self._init_script: # pragma: no cover + await self.context.add_init_script(path=self._init_script) - if self.cookies: - await self.context.add_cookies(self.cookies) # pyright: ignore [reportArgumentType] + if self._cookies: + await self.context.add_cookies(self._cookies) # pyright: ignore [reportArgumentType] - async def _solve_cloudflare(self, page: async_Page): # pragma: no cover + async def _cloudflare_solver(self, page: async_Page): # pragma: no cover """Solve the cloudflare challenge displayed on the playwright page passed. The async version :param page: The async targeted page @@ -534,59 +378,28 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession): log.info("Cloudflare captcha is solved") return - async def fetch( - self, - url: str, - google_search: bool = _UNSET, - timeout: int | float = _UNSET, - wait: int | float = _UNSET, - page_action: Optional[Callable] = _UNSET, - extra_headers: Optional[Dict[str, str]] = _UNSET, - disable_resources: bool = _UNSET, - wait_selector: Optional[str] = _UNSET, - wait_selector_state: SelectorWaitStates = _UNSET, - network_idle: bool = _UNSET, - load_dom: bool = _UNSET, - solve_cloudflare: bool = _UNSET, - selector_config: Optional[Dict] = _UNSET, - ) -> Response: + async def fetch(self, url: str, **kwargs: Unpack[CamoufoxFetchParams]) -> Response: """Opens up the browser and do your request based on your chosen options. :param url: The Target url. - :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name. - :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000 - :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object. - :param page_action: Added for automation. A function that takes the `page` object and does the automation you need. - :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ - :param disable_resources: Drop requests of unnecessary resources for a speed boost. It depends, but it made requests ~25% faster in my tests for some websites. - Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`. - This can help save your proxy usage but be careful with this option as it makes some websites never finish loading. - :param wait_selector: Wait for a specific CSS selector to be in a specific state. - :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`. - :param network_idle: Wait for the page until there are no network connections for at least 500 ms. - :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute. - :param solve_cloudflare: Solves all types of the Cloudflare's Turnstile/Interstitial challenges before returning the response to you. - :param selector_config: The arguments that will be passed in the end while creating the final Selector's class. + :param kwargs: Additional keyword arguments including: + - google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name. + - timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000 + - wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object. + - page_action: Added for automation. A function that takes the `page` object and does the automation you need. + - extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ + - disable_resources: Drop requests of unnecessary resources for a speed boost. It depends, but it made requests ~25% faster in my tests for some websites. + Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`. + This can help save your proxy usage but be careful with this option as it makes some websites never finish loading. + - wait_selector: Wait for a specific CSS selector to be in a specific state. + - wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`. + - network_idle: Wait for the page until there are no network connections for at least 500 ms. + - load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute. + - solve_cloudflare: Solves all types of the Cloudflare's Turnstile/Interstitial challenges before returning the response to you. + - selector_config: The arguments that will be passed in the end while creating the final Selector's class. :return: A `Response` object. """ - params = _validate( - [ - ("google_search", google_search, self.google_search), - ("timeout", timeout, self.timeout), - ("wait", wait, self.wait), - ("page_action", page_action, self.page_action), - ("extra_headers", extra_headers, self.extra_headers), - ("disable_resources", disable_resources, self.disable_resources), - ("wait_selector", wait_selector, self.wait_selector), - ("wait_selector_state", wait_selector_state, self.wait_selector_state), - ("network_idle", network_idle, self.network_idle), - ("load_dom", load_dom, self.load_dom), - ("solve_cloudflare", solve_cloudflare, self.solve_cloudflare), - ("selector_config", selector_config, self.selector_config), - ], - CamoufoxConfig, - _UNSET, - ) + params = _validate(kwargs, self, CamoufoxConfig) if self._closed: # pragma: no cover raise RuntimeError("Context manager has been closed") @@ -613,7 +426,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession): raise RuntimeError(f"Failed to get response for {url}") if params.solve_cloudflare: - await self._solve_cloudflare(page_info.page) + await self._cloudflare_solver(page_info.page) # Make sure the page is fully loaded after the captcha await self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle) diff --git a/scrapling/engines/_browsers/_controllers.py b/scrapling/engines/_browsers/_controllers.py index 9fc3490..9ab0e3c 100644 --- a/scrapling/engines/_browsers/_controllers.py +++ b/scrapling/engines/_browsers/_controllers.py @@ -13,92 +13,55 @@ from patchright.sync_api import sync_playwright as sync_patchright from patchright.async_api import async_playwright as async_patchright from scrapling.core.utils import log +from scrapling.core._types import Unpack, TYPE_CHECKING +from ._types import PlaywrightSession, PlaywrightFetchParams from ._base import SyncSession, AsyncSession, DynamicSessionMixin from ._validators import validate_fetch as _validate, PlaywrightConfig -from scrapling.core._types import ( - Any, - Dict, - List, - Optional, - Callable, - TYPE_CHECKING, - SelectorWaitStates, -) -from scrapling.engines.toolbelt.convertor import ( - Response, - ResponseFactory, -) +from scrapling.engines.toolbelt.convertor import Response, ResponseFactory from scrapling.engines.toolbelt.fingerprints import generate_convincing_referer -_UNSET: Any = object() - class DynamicSession(DynamicSessionMixin, SyncSession): """A Browser session manager with page pooling.""" __slots__ = ( - "max_pages", - "headless", - "hide_canvas", - "disable_webgl", - "real_chrome", - "stealth", - "google_search", - "proxy", - "locale", - "extra_headers", - "useragent", - "timeout", - "cookies", - "disable_resources", - "network_idle", - "load_dom", - "wait_selector", - "init_script", - "wait_selector_state", - "wait", + "_max_pages", + "_headless", + "_hide_canvas", + "_disable_webgl", + "_real_chrome", + "_stealth", + "_google_search", + "_proxy", + "_locale", + "_extra_headers", + "_useragent", + "_timeout", + "_cookies", + "_disable_resources", + "_network_idle", + "_load_dom", + "_wait_selector", + "_init_script", + "_wait_selector_state", + "_wait", "playwright", "browser", "context", "page_pool", "_closed", - "selector_config", - "page_action", + "_selector_config", + "_page_action", "launch_options", "context_options", - "cdp_url", + "_cdp_url", "_headers_keys", + "_extra_flags", + "_additional_args", + "_user_data_dir", ) - def __init__( - self, - __max_pages: int = 1, - headless: bool = True, - google_search: bool = True, - hide_canvas: bool = False, - disable_webgl: bool = False, - real_chrome: bool = False, - stealth: bool = False, - wait: int | float = 0, - page_action: Optional[Callable] = None, - proxy: Optional[str | Dict[str, str]] = None, - locale: str = "en-US", - extra_headers: Optional[Dict[str, str]] = None, - useragent: Optional[str] = None, - cdp_url: Optional[str] = None, - timeout: int | float = 30000, - disable_resources: bool = False, - wait_selector: Optional[str] = None, - init_script: Optional[str] = None, - cookies: Optional[List[Dict]] = None, - network_idle: bool = False, - load_dom: bool = True, - wait_selector_state: SelectorWaitStates = "attached", - user_data_dir: str = "", - extra_flags: Optional[List[str]] = None, - selector_config: Optional[Dict] = None, - additional_args: Optional[Dict] = None, - ): + def __init__(self, **kwargs: Unpack[PlaywrightSession]): """A Browser session manager with page pooling, it's using a persistent browser Context by default with a temporary user profile directory. :param headless: Run the browser in headless/hidden (default), or headful/visible mode. @@ -129,105 +92,49 @@ class DynamicSession(DynamicSessionMixin, SyncSession): :param selector_config: The arguments that will be passed in the end while creating the final Selector's class. :param additional_args: Additional arguments to be passed to Playwright's context as additional settings, and it takes higher priority than Scrapling's settings. """ - self.__validate__( - wait=wait, - proxy=proxy, - locale=locale, - timeout=timeout, - stealth=stealth, - cdp_url=cdp_url, - cookies=cookies, - load_dom=load_dom, - headless=headless, - useragent=useragent, - max_pages=__max_pages, - real_chrome=real_chrome, - page_action=page_action, - hide_canvas=hide_canvas, - init_script=init_script, - network_idle=network_idle, - user_data_dir=user_data_dir, - google_search=google_search, - extra_headers=extra_headers, - wait_selector=wait_selector, - disable_webgl=disable_webgl, - extra_flags=extra_flags, - selector_config=selector_config, - additional_args=additional_args, - disable_resources=disable_resources, - wait_selector_state=wait_selector_state, - ) - super().__init__(max_pages=self.max_pages) + self.__validate__(**kwargs) + super().__init__(max_pages=self._max_pages) def __create__(self): """Create a browser for this instance and context.""" - sync_context = sync_patchright if self.stealth else sync_playwright + sync_context = sync_patchright if self._stealth else sync_playwright self.playwright: Playwright = sync_context().start() # pyright: ignore [reportAttributeAccessIssue] - if self.cdp_url: # pragma: no cover - self.context = self.playwright.chromium.connect_over_cdp(endpoint_url=self.cdp_url).new_context( + if self._cdp_url: # pragma: no cover + self.context = self.playwright.chromium.connect_over_cdp(endpoint_url=self._cdp_url).new_context( **self.context_options ) else: self.context = self.playwright.chromium.launch_persistent_context(**self.launch_options) - if self.init_script: # pragma: no cover - self.context.add_init_script(path=self.init_script) + if self._init_script: # pragma: no cover + self.context.add_init_script(path=self._init_script) - if self.cookies: # pragma: no cover - self.context.add_cookies(self.cookies) + if self._cookies: # pragma: no cover + self.context.add_cookies(self._cookies) - def fetch( - self, - url: str, - google_search: bool = _UNSET, - timeout: int | float = _UNSET, - wait: int | float = _UNSET, - page_action: Optional[Callable] = _UNSET, - extra_headers: Optional[Dict[str, str]] = _UNSET, - disable_resources: bool = _UNSET, - wait_selector: Optional[str] = _UNSET, - wait_selector_state: SelectorWaitStates = _UNSET, - network_idle: bool = _UNSET, - load_dom: bool = _UNSET, - selector_config: Optional[Dict] = _UNSET, - ) -> Response: + def fetch(self, url: str, **kwargs: Unpack[PlaywrightFetchParams]) -> Response: """Opens up the browser and do your request based on your chosen options. :param url: The Target url. - :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name. - :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000 - :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object. - :param page_action: Added for automation. A function that takes the `page` object and does the automation you need. - :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ - :param disable_resources: Drop requests of unnecessary resources for a speed boost. It depends, but it made requests ~25% faster in my tests for some websites. - Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`. - This can help save your proxy usage but be careful with this option as it makes some websites never finish loading. - :param wait_selector: Wait for a specific CSS selector to be in a specific state. - :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`. - :param network_idle: Wait for the page until there are no network connections for at least 500 ms. - :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute. - :param selector_config: The arguments that will be passed in the end while creating the final Selector's class. + :param kwargs: Additional keyword arguments including: + - google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name. + - timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000 + - wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object. + - page_action: Added for automation. A function that takes the `page` object and does the automation you need. + - extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ + - disable_resources: Drop requests of unnecessary resources for a speed boost. It depends, but it made requests ~25% faster in my tests for some websites. + Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`. + This can help save your proxy usage but be careful with this option as it makes some websites never finish loading. + - wait_selector: Wait for a specific CSS selector to be in a specific state. + - wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`. + - network_idle: Wait for the page until there are no network connections for at least 500 ms. + - load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute. + - selector_config: The arguments that will be passed in the end while creating the final Selector's class. :return: A `Response` object. """ - params = _validate( - [ - ("google_search", google_search, self.google_search), - ("timeout", timeout, self.timeout), - ("wait", wait, self.wait), - ("page_action", page_action, self.page_action), - ("extra_headers", extra_headers, self.extra_headers), - ("disable_resources", disable_resources, self.disable_resources), - ("wait_selector", wait_selector, self.wait_selector), - ("wait_selector_state", wait_selector_state, self.wait_selector_state), - ("network_idle", network_idle, self.network_idle), - ("load_dom", load_dom, self.load_dom), - ("selector_config", selector_config, self.selector_config), - ], - PlaywrightConfig, - _UNSET, - ) + params = _validate(kwargs, self, PlaywrightConfig) if self._closed: # pragma: no cover raise RuntimeError("Context manager has been closed") @@ -285,35 +192,7 @@ class DynamicSession(DynamicSessionMixin, SyncSession): class AsyncDynamicSession(DynamicSessionMixin, AsyncSession): """An async Browser session manager with page pooling, it's using a persistent browser Context by default with a temporary user profile directory.""" - def __init__( - self, - max_pages: int = 1, - headless: bool = True, - google_search: bool = True, - hide_canvas: bool = False, - disable_webgl: bool = False, - real_chrome: bool = False, - stealth: bool = False, - wait: int | float = 0, - page_action: Optional[Callable] = None, - proxy: Optional[str | Dict[str, str]] = None, - locale: str = "en-US", - extra_headers: Optional[Dict[str, str]] = None, - useragent: Optional[str] = None, - cdp_url: Optional[str] = None, - timeout: int | float = 30000, - disable_resources: bool = False, - wait_selector: Optional[str] = None, - init_script: Optional[str] = None, - cookies: Optional[List[Dict]] = None, - network_idle: bool = False, - load_dom: bool = True, - wait_selector_state: SelectorWaitStates = "attached", - user_data_dir: str = "", - extra_flags: Optional[List[str]] = None, - selector_config: Optional[Dict] = None, - additional_args: Optional[Dict] = None, - ): + def __init__(self, **kwargs: Unpack[PlaywrightSession]): """A Browser session manager with page pooling :param headless: Run the browser in headless/hidden (default), or headful/visible mode. @@ -345,107 +224,50 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession): :param selector_config: The arguments that will be passed in the end while creating the final Selector's class. :param additional_args: Additional arguments to be passed to Playwright's context as additional settings, and it takes higher priority than Scrapling's settings. """ - - self.__validate__( - wait=wait, - proxy=proxy, - locale=locale, - timeout=timeout, - stealth=stealth, - cdp_url=cdp_url, - cookies=cookies, - load_dom=load_dom, - headless=headless, - useragent=useragent, - max_pages=max_pages, - real_chrome=real_chrome, - page_action=page_action, - hide_canvas=hide_canvas, - init_script=init_script, - network_idle=network_idle, - user_data_dir=user_data_dir, - google_search=google_search, - extra_headers=extra_headers, - wait_selector=wait_selector, - disable_webgl=disable_webgl, - extra_flags=extra_flags, - selector_config=selector_config, - additional_args=additional_args, - disable_resources=disable_resources, - wait_selector_state=wait_selector_state, - ) - super().__init__(max_pages=self.max_pages) + self.__validate__(**kwargs) + super().__init__(max_pages=self._max_pages) async def __create__(self): """Create a browser for this instance and context.""" - async_context = async_patchright if self.stealth else async_playwright + async_context = async_patchright if self._stealth else async_playwright self.playwright: AsyncPlaywright = await async_context().start() # pyright: ignore [reportAttributeAccessIssue] - if self.cdp_url: - browser = await self.playwright.chromium.connect_over_cdp(endpoint_url=self.cdp_url) + if self._cdp_url: + browser = await self.playwright.chromium.connect_over_cdp(endpoint_url=self._cdp_url) self.context: AsyncBrowserContext = await browser.new_context(**self.context_options) else: self.context: AsyncBrowserContext = await self.playwright.chromium.launch_persistent_context( **self.launch_options ) - if self.init_script: # pragma: no cover - await self.context.add_init_script(path=self.init_script) + if self._init_script: # pragma: no cover + await self.context.add_init_script(path=self._init_script) - if self.cookies: - await self.context.add_cookies(self.cookies) # pyright: ignore + if self._cookies: + await self.context.add_cookies(self._cookies) # pyright: ignore - async def fetch( - self, - url: str, - google_search: bool = _UNSET, - timeout: int | float = _UNSET, - wait: int | float = _UNSET, - page_action: Optional[Callable] = _UNSET, - extra_headers: Optional[Dict[str, str]] = _UNSET, - disable_resources: bool = _UNSET, - wait_selector: Optional[str] = _UNSET, - wait_selector_state: SelectorWaitStates = _UNSET, - network_idle: bool = _UNSET, - load_dom: bool = _UNSET, - selector_config: Optional[Dict] = _UNSET, - ) -> Response: + async def fetch(self, url: str, **kwargs: Unpack[PlaywrightFetchParams]) -> Response: """Opens up the browser and do your request based on your chosen options. :param url: The Target url. - :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name. - :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000 - :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object. - :param page_action: Added for automation. A function that takes the `page` object and does the automation you need. - :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ - :param disable_resources: Drop requests of unnecessary resources for a speed boost. It depends, but it made requests ~25% faster in my tests for some websites. - Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`. - This can help save your proxy usage but be careful with this option as it makes some websites never finish loading. - :param wait_selector: Wait for a specific CSS selector to be in a specific state. - :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`. - :param network_idle: Wait for the page until there are no network connections for at least 500 ms. - :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute. - :param selector_config: The arguments that will be passed in the end while creating the final Selector's class. + :param kwargs: Additional keyword arguments including: + - google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name. + - timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000 + - wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object. + - page_action: Added for automation. A function that takes the `page` object and does the automation you need. + - extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ + - disable_resources: Drop requests of unnecessary resources for a speed boost. It depends, but it made requests ~25% faster in my tests for some websites. + Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`. + This can help save your proxy usage but be careful with this option as it makes some websites never finish loading. + - wait_selector: Wait for a specific CSS selector to be in a specific state. + - wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`. + - network_idle: Wait for the page until there are no network connections for at least 500 ms. + - load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute. + - selector_config: The arguments that will be passed in the end while creating the final Selector's class. :return: A `Response` object. """ - params = _validate( - [ - ("google_search", google_search, self.google_search), - ("timeout", timeout, self.timeout), - ("wait", wait, self.wait), - ("page_action", page_action, self.page_action), - ("extra_headers", extra_headers, self.extra_headers), - ("disable_resources", disable_resources, self.disable_resources), - ("wait_selector", wait_selector, self.wait_selector), - ("wait_selector_state", wait_selector_state, self.wait_selector_state), - ("network_idle", network_idle, self.network_idle), - ("load_dom", load_dom, self.load_dom), - ("selector_config", selector_config, self.selector_config), - ], - PlaywrightConfig, - _UNSET, - ) + params = _validate(kwargs, self, PlaywrightConfig) if self._closed: # pragma: no cover raise RuntimeError("Context manager has been closed") diff --git a/scrapling/engines/_browsers/_page.py b/scrapling/engines/_browsers/_page.py index 821fbbb..655d3d1 100644 --- a/scrapling/engines/_browsers/_page.py +++ b/scrapling/engines/_browsers/_page.py @@ -1,20 +1,21 @@ from threading import RLock from dataclasses import dataclass -from playwright.sync_api import Page as SyncPage -from playwright.async_api import Page as AsyncPage +from playwright.sync_api._generated import Page as SyncPage +from playwright.async_api._generated import Page as AsyncPage -from scrapling.core._types import Optional, List, Literal +from scrapling.core._types import Optional, List, Literal, overload, TypeVar, Generic, cast PageState = Literal["ready", "busy", "error"] # States that a page can be in +PageType = TypeVar("PageType", SyncPage, AsyncPage) @dataclass -class PageInfo: +class PageInfo(Generic[PageType]): """Information about the page and its current state""" __slots__ = ("page", "state", "url") - page: SyncPage | AsyncPage + page: PageType state: PageState url: Optional[str] @@ -44,16 +45,26 @@ class PagePool: def __init__(self, max_pages: int = 5): self.max_pages = max_pages - self.pages: List[PageInfo] = [] + self.pages: List[PageInfo[SyncPage] | PageInfo[AsyncPage]] = [] self._lock = RLock() - def add_page(self, page: SyncPage | AsyncPage) -> PageInfo: + @overload + def add_page(self, page: SyncPage) -> PageInfo[SyncPage]: ... + + @overload + def add_page(self, page: AsyncPage) -> PageInfo[AsyncPage]: ... + + def add_page(self, page: SyncPage | AsyncPage) -> PageInfo[SyncPage] | PageInfo[AsyncPage]: """Add a new page to the pool""" with self._lock: if len(self.pages) >= self.max_pages: raise RuntimeError(f"Maximum page limit ({self.max_pages}) reached") - page_info = PageInfo(page, "ready", "") + if isinstance(page, AsyncPage): + page_info = cast(PageInfo[AsyncPage], PageInfo(page, "ready", "")) + else: + page_info = cast(PageInfo[SyncPage], PageInfo(page, "ready", "")) + self.pages.append(page_info) return page_info diff --git a/scrapling/engines/_browsers/_types.py b/scrapling/engines/_browsers/_types.py index 0dd0c91..d1edca5 100644 --- a/scrapling/engines/_browsers/_types.py +++ b/scrapling/engines/_browsers/_types.py @@ -10,8 +10,11 @@ from scrapling.core._types import ( Tuple, Mapping, Optional, + Callable, + Iterable, TypedDict, TypeAlias, + SelectorWaitStates, TYPE_CHECKING, ) @@ -49,7 +52,69 @@ if TYPE_CHECKING: # pragma: no cover data: Optional[Dict | str] json: Optional[Dict | List] + # Types for browser session + class BrowserSession(TypedDict, total=False): + max_pages: int + headless: bool + disable_resources: bool + network_idle: bool + load_dom: bool + wait_selector: Optional[str] + wait_selector_state: SelectorWaitStates + cookies: Optional[Iterable[Dict]] + google_search: bool + wait: int | float + page_action: Optional[Callable] + proxy: Optional[str | Dict[str, str] | Tuple] + extra_headers: Optional[Dict[str, str]] + timeout: int | float + init_script: Optional[str] + user_data_dir: str + selector_config: Optional[Dict] + additional_args: Optional[Dict] + + class PlaywrightSession(BrowserSession, total=False): + cdp_url: Optional[str] + hide_canvas: bool + disable_webgl: bool + real_chrome: bool + stealth: bool + locale: str + useragent: Optional[str] + extra_flags: Optional[List[str]] + + class PlaywrightFetchParams(TypedDict, total=False): + google_search: bool + timeout: int | float + wait: int | float + page_action: Optional[Callable] + extra_headers: Optional[Dict[str, str]] + disable_resources: bool + wait_selector: Optional[str] + wait_selector_state: SelectorWaitStates + network_idle: bool + load_dom: bool + selector_config: Optional[Dict] + + class CamoufoxSession(BrowserSession, total=False): + block_images: bool + block_webrtc: bool + allow_webgl: bool + humanize: bool | float + solve_cloudflare: bool + addons: Optional[List[str]] + os_randomize: bool + disable_ads: bool + geoip: bool + + class CamoufoxFetchParams(PlaywrightFetchParams, total=False): + solve_cloudflare: bool + else: # pragma: no cover RequestsSession = TypedDict GetRequestParams = TypedDict DataRequestParams = TypedDict + PlaywrightSession = TypedDict + PlaywrightFetchParams = TypedDict + CamoufoxSession = TypedDict + CamoufoxFetchParams = TypedDict diff --git a/scrapling/engines/_browsers/_validators.py b/scrapling/engines/_browsers/_validators.py index 5442102..a867a37 100644 --- a/scrapling/engines/_browsers/_validators.py +++ b/scrapling/engines/_browsers/_validators.py @@ -7,6 +7,7 @@ from dataclasses import dataclass, fields from msgspec import Struct, Meta, convert, ValidationError from scrapling.core._types import ( + Any, Dict, List, Tuple, @@ -17,6 +18,7 @@ from scrapling.core._types import ( overload, ) from scrapling.engines.toolbelt.navigation import construct_proxy_dict +from scrapling.engines._browsers._types import PlaywrightFetchParams, CamoufoxFetchParams # Custom validators for msgspec @@ -194,16 +196,24 @@ class _fetch_params: def validate_fetch( - params: List[Tuple], model: type[PlaywrightConfig] | type[CamoufoxConfig], sentinel=None + method_kwargs: Dict | PlaywrightFetchParams | CamoufoxFetchParams, + session: Any, + model: type[PlaywrightConfig] | type[CamoufoxConfig], ) -> _fetch_params: # pragma: no cover result = {} overrides = {} - for arg, request_value, session_value in params: - if request_value is not sentinel: - overrides[arg] = request_value + # Get all field names that _fetch_params needs + fetch_param_fields = {f.name for f in fields(_fetch_params)} + + for key in fetch_param_fields: + if key in method_kwargs: + overrides[key] = method_kwargs[key] else: - result[arg] = session_value + # Check for underscore-prefixed attribute (private) + attr_name = f"_{key}" + if hasattr(session, attr_name): + result[key] = getattr(session, attr_name) if overrides: validated_config = validate(overrides, model) From 5376c97996b371dadc3488f4952df8a0dbe20101 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Tue, 25 Nov 2025 20:25:09 +0200 Subject: [PATCH 12/21] fix(fetcher): Remove non-keywords arguments --- scrapling/core/shell.py | 2 +- scrapling/engines/static.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/scrapling/core/shell.py b/scrapling/core/shell.py index 0799496..3e0f1e2 100644 --- a/scrapling/core/shell.py +++ b/scrapling/core/shell.py @@ -421,7 +421,7 @@ Type 'exit' or press Ctrl+D to exit. if isinstance(result, (Response, Selector)): self.pages.append(result) if len(self.pages) > 5: - self.pages.pop(0) # Remove oldest item + self.pages.pop(0) # Remove the oldest item # Update in IPython namespace too if self.shell: diff --git a/scrapling/engines/static.py b/scrapling/engines/static.py index a442002..951659b 100644 --- a/scrapling/engines/static.py +++ b/scrapling/engines/static.py @@ -683,8 +683,8 @@ class FetcherSession: class FetcherClient(_SyncSessionLogic): __slots__ = ("__enter__", "__exit__") - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) + def __init__(self, **kwargs): + super().__init__(**kwargs) self.__enter__: Any = None self.__exit__: Any = None self._curl_session: Any = _NO_SESSION @@ -693,8 +693,8 @@ class FetcherClient(_SyncSessionLogic): class AsyncFetcherClient(_ASyncSessionLogic): __slots__ = ("__aenter__", "__aexit__") - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) + def __init__(self, **kwargs): + super().__init__(**kwargs) self.__aenter__: Any = None self.__aexit__: Any = None self._async_curl_session: Any = _NO_SESSION From 57e11cef76a587d62d70c2fd02f815d7fa0d26d9 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Tue, 25 Nov 2025 20:25:23 +0200 Subject: [PATCH 13/21] tests: Update tests accordingly --- tests/fetchers/sync/test_camoufox_session.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/fetchers/sync/test_camoufox_session.py b/tests/fetchers/sync/test_camoufox_session.py index 4ea6f44..e282d98 100644 --- a/tests/fetchers/sync/test_camoufox_session.py +++ b/tests/fetchers/sync/test_camoufox_session.py @@ -63,12 +63,12 @@ class TestStealthySession: ) as session: assert session.max_pages == 1 - assert session.headless is True - assert session.block_images is True - assert session.disable_resources is True - assert session.solve_cloudflare is True - assert session.wait == 1000 - assert session.timeout == 60000 + assert session._headless is True + assert session._block_images is True + assert session._disable_resources is True + assert session._solve_cloudflare is True + assert session._wait == 1000 + assert session._timeout == 60000 assert session.context is not None # Test Cloudflare detection From 04a612e64e317f712994eadd22cb5953e8aa61d4 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Tue, 25 Nov 2025 21:05:21 +0200 Subject: [PATCH 14/21] docs: fix docstrings for all requests methods --- scrapling/engines/static.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/scrapling/engines/static.py b/scrapling/engines/static.py index 951659b..c7afa79 100644 --- a/scrapling/engines/static.py +++ b/scrapling/engines/static.py @@ -243,7 +243,7 @@ class _SyncSessionLogic(_ConfigurationLogic): """ Perform a GET request. - Any additional keyword arguments are passed to the [`curl_cffi.requests.Session().request()`, `curl_cffi.requests.AsyncSession().request()`] method. + Any additional keyword arguments are passed to the `curl_cffi.requests.Session().request()` method. :param url: Target URL for the request. :param kwargs: Additional keyword arguments including: @@ -273,6 +273,8 @@ class _SyncSessionLogic(_ConfigurationLogic): """ Perform a POST request. + Any additional keyword arguments are passed to the `curl_cffi.requests.Session().request()` method. + :param url: Target URL for the request. :param kwargs: Additional keyword arguments including: - data: Form data to include in the request body. @@ -303,6 +305,8 @@ class _SyncSessionLogic(_ConfigurationLogic): """ Perform a PUT request. + Any additional keyword arguments are passed to the `curl_cffi.requests.Session().request()` method. + :param url: Target URL for the request. :param kwargs: Additional keyword arguments including: - data: Form data to include in the request body. @@ -333,6 +337,8 @@ class _SyncSessionLogic(_ConfigurationLogic): """ Perform a DELETE request. + Any additional keyword arguments are passed to the `curl_cffi.requests.Session().request()` method. + :param url: Target URL for the request. :param kwargs: Additional keyword arguments including: - data: Form data to include in the request body. @@ -435,7 +441,7 @@ class _ASyncSessionLogic(_ConfigurationLogic): """ Perform a GET request. - Any additional keyword arguments are passed to the [`curl_cffi.requests.Session().request()`, `curl_cffi.requests.AsyncSession().request()`] method. + Any additional keyword arguments are passed to the `curl_cffi.requests.AsyncSession().request()` method. :param url: Target URL for the request. :param kwargs: Additional keyword arguments including: @@ -465,7 +471,7 @@ class _ASyncSessionLogic(_ConfigurationLogic): """ Perform a POST request. - Any additional keyword arguments are passed to the [`curl_cffi.requests.Session().request()`, `curl_cffi.requests.AsyncSession().request()`] method. + Any additional keyword arguments are passed to the `curl_cffi.requests.AsyncSession().request()` method. :param url: Target URL for the request. :param kwargs: Additional keyword arguments including: @@ -497,7 +503,7 @@ class _ASyncSessionLogic(_ConfigurationLogic): """ Perform a PUT request. - Any additional keyword arguments are passed to the [`curl_cffi.requests.Session().request()`, `curl_cffi.requests.AsyncSession().request()`] method. + Any additional keyword arguments are passed to the `curl_cffi.requests.AsyncSession().request()` method. :param url: Target URL for the request. :param kwargs: Additional keyword arguments including: @@ -529,7 +535,7 @@ class _ASyncSessionLogic(_ConfigurationLogic): """ Perform a DELETE request. - Any additional keyword arguments are passed to the [`curl_cffi.requests.Session().request()`, `curl_cffi.requests.AsyncSession().request()`] method. + Any additional keyword arguments are passed to the `curl_cffi.requests.AsyncSession().request()` method. :param url: Target URL for the request. :param kwargs: Additional keyword arguments including: From 8940bdeb5575357e9df8ee825e34b8a38c0a40f3 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Tue, 25 Nov 2025 21:23:06 +0200 Subject: [PATCH 15/21] fix(shell): dynamically build the signature of shortcuts after last changes --- scrapling/core/_shell_signatures.py | 95 +++++++++++++++++++++++++++++ scrapling/core/shell.py | 54 ++++++++++++++-- 2 files changed, 143 insertions(+), 6 deletions(-) create mode 100644 scrapling/core/_shell_signatures.py diff --git a/scrapling/core/_shell_signatures.py b/scrapling/core/_shell_signatures.py new file mode 100644 index 0000000..907a9f8 --- /dev/null +++ b/scrapling/core/_shell_signatures.py @@ -0,0 +1,95 @@ +from scrapling.core._types import ( + Dict, + Any, + List, + Tuple, + Optional, +) + +# Parameter definitions for shell function signatures (defined once at module level) +# Mirrors TypedDict definitions from _types.py but runtime-accessible for IPython introspection +_REQUESTS_PARAMS = { + "params": Optional[Dict | List | Tuple], + "cookies": Any, + "auth": Optional[Tuple[str, str]], + "impersonate": Any, + "http3": Optional[bool], + "stealthy_headers": Optional[bool], + "proxies": Any, + "proxy": Optional[str], + "proxy_auth": Optional[Tuple[str, str]], + "timeout": Optional[int | float], + "headers": Any, + "retries": Optional[int], + "retry_delay": Optional[int], + "follow_redirects": Optional[bool], + "max_redirects": Optional[int], + "verify": Optional[bool], + "cert": Optional[str | Tuple[str, str]], + "selector_config": Optional[Dict], +} + +_FETCH_PARAMS = { + "headless": bool, + "google_search": bool, + "hide_canvas": bool, + "disable_webgl": bool, + "real_chrome": bool, + "stealth": bool, + "wait": int | float, + "page_action": Optional[Any], + "proxy": Optional[str | Dict], + "locale": str, + "extra_headers": Optional[Dict[str, str]], + "useragent": Optional[str], + "cdp_url": Optional[str], + "timeout": int | float, + "disable_resources": bool, + "wait_selector": Optional[str], + "init_script": Optional[str], + "cookies": Optional[List[Dict]], + "network_idle": bool, + "load_dom": bool, + "wait_selector_state": Any, + "extra_flags": Optional[List[str]], + "additional_args": Optional[Dict], + "custom_config": Optional[Dict], +} + +_STEALTHY_FETCH_PARAMS = { + "headless": bool, + "block_images": bool, + "disable_resources": bool, + "block_webrtc": bool, + "allow_webgl": bool, + "network_idle": bool, + "load_dom": bool, + "humanize": bool | float, + "solve_cloudflare": bool, + "wait": int | float, + "timeout": int | float, + "page_action": Optional[Any], + "wait_selector": Optional[str], + "init_script": Optional[str], + "addons": Optional[List[str]], + "wait_selector_state": Any, + "cookies": Optional[List[Dict]], + "google_search": bool, + "extra_headers": Optional[Dict[str, str]], + "proxy": Optional[str | Dict], + "os_randomize": bool, + "disable_ads": bool, + "geoip": bool, + "custom_config": Optional[Dict], + "additional_args": Optional[Dict], +} + +# Mapping of function names to their parameter definitions +Signatures_map = { + "get": _REQUESTS_PARAMS, + "post": {**_REQUESTS_PARAMS, "data": Optional[Dict | str], "json": Optional[Dict | List]}, + "put": {**_REQUESTS_PARAMS, "data": Optional[Dict | str], "json": Optional[Dict | List]}, + "delete": _REQUESTS_PARAMS, + "fetch": _FETCH_PARAMS, + "stealthy_fetch": _STEALTHY_FETCH_PARAMS, +} diff --git a/scrapling/core/shell.py b/scrapling/core/shell.py index 3e0f1e2..143d8be 100644 --- a/scrapling/core/shell.py +++ b/scrapling/core/shell.py @@ -1,13 +1,14 @@ # -*- coding: utf-8 -*- -from re import sub as re_sub from sys import stderr from functools import wraps +from re import sub as re_sub from collections import namedtuple from shlex import split as shlex_split +from inspect import signature, Parameter from tempfile import mkstemp as make_temp_file -from urllib.parse import urlparse, urlunparse, parse_qsl from argparse import ArgumentParser, SUPPRESS from webbrowser import open as open_in_browser +from urllib.parse import urlparse, urlunparse, parse_qsl from logging import ( DEBUG, INFO, @@ -21,6 +22,7 @@ from logging import ( from orjson import loads as json_loads, JSONDecodeError +from ._shell_signatures import Signatures_map from scrapling import __version__ from scrapling.core.utils import log from scrapling.parser import Selector, Selectors @@ -28,12 +30,12 @@ from scrapling.core.custom_types import TextHandler from scrapling.engines.toolbelt.custom import Response from scrapling.core.utils._shell import _ParseHeaders, _CookieParser from scrapling.core._types import ( - Optional, Dict, Any, cast, - extraction_types, + Optional, Generator, + extraction_types, ) @@ -312,6 +314,40 @@ class CurlParser: return None +def _unpack_signature(func): + """ + Unpack TypedDict from Unpack[TypedDict] annotations in **kwargs and reconstruct the signature. + + This allows the interactive shell to show individual parameters instead of just **kwargs, similar to how IDEs display them. + """ + try: + sig = signature(func) + func_name = getattr(func, "__name__", None) + + # Check if this function has known parameters + if func_name not in Signatures_map: + return sig + + new_params = [] + for param in sig.parameters.values(): + if param.kind == Parameter.VAR_KEYWORD: + # Replace **kwargs with individual keyword-only parameters + for field_name, field_type in Signatures_map[func_name].items(): + new_params.append( + Parameter(field_name, Parameter.KEYWORD_ONLY, default=Parameter.empty, annotation=field_type) + ) + else: + new_params.append(param) + + # Reconstruct signature with unpacked parameters + if len(new_params) != len(sig.parameters): + return sig.replace(parameters=new_params) + return sig + + except Exception: # pragma: no cover + return signature(func) + + def show_page_in_browser(page: Selector): # pragma: no cover if not page or not isinstance(page, Selector): log.error("Input must be of type `Selector`") @@ -431,7 +467,7 @@ Type 'exit' or press Ctrl+D to exit. return result - def create_wrapper(self, func): + def create_wrapper(self, func, get_signature=True): """Create a wrapper that preserves function signature but updates page""" @wraps(func) @@ -439,6 +475,12 @@ Type 'exit' or press Ctrl+D to exit. result = func(*args, **kwargs) return self.update_page(result) + if get_signature: + # Explicitly preserve and unpack signature for IPython introspection and autocompletion + wrapper.__signature__ = _unpack_signature(func) # pyright: ignore + else: + wrapper.__signature__ = signature(func) # pyright: ignore + return wrapper def get_namespace(self): @@ -451,7 +493,7 @@ Type 'exit' or press Ctrl+D to exit. delete = self.create_wrapper(self.__Fetcher.delete) dynamic_fetch = self.create_wrapper(self.__DynamicFetcher.fetch) stealthy_fetch = self.create_wrapper(self.__StealthyFetcher.fetch) - curl2fetcher = self.create_wrapper(self._curl_parser.convert2fetcher) + curl2fetcher = self.create_wrapper(self._curl_parser.convert2fetcher, get_signature=False) # Create the namespace dictionary return { From 880b144af0b0491fde3a24fa6fbeaf946179ecb2 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Tue, 25 Nov 2025 22:03:45 +0200 Subject: [PATCH 16/21] refactor(browser fetchers): Make all the type hints dynamic + Faster validation + Also renamed `custom_config` to `selector_config` so it matches the session class. --- scrapling/fetchers/chrome.py | 246 ++++++++++---------------------- scrapling/fetchers/firefox.py | 254 ++++++++++------------------------ 2 files changed, 142 insertions(+), 358 deletions(-) diff --git a/scrapling/fetchers/chrome.py b/scrapling/fetchers/chrome.py index 0c2ab84..44e7a44 100644 --- a/scrapling/fetchers/chrome.py +++ b/scrapling/fetchers/chrome.py @@ -1,10 +1,5 @@ -from scrapling.core._types import ( - Callable, - List, - Dict, - Optional, - SelectorWaitStates, -) +from scrapling.core._types import Unpack +from scrapling.engines._browsers._types import PlaywrightSession from scrapling.engines.toolbelt.custom import BaseFetcher, Response from scrapling.engines._browsers._controllers import DynamicSession, AsyncDynamicSession @@ -26,190 +21,89 @@ class DynamicFetcher(BaseFetcher): """ @classmethod - def fetch( - cls, - url: str, - headless: bool = True, - google_search: bool = True, - hide_canvas: bool = False, - disable_webgl: bool = False, - real_chrome: bool = False, - stealth: bool = False, - wait: int | float = 0, - page_action: Optional[Callable] = None, - proxy: Optional[str | Dict[str, str]] = None, - locale: str = "en-US", - extra_headers: Optional[Dict[str, str]] = None, - useragent: Optional[str] = None, - cdp_url: Optional[str] = None, - timeout: int | float = 30000, - disable_resources: bool = False, - wait_selector: Optional[str] = None, - init_script: Optional[str] = None, - cookies: Optional[List[Dict]] = None, - network_idle: bool = False, - load_dom: bool = True, - wait_selector_state: SelectorWaitStates = "attached", - extra_flags: Optional[List[str]] = None, - additional_args: Optional[Dict] = None, - custom_config: Optional[Dict] = None, - ) -> Response: + def fetch(cls, url: str, **kwargs: Unpack[PlaywrightSession]) -> Response: """Opens up a browser and do your request based on your chosen options below. :param url: Target url. - :param headless: Run the browser in headless/hidden (default), or headful/visible mode. - :param disable_resources: Drop requests of unnecessary resources for a speed boost. It depends, but it made requests ~25% faster in my tests for some websites. - Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`. - This can help save your proxy usage but be careful with this option as it makes some websites never finish loading. - :param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it. - :param cookies: Set cookies for the next request. - :param network_idle: Wait for the page until there are no network connections for at least 500 ms. - :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute. - :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000 - :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object. - :param page_action: Added for automation. A function that takes the `page` object and does the automation you need. - :param wait_selector: Wait for a specific CSS selector to be in a specific state. - :param init_script: An absolute path to a JavaScript file to be executed on page creation with this request. - :param locale: Set the locale for the browser if wanted. The default value is `en-US`. - :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`. - :param stealth: Enables stealth mode, check the documentation to see what stealth mode does currently. - :param real_chrome: If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch an instance of your browser and use it. - :param hide_canvas: Add random noise to canvas operations to prevent fingerprinting. - :param disable_webgl: Disables WebGL and WebGL 2.0 support entirely. - :param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers through CDP. - :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name. - :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ - :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only. - :param extra_flags: A list of additional browser flags to pass to the browser on launch. - :param custom_config: A dictionary of custom parser arguments to use with this request. Any argument passed will override any class parameters values. - :param additional_args: Additional arguments to be passed to Playwright's context as additional settings, and it takes higher priority than Scrapling's settings. + :param kwargs: Browser session configuration options including: + - headless: Run the browser in headless/hidden (default), or headful/visible mode. + - disable_resources: Drop requests of unnecessary resources for a speed boost. + - useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it. + - cookies: Set cookies for the next request. + - network_idle: Wait for the page until there are no network connections for at least 500 ms. + - load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute. + - timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000 + - wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the Response object. + - page_action: Added for automation. A function that takes the `page` object and does the automation you need. + - wait_selector: Wait for a specific CSS selector to be in a specific state. + - init_script: An absolute path to a JavaScript file to be executed on page creation with this request. + - locale: Set the locale for the browser if wanted. The default value is `en-US`. + - wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`. + - stealth: Enables stealth mode, check the documentation to see what stealth mode does currently. + - real_chrome: If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch an instance of your browser and use it. + - hide_canvas: Add random noise to canvas operations to prevent fingerprinting. + - disable_webgl: Disables WebGL and WebGL 2.0 support entirely. + - cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers through CDP. + - google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name. + - extra_headers: A dictionary of extra headers to add to the request. + - proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only. + - extra_flags: A list of additional browser flags to pass to the browser on launch. + - selector_config: The arguments that will be passed in the end while creating the final Selector's class. + - additional_args: Additional arguments to be passed to Playwright's context as additional settings. :return: A `Response` object. """ - if not custom_config: - custom_config = {} - elif not isinstance(custom_config, dict): - raise ValueError(f"The custom parser config must be of type dictionary, got {cls.__class__}") + # Get selector_config from kwargs if provided, otherwise use empty dict + selector_config = kwargs.get("selector_config", {}) + if not isinstance(selector_config, dict): + raise TypeError("Argument `selector_config` must be a dictionary.") - with DynamicSession( - wait=wait, - proxy=proxy, - locale=locale, - timeout=timeout, - stealth=stealth, - cdp_url=cdp_url, - cookies=cookies, - headless=headless, - load_dom=load_dom, - useragent=useragent, - real_chrome=real_chrome, - page_action=page_action, - hide_canvas=hide_canvas, - init_script=init_script, - network_idle=network_idle, - google_search=google_search, - extra_headers=extra_headers, - wait_selector=wait_selector, - disable_webgl=disable_webgl, - extra_flags=extra_flags, - additional_args=additional_args, - disable_resources=disable_resources, - wait_selector_state=wait_selector_state, - selector_config={**cls._generate_parser_arguments(), **custom_config}, - ) as session: + # Merge selector_config with class defaults + kwargs["selector_config"] = {**cls._generate_parser_arguments(), **selector_config} + + with DynamicSession(**kwargs) as session: return session.fetch(url) @classmethod - async def async_fetch( - cls, - url: str, - headless: bool = True, - google_search: bool = True, - hide_canvas: bool = False, - disable_webgl: bool = False, - real_chrome: bool = False, - stealth: bool = False, - wait: int | float = 0, - page_action: Optional[Callable] = None, - proxy: Optional[str | Dict[str, str]] = None, - locale: str = "en-US", - extra_headers: Optional[Dict[str, str]] = None, - useragent: Optional[str] = None, - cdp_url: Optional[str] = None, - timeout: int | float = 30000, - disable_resources: bool = False, - wait_selector: Optional[str] = None, - init_script: Optional[str] = None, - cookies: Optional[List[Dict]] = None, - network_idle: bool = False, - load_dom: bool = True, - wait_selector_state: SelectorWaitStates = "attached", - extra_flags: Optional[List[str]] = None, - additional_args: Optional[Dict] = None, - custom_config: Optional[Dict] = None, - ) -> Response: + async def async_fetch(cls, url: str, **kwargs: Unpack[PlaywrightSession]) -> Response: """Opens up a browser and do your request based on your chosen options below. :param url: Target url. - :param headless: Run the browser in headless/hidden (default), or headful/visible mode. - :param disable_resources: Drop requests of unnecessary resources for a speed boost. It depends, but it made requests ~25% faster in my tests for some websites. - Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`. - This can help save your proxy usage but be careful with this option as it makes some websites never finish loading. - :param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it. - :param cookies: Set cookies for the next request. - :param network_idle: Wait for the page until there are no network connections for at least 500 ms. - :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute. - :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000 - :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object. - :param page_action: Added for automation. A function that takes the `page` object and does the automation you need. - :param wait_selector: Wait for a specific CSS selector to be in a specific state. - :param init_script: An absolute path to a JavaScript file to be executed on page creation with this request. - :param locale: Set the locale for the browser if wanted. The default value is `en-US`. - :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`. - :param stealth: Enables stealth mode, check the documentation to see what stealth mode does currently. - :param real_chrome: If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch an instance of your browser and use it. - :param hide_canvas: Add random noise to canvas operations to prevent fingerprinting. - :param disable_webgl: Disables WebGL and WebGL 2.0 support entirely. - :param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers through CDP. - :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name. - :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ - :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only. - :param extra_flags: A list of additional browser flags to pass to the browser on launch. - :param custom_config: A dictionary of custom parser arguments to use with this request. Any argument passed will override any class parameters values. - :param additional_args: Additional arguments to be passed to Playwright's context as additional settings, and it takes higher priority than Scrapling's settings. + :param kwargs: Browser session configuration options including: + - headless: Run the browser in headless/hidden (default), or headful/visible mode. + - disable_resources: Drop requests of unnecessary resources for a speed boost. + - useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it. + - cookies: Set cookies for the next request. + - network_idle: Wait for the page until there are no network connections for at least 500 ms. + - load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute. + - timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000 + - wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the Response object. + - page_action: Added for automation. A function that takes the `page` object and does the automation you need. + - wait_selector: Wait for a specific CSS selector to be in a specific state. + - init_script: An absolute path to a JavaScript file to be executed on page creation with this request. + - locale: Set the locale for the browser if wanted. The default value is `en-US`. + - wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`. + - stealth: Enables stealth mode, check the documentation to see what stealth mode does currently. + - real_chrome: If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch an instance of your browser and use it. + - hide_canvas: Add random noise to canvas operations to prevent fingerprinting. + - disable_webgl: Disables WebGL and WebGL 2.0 support entirely. + - cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers through CDP. + - google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name. + - extra_headers: A dictionary of extra headers to add to the request. + - proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only. + - extra_flags: A list of additional browser flags to pass to the browser on launch. + - selector_config: The arguments that will be passed in the end while creating the final Selector's class. + - additional_args: Additional arguments to be passed to Playwright's context as additional settings. :return: A `Response` object. """ - if not custom_config: - custom_config = {} - elif not isinstance(custom_config, dict): - raise ValueError(f"The custom parser config must be of type dictionary, got {cls.__class__}") + # Get selector_config from kwargs if provided, otherwise use empty dict + selector_config = kwargs.get("selector_config", {}) + if not isinstance(selector_config, dict): + raise TypeError("Argument `selector_config` must be a dictionary.") - async with AsyncDynamicSession( - wait=wait, - max_pages=1, - proxy=proxy, - locale=locale, - timeout=timeout, - stealth=stealth, - cdp_url=cdp_url, - cookies=cookies, - headless=headless, - load_dom=load_dom, - useragent=useragent, - real_chrome=real_chrome, - page_action=page_action, - hide_canvas=hide_canvas, - init_script=init_script, - network_idle=network_idle, - google_search=google_search, - extra_headers=extra_headers, - wait_selector=wait_selector, - disable_webgl=disable_webgl, - extra_flags=extra_flags, - additional_args=additional_args, - disable_resources=disable_resources, - wait_selector_state=wait_selector_state, - selector_config={**cls._generate_parser_arguments(), **custom_config}, - ) as session: + # Merge selector_config with class defaults + kwargs["selector_config"] = {**cls._generate_parser_arguments(), **selector_config} + + async with AsyncDynamicSession(**kwargs) as session: return await session.fetch(url) diff --git a/scrapling/fetchers/firefox.py b/scrapling/fetchers/firefox.py index 5986096..825b917 100644 --- a/scrapling/fetchers/firefox.py +++ b/scrapling/fetchers/firefox.py @@ -1,10 +1,5 @@ -from scrapling.core._types import ( - Callable, - Dict, - List, - Optional, - SelectorWaitStates, -) +from scrapling.core._types import Unpack +from scrapling.engines._browsers._types import CamoufoxSession from scrapling.engines.toolbelt.custom import BaseFetcher, Response from scrapling.engines._browsers._camoufox import StealthySession, AsyncStealthySession @@ -17,196 +12,91 @@ class StealthyFetcher(BaseFetcher): """ @classmethod - def fetch( - cls, - url: str, - headless: bool = True, # noqa: F821 - block_images: bool = False, - disable_resources: bool = False, - block_webrtc: bool = False, - allow_webgl: bool = True, - network_idle: bool = False, - load_dom: bool = True, - humanize: bool | float = True, - solve_cloudflare: bool = False, - wait: int | float = 0, - timeout: int | float = 30000, - page_action: Optional[Callable] = None, - wait_selector: Optional[str] = None, - init_script: Optional[str] = None, - addons: Optional[List[str]] = None, - wait_selector_state: SelectorWaitStates = "attached", - cookies: Optional[List[Dict]] = None, - google_search: bool = True, - extra_headers: Optional[Dict[str, str]] = None, - proxy: Optional[str | Dict[str, str]] = None, - os_randomize: bool = False, - disable_ads: bool = False, - geoip: bool = False, - custom_config: Optional[Dict] = None, - additional_args: Optional[Dict] = None, - ) -> Response: + def fetch(cls, url: str, **kwargs: Unpack[CamoufoxSession]) -> Response: """ Opens up a browser and do your request based on your chosen options below. :param url: Target url. - :param headless: Run the browser in headless/hidden (default), or headful/visible mode. - :param block_images: Prevent the loading of images through Firefox preferences. - This can help save your proxy usage but be careful with this option as it makes some websites never finish loading. - :param disable_resources: Drop requests of unnecessary resources for a speed boost. It depends, but it made requests ~25% faster in my tests for some websites. - Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`. - This can help save your proxy usage but be careful with this option as it makes some websites never finish loading. - :param block_webrtc: Blocks WebRTC entirely. - :param cookies: Set cookies for the next request. - :param addons: List of Firefox addons to use. Must be paths to extracted addons. - :param humanize: Humanize the cursor movement. Takes either True or the MAX duration in seconds of the cursor movement. The cursor typically takes up to 1.5 seconds to move across the window. - :param solve_cloudflare: Solves all types of the Cloudflare's Turnstile/Interstitial challenges before returning the response to you. - :param allow_webgl: Enabled by default. Disabling WebGL is not recommended as many WAFs now check if WebGL is enabled. - :param network_idle: Wait for the page until there are no network connections for at least 500 ms. - :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute. - :param disable_ads: Disabled by default, this installs the `uBlock Origin` addon on the browser if enabled. - :param os_randomize: If enabled, Scrapling will randomize the OS fingerprints used. The default is Scrapling matching the fingerprints with the current OS. - :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object. - :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000 - :param page_action: Added for automation. A function that takes the `page` object and does the automation you need. - :param wait_selector: Wait for a specific CSS selector to be in a specific state. - :param init_script: An absolute path to a JavaScript file to be executed on page creation with this request. - :param geoip: Recommended to use with proxies; Automatically use IP's longitude, latitude, timezone, country, locale, and spoof the WebRTC IP address. - It will also calculate and spoof the browser's language based on the distribution of language speakers in the target region. - :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`. - :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name. - :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ - :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only. - :param custom_config: A dictionary of custom parser arguments to use with this request. Any argument passed will override any class parameters values. - :param additional_args: Additional arguments to be passed to Camoufox as additional settings, and it takes higher priority than Scrapling's settings. + :param kwargs: Browser session configuration options including: + - headless: Run the browser in headless/hidden (default), or headful/visible mode. + - block_images: Prevent the loading of images through Firefox preferences. + - disable_resources: Drop requests of unnecessary resources for a speed boost. + - block_webrtc: Blocks WebRTC entirely. + - allow_webgl: Enabled by default. Disabling WebGL is not recommended as many WAFs now check if WebGL is enabled. + - network_idle: Wait for the page until there are no network connections for at least 500 ms. + - load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute. + - humanize: Humanize the cursor movement. Takes either True or the MAX duration in seconds of the cursor movement. + - solve_cloudflare: Solves all types of the Cloudflare's Turnstile/Interstitial challenges before returning the response to you. + - wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the Response object. + - timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000 + - page_action: Added for automation. A function that takes the `page` object and does the automation you need. + - wait_selector: Wait for a specific CSS selector to be in a specific state. + - init_script: An absolute path to a JavaScript file to be executed on page creation with this request. + - addons: List of Firefox addons to use. Must be paths to extracted addons. + - wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`. + - cookies: Set cookies for the next request. + - google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name. + - extra_headers: A dictionary of extra headers to add to the request. + - proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only. + - os_randomize: If enabled, Scrapling will randomize the OS fingerprints used. + - disable_ads: Disabled by default, this installs the `uBlock Origin` addon on the browser if enabled. + - geoip: Recommended to use with proxies; Automatically use IP's longitude, latitude, timezone, country, locale, and spoof the WebRTC IP address. + - selector_config: The arguments that will be passed in the end while creating the final Selector's class. + - additional_args: Additional arguments to be passed to Camoufox as additional settings. :return: A `Response` object. """ - if not custom_config: - custom_config = {} + # Get selector_config from kwargs if provided, otherwise use empty dict + selector_config = kwargs.get("selector_config", {}) + if not isinstance(selector_config, dict): + raise TypeError("Argument `selector_config` must be a dictionary.") - with StealthySession( - wait=wait, - proxy=proxy, - geoip=geoip, - addons=addons, - timeout=timeout, - cookies=cookies, - headless=headless, - humanize=humanize, - load_dom=load_dom, - disable_ads=disable_ads, - allow_webgl=allow_webgl, - page_action=page_action, - init_script=init_script, - network_idle=network_idle, - block_images=block_images, - block_webrtc=block_webrtc, - os_randomize=os_randomize, - wait_selector=wait_selector, - google_search=google_search, - extra_headers=extra_headers, - solve_cloudflare=solve_cloudflare, - disable_resources=disable_resources, - wait_selector_state=wait_selector_state, - selector_config={**cls._generate_parser_arguments(), **custom_config}, - additional_args=additional_args or {}, - ) as engine: + # Merge selector_config with class defaults + kwargs["selector_config"] = {**cls._generate_parser_arguments(), **selector_config} + + with StealthySession(**kwargs) as engine: return engine.fetch(url) @classmethod - async def async_fetch( - cls, - url: str, - headless: bool = True, # noqa: F821 - block_images: bool = False, - disable_resources: bool = False, - block_webrtc: bool = False, - allow_webgl: bool = True, - network_idle: bool = False, - load_dom: bool = True, - humanize: bool | float = True, - solve_cloudflare: bool = False, - wait: int | float = 0, - timeout: int | float = 30000, - page_action: Optional[Callable] = None, - wait_selector: Optional[str] = None, - init_script: Optional[str] = None, - addons: Optional[List[str]] = None, - wait_selector_state: SelectorWaitStates = "attached", - cookies: Optional[List[Dict]] = None, - google_search: bool = True, - extra_headers: Optional[Dict[str, str]] = None, - proxy: Optional[str | Dict[str, str]] = None, - os_randomize: bool = False, - disable_ads: bool = False, - geoip: bool = False, - custom_config: Optional[Dict] = None, - additional_args: Optional[Dict] = None, - ) -> Response: + async def async_fetch(cls, url: str, **kwargs: Unpack[CamoufoxSession]) -> Response: """ Opens up a browser and do your request based on your chosen options below. :param url: Target url. - :param headless: Run the browser in headless/hidden (default), or headful/visible mode. - :param block_images: Prevent the loading of images through Firefox preferences. - This can help save your proxy usage but be careful with this option as it makes some websites never finish loading. - :param disable_resources: Drop requests of unnecessary resources for a speed boost. It depends, but it made requests ~25% faster in my tests for some websites. - Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`. - This can help save your proxy usage but be careful with this option as it makes some websites never finish loading. - :param block_webrtc: Blocks WebRTC entirely. - :param cookies: Set cookies for the next request. - :param addons: List of Firefox addons to use. Must be paths to extracted addons. - :param humanize: Humanize the cursor movement. Takes either True or the MAX duration in seconds of the cursor movement. The cursor typically takes up to 1.5 seconds to move across the window. - :param solve_cloudflare: Solves all types of the Cloudflare's Turnstile/Interstitial challenges before returning the response to you. - :param allow_webgl: Enabled by default. Disabling WebGL is not recommended as many WAFs now check if WebGL is enabled. - :param network_idle: Wait for the page until there are no network connections for at least 500 ms. - :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute. - :param disable_ads: Disabled by default, this installs the `uBlock Origin` addon on the browser if enabled. - :param os_randomize: If enabled, Scrapling will randomize the OS fingerprints used. The default is Scrapling matching the fingerprints with the current OS. - :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object. - :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000 - :param page_action: Added for automation. A function that takes the `page` object and does the automation you need. - :param wait_selector: Wait for a specific CSS selector to be in a specific state. - :param init_script: An absolute path to a JavaScript file to be executed on page creation with this request. - :param geoip: Recommended to use with proxies; Automatically use IP's longitude, latitude, timezone, country, locale, and spoof the WebRTC IP address. - It will also calculate and spoof the browser's language based on the distribution of language speakers in the target region. - :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`. - :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name. - :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ - :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only. - :param custom_config: A dictionary of custom parser arguments to use with this request. Any argument passed will override any class parameters values. - :param additional_args: Additional arguments to be passed to Camoufox as additional settings, and it takes higher priority than Scrapling's settings. + :param kwargs: Browser session configuration options including: + - headless: Run the browser in headless/hidden (default), or headful/visible mode. + - block_images: Prevent the loading of images through Firefox preferences. + - disable_resources: Drop requests of unnecessary resources for a speed boost. + - block_webrtc: Blocks WebRTC entirely. + - allow_webgl: Enabled by default. Disabling WebGL is not recommended as many WAFs now check if WebGL is enabled. + - network_idle: Wait for the page until there are no network connections for at least 500 ms. + - load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute. + - humanize: Humanize the cursor movement. Takes either True or the MAX duration in seconds of the cursor movement. + - solve_cloudflare: Solves all types of the Cloudflare's Turnstile/Interstitial challenges before returning the response to you. + - wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the Response object. + - timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000 + - page_action: Added for automation. A function that takes the `page` object and does the automation you need. + - wait_selector: Wait for a specific CSS selector to be in a specific state. + - init_script: An absolute path to a JavaScript file to be executed on page creation with this request. + - addons: List of Firefox addons to use. Must be paths to extracted addons. + - wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`. + - cookies: Set cookies for the next request. + - google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name. + - extra_headers: A dictionary of extra headers to add to the request. + - proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only. + - os_randomize: If enabled, Scrapling will randomize the OS fingerprints used. + - disable_ads: Disabled by default, this installs the `uBlock Origin` addon on the browser if enabled. + - geoip: Recommended to use with proxies; Automatically use IP's longitude, latitude, timezone, country, locale, and spoof the WebRTC IP address. + - selector_config: The arguments that will be passed in the end while creating the final Selector's class. + - additional_args: Additional arguments to be passed to Camoufox as additional settings. :return: A `Response` object. """ - if not custom_config: - custom_config = {} + # Get selector_config from kwargs if provided, otherwise use empty dict + selector_config = kwargs.get("selector_config", {}) + if not isinstance(selector_config, dict): + raise TypeError("Argument `selector_config` must be a dictionary.") - async with AsyncStealthySession( - wait=wait, - max_pages=1, - proxy=proxy, - geoip=geoip, - addons=addons, - timeout=timeout, - cookies=cookies, - headless=headless, - humanize=humanize, - load_dom=load_dom, - disable_ads=disable_ads, - allow_webgl=allow_webgl, - page_action=page_action, - init_script=init_script, - network_idle=network_idle, - block_images=block_images, - block_webrtc=block_webrtc, - os_randomize=os_randomize, - wait_selector=wait_selector, - google_search=google_search, - extra_headers=extra_headers, - solve_cloudflare=solve_cloudflare, - disable_resources=disable_resources, - wait_selector_state=wait_selector_state, - selector_config={**cls._generate_parser_arguments(), **custom_config}, - additional_args=additional_args or {}, - ) as engine: + # Merge selector_config with class defaults + kwargs["selector_config"] = {**cls._generate_parser_arguments(), **selector_config} + + async with AsyncStealthySession(**kwargs) as engine: return await engine.fetch(url) From f84fbe88148f6a4c99fc0b01da446bf4eb3b1b9c Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Wed, 26 Nov 2025 02:12:50 +0200 Subject: [PATCH 17/21] docs: update the renamed argument --- docs/fetching/choosing.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/fetching/choosing.md b/docs/fetching/choosing.md index 50ddf37..a56459e 100644 --- a/docs/fetching/choosing.md +++ b/docs/fetching/choosing.md @@ -57,7 +57,7 @@ The available configuration arguments are: `adaptive`, `huge_tree`, `keep_commen ### Set parser config per request As you probably understand, the logic above for setting the parser config will apply globally to all requests/fetches made through that class, and it's intended for simplicity. -If your use case requires a different configuration for each request/fetch, you can pass a dictionary to the request method (`fetch`/`get`/`post`/...) to an argument named `custom_config`. +If your use case requires a different configuration for each request/fetch, you can pass a dictionary to the request method (`fetch`/`get`/`post`/...) to an argument named `selector_config`. ## Response Object The `Response` object is the same as the [Selector](../parsing/main_classes.md#selector) class, but it has additional details about the response, like response headers, status, cookies, etc., as shown below: From 9b3309f825ec45aec551e628bdb6e71d522cf32a Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Wed, 26 Nov 2025 02:22:29 +0200 Subject: [PATCH 18/21] fix(shell): use correct argument name in signatures --- scrapling/core/_shell_signatures.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapling/core/_shell_signatures.py b/scrapling/core/_shell_signatures.py index 907a9f8..803e6d5 100644 --- a/scrapling/core/_shell_signatures.py +++ b/scrapling/core/_shell_signatures.py @@ -53,7 +53,7 @@ _FETCH_PARAMS = { "wait_selector_state": Any, "extra_flags": Optional[List[str]], "additional_args": Optional[Dict], - "custom_config": Optional[Dict], + "selector_config": Optional[Dict], } _STEALTHY_FETCH_PARAMS = { @@ -80,7 +80,7 @@ _STEALTHY_FETCH_PARAMS = { "os_randomize": bool, "disable_ads": bool, "geoip": bool, - "custom_config": Optional[Dict], + "selector_config": Optional[Dict], "additional_args": Optional[Dict], } From 0e1f3b9b05958d7b3b7340c2acb19ce52a092a22 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Wed, 26 Nov 2025 02:23:17 +0200 Subject: [PATCH 19/21] fix(fetchers): Add backward compatibility for `custom_config` argument --- scrapling/fetchers/chrome.py | 12 ++++++------ scrapling/fetchers/firefox.py | 12 ++++++------ 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/scrapling/fetchers/chrome.py b/scrapling/fetchers/chrome.py index 44e7a44..a706826 100644 --- a/scrapling/fetchers/chrome.py +++ b/scrapling/fetchers/chrome.py @@ -52,12 +52,12 @@ class DynamicFetcher(BaseFetcher): - additional_args: Additional arguments to be passed to Playwright's context as additional settings. :return: A `Response` object. """ - # Get selector_config from kwargs if provided, otherwise use empty dict - selector_config = kwargs.get("selector_config", {}) + selector_config = kwargs.get("selector_config", {}) or kwargs.get( + "custom_config", {} + ) # Checking `custom_config` for backward compatibility if not isinstance(selector_config, dict): raise TypeError("Argument `selector_config` must be a dictionary.") - # Merge selector_config with class defaults kwargs["selector_config"] = {**cls._generate_parser_arguments(), **selector_config} with DynamicSession(**kwargs) as session: @@ -95,12 +95,12 @@ class DynamicFetcher(BaseFetcher): - additional_args: Additional arguments to be passed to Playwright's context as additional settings. :return: A `Response` object. """ - # Get selector_config from kwargs if provided, otherwise use empty dict - selector_config = kwargs.get("selector_config", {}) + selector_config = kwargs.get("selector_config", {}) or kwargs.get( + "custom_config", {} + ) # Checking `custom_config` for backward compatibility if not isinstance(selector_config, dict): raise TypeError("Argument `selector_config` must be a dictionary.") - # Merge selector_config with class defaults kwargs["selector_config"] = {**cls._generate_parser_arguments(), **selector_config} async with AsyncDynamicSession(**kwargs) as session: diff --git a/scrapling/fetchers/firefox.py b/scrapling/fetchers/firefox.py index 825b917..a9361d6 100644 --- a/scrapling/fetchers/firefox.py +++ b/scrapling/fetchers/firefox.py @@ -45,12 +45,12 @@ class StealthyFetcher(BaseFetcher): - additional_args: Additional arguments to be passed to Camoufox as additional settings. :return: A `Response` object. """ - # Get selector_config from kwargs if provided, otherwise use empty dict - selector_config = kwargs.get("selector_config", {}) + selector_config = kwargs.get("selector_config", {}) or kwargs.get( + "custom_config", {} + ) # Checking `custom_config` for backward compatibility if not isinstance(selector_config, dict): raise TypeError("Argument `selector_config` must be a dictionary.") - # Merge selector_config with class defaults kwargs["selector_config"] = {**cls._generate_parser_arguments(), **selector_config} with StealthySession(**kwargs) as engine: @@ -90,12 +90,12 @@ class StealthyFetcher(BaseFetcher): - additional_args: Additional arguments to be passed to Camoufox as additional settings. :return: A `Response` object. """ - # Get selector_config from kwargs if provided, otherwise use empty dict - selector_config = kwargs.get("selector_config", {}) + selector_config = kwargs.get("selector_config", {}) or kwargs.get( + "custom_config", {} + ) # Checking `custom_config` for backward compatibility if not isinstance(selector_config, dict): raise TypeError("Argument `selector_config` must be a dictionary.") - # Merge selector_config with class defaults kwargs["selector_config"] = {**cls._generate_parser_arguments(), **selector_config} async with AsyncStealthySession(**kwargs) as engine: From ab23b71836e08addb57119d7ec559bcd21857712 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Wed, 26 Nov 2025 02:23:54 +0200 Subject: [PATCH 20/21] tests(fetchers): Update tests to use the new argument naming --- tests/fetchers/async/test_camoufox.py | 2 +- tests/fetchers/async/test_dynamic.py | 2 +- tests/fetchers/sync/test_camoufox.py | 2 +- tests/fetchers/sync/test_dynamic.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/fetchers/async/test_camoufox.py b/tests/fetchers/async/test_camoufox.py index ffe6eac..e8bbd3f 100644 --- a/tests/fetchers/async/test_camoufox.py +++ b/tests/fetchers/async/test_camoufox.py @@ -70,7 +70,7 @@ class TestStealthyFetcher: "os_randomize": True, "disable_ads": True, # "geoip": True, - "custom_config": {"keep_comments": False, "keep_cdata": False}, + "selector_config": {"keep_comments": False, "keep_cdata": False}, "additional_args": {"window": (1920, 1080)}, }, ], diff --git a/tests/fetchers/async/test_dynamic.py b/tests/fetchers/async/test_dynamic.py index 3f5f7b5..0d46365 100644 --- a/tests/fetchers/async/test_dynamic.py +++ b/tests/fetchers/async/test_dynamic.py @@ -68,7 +68,7 @@ class TestDynamicFetcherAsync: "useragent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:131.0) Gecko/20100101 Firefox/131.0", "cookies": [{"name": "test", "value": "123", "domain": "example.com", "path": "/"}], "network_idle": True, - "custom_config": {"keep_comments": False, "keep_cdata": False}, + "selector_config": {"keep_comments": False, "keep_cdata": False}, }, ], ) diff --git a/tests/fetchers/sync/test_camoufox.py b/tests/fetchers/sync/test_camoufox.py index 83c5f6c..269238f 100644 --- a/tests/fetchers/sync/test_camoufox.py +++ b/tests/fetchers/sync/test_camoufox.py @@ -65,7 +65,7 @@ class TestStealthyFetcher: "os_randomize": True, "disable_ads": True, # "geoip": True, - "custom_config": {"keep_comments": False, "keep_cdata": False}, + "selector_config": {"keep_comments": False, "keep_cdata": False}, "additional_args": {"window": (1920, 1080)}, }, ], diff --git a/tests/fetchers/sync/test_dynamic.py b/tests/fetchers/sync/test_dynamic.py index a60d9d8..2b1a54d 100644 --- a/tests/fetchers/sync/test_dynamic.py +++ b/tests/fetchers/sync/test_dynamic.py @@ -66,7 +66,7 @@ class TestDynamicFetcher: "useragent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:131.0) Gecko/20100101 Firefox/131.0", "cookies": [{"name": "test", "value": "123", "domain": "example.com", "path": "/"}], "network_idle": True, - "custom_config": {"keep_comments": False, "keep_cdata": False}, + "selector_config": {"keep_comments": False, "keep_cdata": False}, }, ], ) From eb15aae444a0c2f383b00ec572be707366c2f27e Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Wed, 26 Nov 2025 02:50:01 +0200 Subject: [PATCH 21/21] docs: update benchmarks with the current results --- docs/README.md | 21 +++++++++++---------- docs/README_AR.md | 27 ++++++++++++++------------- docs/README_CN.md | 29 +++++++++++++++-------------- docs/README_DE.md | 27 ++++++++++++++------------- docs/README_ES.md | 29 +++++++++++++++-------------- docs/README_JP.md | 29 +++++++++++++++-------------- docs/README_RU.md | 27 ++++++++++++++------------- docs/benchmarks.md | 22 +++++++++++----------- 8 files changed, 109 insertions(+), 102 deletions(-) diff --git a/docs/README.md b/docs/README.md index d5b9a75..fa1f217 100644 --- a/docs/README.md +++ b/docs/README.md @@ -244,14 +244,15 @@ Scrapling isn't just powerful—it's also blazing fast, and the updates since ve | # | Library | Time (ms) | vs Scrapling | |---|:-----------------:|:---------:|:------------:| -| 1 | Scrapling | 1.92 | 1.0x | -| 2 | Parsel/Scrapy | 1.99 | 1.036x | -| 3 | Raw Lxml | 2.33 | 1.214x | -| 4 | PyQuery | 20.61 | ~11x | -| 5 | Selectolax | 80.65 | ~42x | -| 6 | BS4 with Lxml | 1283.21 | ~698x | -| 7 | MechanicalSoup | 1304.57 | ~679x | -| 8 | BS4 with html5lib | 3331.96 | ~1735x | +| 1 | Scrapling | 1.99 | 1.0x | +| 2 | Parsel/Scrapy | 2.01 | 1.01x | +| 3 | Raw Lxml | 2.5 | 1.256x | +| 4 | PyQuery | 22.93 | ~11.5x | +| 5 | Selectolax | 80.57 | ~40.5x | +| 6 | BS4 with Lxml | 1541.37 | ~774.6x | +| 7 | MechanicalSoup | 1547.35 | ~777.6x | +| 8 | BS4 with html5lib | 3410.58 | ~1713.9x | + ### Element Similarity & Text Search Performance @@ -259,8 +260,8 @@ Scrapling's adaptive element finding capabilities significantly outperform alter | Library | Time (ms) | vs Scrapling | |-------------|:---------:|:------------:| -| Scrapling | 1.87 | 1.0x | -| AutoScraper | 10.24 | 5.476x | +| Scrapling | 2.46 | 1.0x | +| AutoScraper | 13.3 | 5.407x | > All benchmarks represent averages of 100+ runs. See [benchmarks.py](https://github.com/D4Vinci/Scrapling/blob/main/benchmarks.py) for methodology. diff --git a/docs/README_AR.md b/docs/README_AR.md index b3d6416..eab7c2f 100644 --- a/docs/README_AR.md +++ b/docs/README_AR.md @@ -233,24 +233,25 @@ Scrapling ليس قوياً فقط - إنه أيضاً سريع بشكل مذه ### اختبار سرعة استخراج النص (5000 عنصر متداخل) | # | المكتبة | الوقت (ms) | vs Scrapling | -|---|:-----------------:|:---------:|:------------:| -| 1 | Scrapling | 1.92 | 1.0x | -| 2 | Parsel/Scrapy | 1.99 | 1.036x | -| 3 | Raw Lxml | 2.33 | 1.214x | -| 4 | PyQuery | 20.61 | ~11x | -| 5 | Selectolax | 80.65 | ~42x | -| 6 | BS4 with Lxml | 1283.21 | ~698x | -| 7 | MechanicalSoup | 1304.57 | ~679x | -| 8 | BS4 with html5lib | 3331.96 | ~1735x | +|---|:-----------------:|:----------:|:------------:| +| 1 | Scrapling | 1.99 | 1.0x | +| 2 | Parsel/Scrapy | 2.01 | 1.01x | +| 3 | Raw Lxml | 2.5 | 1.256x | +| 4 | PyQuery | 22.93 | ~11.5x | +| 5 | Selectolax | 80.57 | ~40.5x | +| 6 | BS4 with Lxml | 1541.37 | ~774.6x | +| 7 | MechanicalSoup | 1547.35 | ~777.6x | +| 8 | BS4 with html5lib | 3410.58 | ~1713.9x | + ### أداء تشابه العناصر والبحث النصي قدرات العثور على العناصر التكيفية لـ Scrapling تتفوق بشكل كبير على البدائل: -| المكتبة | الوقت (ms) | vs Scrapling | -|-------------|:---------:|:------------:| -| Scrapling | 1.87 | 1.0x | -| AutoScraper | 10.24 | 5.476x | +| المكتبة | الوقت (ms) | vs Scrapling | +|-------------|:----------:|:------------:| +| Scrapling | 2.46 | 1.0x | +| AutoScraper | 13.3 | 5.407x | > تمثل جميع المعايير متوسطات أكثر من 100 تشغيل. انظر [benchmarks.py](https://github.com/D4Vinci/Scrapling/blob/main/benchmarks.py) للمنهجية. diff --git a/docs/README_CN.md b/docs/README_CN.md index c013b0f..b06a0cf 100644 --- a/docs/README_CN.md +++ b/docs/README_CN.md @@ -232,25 +232,26 @@ Scrapling不仅功能强大——它还速度极快，自0.3版本以来的更 ### 文本提取速度测试（5000个嵌套元素） -| # | 库 | 时间(ms) | vs Scrapling | -|---|:--------------:|:--------:|:------------:| -| 1 | Scrapling | 1.92 | 1.0x | -| 2 | Parsel/Scrapy | 1.99 | 1.036x | -| 3 | Raw Lxml | 2.33 | 1.214x | -| 4 | PyQuery | 20.61 | ~11x | -| 5 | Selectolax | 80.65 | ~42x | -| 6 | BS4 with Lxml | 1283.21 | ~698x | -| 7 | MechanicalSoup | 1304.57 | ~679x | -| 8 |BS4 with html5lib| 3331.96 | ~1735x | +| # | 库 | 时间(ms) | vs Scrapling | +|---|:-----------------:|:-------:|:------------:| +| 1 | Scrapling | 1.99 | 1.0x | +| 2 | Parsel/Scrapy | 2.01 | 1.01x | +| 3 | Raw Lxml | 2.5 | 1.256x | +| 4 | PyQuery | 22.93 | ~11.5x | +| 5 | Selectolax | 80.57 | ~40.5x | +| 6 | BS4 with Lxml | 1541.37 | ~774.6x | +| 7 | MechanicalSoup | 1547.35 | ~777.6x | +| 8 | BS4 with html5lib | 3410.58 | ~1713.9x | + ### 元素相似性和文本搜索性能 Scrapling的自适应元素查找功能明显优于替代方案： -| 库 | 时间(ms) | vs Scrapling | -|-------------|:--------:|:------------:| -| Scrapling | 1.87 | 1.0x | -| AutoScraper | 10.24 | 5.476x | +| 库 | 时间(ms) | vs Scrapling | +|-------------|:------:|:------------:| +| Scrapling | 2.46 | 1.0x | +| AutoScraper | 13.3 | 5.407x | > 所有基准测试代表100+次运行的平均值。请参阅[benchmarks.py](https://github.com/D4Vinci/Scrapling/blob/main/benchmarks.py)了解方法。 diff --git a/docs/README_DE.md b/docs/README_DE.md index 6312d5f..0cfbc40 100644 --- a/docs/README_DE.md +++ b/docs/README_DE.md @@ -232,25 +232,26 @@ Scrapling ist nicht nur leistungsstark – es ist auch blitzschnell, und die Upd ### Textextraktions-Geschwindigkeitstest (5000 verschachtelte Elemente) -| # | Bibliothek | Zeit (ms) | vs Scrapling | -|---|:--------------------:|:---------:|:------------:| -| 1 | Scrapling | 1.92 | 1.0x | -| 2 | Parsel/Scrapy | 1.99 | 1.036x | -| 3 | Raw Lxml | 2.33 | 1.214x | -| 4 | PyQuery | 20.61 | ~11x | -| 5 | Selectolax | 80.65 | ~42x | -| 6 | BS4 mit Lxml | 1283.21 | ~698x | -| 7 | MechanicalSoup | 1304.57 | ~679x | -| 8 | BS4 mit html5lib | 3331.96 | ~1735x | +| # | Bibliothek | Zeit (ms) | vs Scrapling | +|---|:-----------------:|:---------:|:------------:| +| 1 | Scrapling | 1.99 | 1.0x | +| 2 | Parsel/Scrapy | 2.01 | 1.01x | +| 3 | Raw Lxml | 2.5 | 1.256x | +| 4 | PyQuery | 22.93 | ~11.5x | +| 5 | Selectolax | 80.57 | ~40.5x | +| 6 | BS4 with Lxml | 1541.37 | ~774.6x | +| 7 | MechanicalSoup | 1547.35 | ~777.6x | +| 8 | BS4 with html5lib | 3410.58 | ~1713.9x | + ### Element-Ähnlichkeit & Textsuche-Leistung Scraplings adaptive Element-Finding-Fähigkeiten übertreffen Alternativen deutlich: -| Bibliothek | Zeit (ms) | vs Scrapling | +| Bibliothek | Zeit (ms) | vs Scrapling | |-------------|:---------:|:------------:| -| Scrapling | 1.87 | 1.0x | -| AutoScraper | 10.24 | 5.476x | +| Scrapling | 2.46 | 1.0x | +| AutoScraper | 13.3 | 5.407x | > Alle Benchmarks stellen Durchschnittswerte von über 100 Durchläufen dar. Siehe [benchmarks.py](https://github.com/D4Vinci/Scrapling/blob/main/benchmarks.py) für die Methodik. diff --git a/docs/README_ES.md b/docs/README_ES.md index 9cca19f..1ed495d 100644 --- a/docs/README_ES.md +++ b/docs/README_ES.md @@ -232,25 +232,26 @@ Scrapling no solo es poderoso, también es increíblemente rápido, y las actual ### Prueba de Velocidad de Extracción de Texto (5000 elementos anidados) -| # | Biblioteca | Tiempo (ms) | vs Scrapling | -|---|:--------------------:|:-----------:|:------------:| -| 1 | Scrapling | 1.92 | 1.0x | -| 2 | Parsel/Scrapy | 1.99 | 1.036x | -| 3 | Raw Lxml | 2.33 | 1.214x | -| 4 | PyQuery | 20.61 | ~11x | -| 5 | Selectolax | 80.65 | ~42x | -| 6 | BS4 con Lxml | 1283.21 | ~698x | -| 7 | MechanicalSoup | 1304.57 | ~679x | -| 8 | BS4 con html5lib | 3331.96 | ~1735x | +| # | Biblioteca | Tiempo (ms) | vs Scrapling | +|---|:-----------------:|:-----------:|:------------:| +| 1 | Scrapling | 1.99 | 1.0x | +| 2 | Parsel/Scrapy | 2.01 | 1.01x | +| 3 | Raw Lxml | 2.5 | 1.256x | +| 4 | PyQuery | 22.93 | ~11.5x | +| 5 | Selectolax | 80.57 | ~40.5x | +| 6 | BS4 with Lxml | 1541.37 | ~774.6x | +| 7 | MechanicalSoup | 1547.35 | ~777.6x | +| 8 | BS4 with html5lib | 3410.58 | ~1713.9x | + ### Rendimiento de Similitud de Elementos y Búsqueda de Texto Las capacidades de búsqueda adaptativa de elementos de Scrapling superan significativamente a las alternativas: -| Biblioteca | Tiempo (ms) | vs Scrapling | -|--------------|:-----------:|:------------:| -| Scrapling | 1.87 | 1.0x | -| AutoScraper | 10.24 | 5.476x | +| Biblioteca | Tiempo (ms) | vs Scrapling | +|-------------|:-----------:|:------------:| +| Scrapling | 2.46 | 1.0x | +| AutoScraper | 13.3 | 5.407x | > Todos los benchmarks representan promedios de más de 100 ejecuciones. Ver [benchmarks.py](https://github.com/D4Vinci/Scrapling/blob/main/benchmarks.py) para la metodología. diff --git a/docs/README_JP.md b/docs/README_JP.md index 597c98f..826d63e 100644 --- a/docs/README_JP.md +++ b/docs/README_JP.md @@ -232,25 +232,26 @@ Scraplingは強力であるだけでなく、驚くほど高速で、バージ ### テキスト抽出速度テスト（5000個のネストされた要素） -| # | ライブラリ | 時間(ms) | vs Scrapling | -|---|:-------------------:|:--------:|:------------:| -| 1 | Scrapling | 1.92 | 1.0x | -| 2 | Parsel/Scrapy | 1.99 | 1.036x | -| 3 | Raw Lxml | 2.33 | 1.214x | -| 4 | PyQuery | 20.61 | ~11x | -| 5 | Selectolax | 80.65 | ~42x | -| 6 | BS4 with Lxml | 1283.21 | ~698x | -| 7 | MechanicalSoup | 1304.57 | ~679x | -| 8 | BS4 with html5lib | 3331.96 | ~1735x | +| # | ライブラリ | 時間(ms) | vs Scrapling | +|---|:-----------------:|:-------:|:------------:| +| 1 | Scrapling | 1.99 | 1.0x | +| 2 | Parsel/Scrapy | 2.01 | 1.01x | +| 3 | Raw Lxml | 2.5 | 1.256x | +| 4 | PyQuery | 22.93 | ~11.5x | +| 5 | Selectolax | 80.57 | ~40.5x | +| 6 | BS4 with Lxml | 1541.37 | ~774.6x | +| 7 | MechanicalSoup | 1547.35 | ~777.6x | +| 8 | BS4 with html5lib | 3410.58 | ~1713.9x | + ### 要素類似性とテキスト検索のパフォーマンス Scraplingの適応型要素検索機能は代替手段を大幅に上回ります： -| ライブラリ | 時間(ms) | vs Scrapling | -|-------------|:--------:|:------------:| -| Scrapling | 1.87 | 1.0x | -| AutoScraper | 10.24 | 5.476x | +| ライブラリ | 時間(ms) | vs Scrapling | +|-------------|:------:|:------------:| +| Scrapling | 2.46 | 1.0x | +| AutoScraper | 13.3 | 5.407x | > すべてのベンチマークは100回以上の実行の平均を表します。方法論については[benchmarks.py](https://github.com/D4Vinci/Scrapling/blob/main/benchmarks.py)を参照してください。 diff --git a/docs/README_RU.md b/docs/README_RU.md index bdcccb1..0d868ca 100644 --- a/docs/README_RU.md +++ b/docs/README_RU.md @@ -232,25 +232,26 @@ Scrapling не только мощный - он также невероятно ### Тест скорости извлечения текста (5000 вложенных элементов) -| # | Библиотека | Время (мс) | vs Scrapling | -|---|:--------------------:|:----------:|:------------:| -| 1 | Scrapling | 1.92 | 1.0x | -| 2 | Parsel/Scrapy | 1.99 | 1.036x | -| 3 | Raw Lxml | 2.33 | 1.214x | -| 4 | PyQuery | 20.61 | ~11x | -| 5 | Selectolax | 80.65 | ~42x | -| 6 | BS4 с Lxml | 1283.21 | ~698x | -| 7 | MechanicalSoup | 1304.57 | ~679x | -| 8 | BS4 с html5lib | 3331.96 | ~1735x | +| # | Библиотека | Время (мс) | vs Scrapling | +|---|:-----------------:|:----------:|:------------:| +| 1 | Scrapling | 1.99 | 1.0x | +| 2 | Parsel/Scrapy | 2.01 | 1.01x | +| 3 | Raw Lxml | 2.5 | 1.256x | +| 4 | PyQuery | 22.93 | ~11.5x | +| 5 | Selectolax | 80.57 | ~40.5x | +| 6 | BS4 with Lxml | 1541.37 | ~774.6x | +| 7 | MechanicalSoup | 1547.35 | ~777.6x | +| 8 | BS4 with html5lib | 3410.58 | ~1713.9x | + ### Производительность подобия элементов и текстового поиска Возможности адаптивного поиска элементов Scrapling значительно превосходят альтернативы: -| Библиотека | Время (мс) | vs Scrapling | +| Библиотека | Время (мс) | vs Scrapling | |-------------|:----------:|:------------:| -| Scrapling | 1.87 | 1.0x | -| AutoScraper | 10.24 | 5.476x | +| Scrapling | 2.46 | 1.0x | +| AutoScraper | 13.3 | 5.407x | > Все тесты производительности представляют собой средние значения более 100 запусков. См. [benchmarks.py](https://github.com/D4Vinci/Scrapling/blob/main/benchmarks.py) для методологии. diff --git a/docs/benchmarks.md b/docs/benchmarks.md index 4e207ad..37fc6b1 100644 --- a/docs/benchmarks.md +++ b/docs/benchmarks.md @@ -8,20 +8,20 @@ Scrapling isn't just powerful—it's also blazing fast, and the updates since ve | # | Library | Time (ms) | vs Scrapling | |---|:-----------------:|:---------:|:------------:| -| 1 | Scrapling | 1.92 | 1.0x | -| 2 | Parsel/Scrapy | 1.99 | 1.036x | -| 3 | Raw Lxml | 2.33 | 1.214x | -| 4 | PyQuery | 20.61 | ~11x | -| 5 | Selectolax | 80.65 | ~42x | -| 6 | BS4 with Lxml | 1283.21 | ~698x | -| 7 | MechanicalSoup | 1304.57 | ~679x | -| 8 | BS4 with html5lib | 3331.96 | ~1735x | +| 1 | Scrapling | 1.99 | 1.0x | +| 2 | Parsel/Scrapy | 2.01 | 1.01x | +| 3 | Raw Lxml | 2.5 | 1.256x | +| 4 | PyQuery | 22.93 | ~11.5x | +| 5 | Selectolax | 80.57 | ~40.5x | +| 6 | BS4 with Lxml | 1541.37 | ~774.6x | +| 7 | MechanicalSoup | 1547.35 | ~777.6x | +| 8 | BS4 with html5lib | 3410.58 | ~1713.9x | ### Element Similarity & Text Search Performance Scrapling's adaptive element finding capabilities significantly outperform alternatives: -| Library | Time (ms) | vs Scrapling | +| Library | Time (ms) | vs Scrapling | |-------------|:---------:|:------------:| -| Scrapling | 1.87 | 1.0x | -| AutoScraper | 10.24 | 5.476x | +| Scrapling | 2.46 | 1.0x | +| AutoScraper | 13.3 | 5.407x |