From 842718ba94dea20f48cd66e6a1711a3195bbaa30 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Sat, 27 Dec 2025 14:30:48 +0200 Subject: [PATCH] fix: update shell signatures for the autocompletion Added missing fields and reordered classes as well for easier comparisons --- scrapling/core/_shell_signatures.py | 77 ++++++++++++---------- scrapling/engines/_browsers/_types.py | 5 +- scrapling/engines/_browsers/_validators.py | 33 +++++----- 3 files changed, 59 insertions(+), 56 deletions(-) diff --git a/scrapling/core/_shell_signatures.py b/scrapling/core/_shell_signatures.py index 803e6d5..d9a42fc 100644 --- a/scrapling/core/_shell_signatures.py +++ b/scrapling/core/_shell_signatures.py @@ -1,9 +1,13 @@ from scrapling.core._types import ( - Dict, Any, + Dict, List, Tuple, + Sequence, + Callable, Optional, + SetCookieParam, + SelectorWaitStates, ) # Parameter definitions for shell function signatures (defined once at module level) @@ -31,57 +35,58 @@ _REQUESTS_PARAMS = { _FETCH_PARAMS = { "headless": bool, - "google_search": bool, - "hide_canvas": bool, - "disable_webgl": bool, - "real_chrome": bool, - "stealth": bool, - "wait": int | float, - "page_action": Optional[Any], - "proxy": Optional[str | Dict], - "locale": str, - "extra_headers": Optional[Dict[str, str]], - "useragent": Optional[str], - "cdp_url": Optional[str], - "timeout": int | float, "disable_resources": bool, - "wait_selector": Optional[str], - "init_script": Optional[str], - "cookies": Optional[List[Dict]], "network_idle": bool, "load_dom": bool, - "wait_selector_state": Any, - "extra_flags": Optional[List[str]], - "additional_args": Optional[Dict], + "wait_selector": Optional[str], + "wait_selector_state": SelectorWaitStates, + "cookies": Sequence[SetCookieParam], + "google_search": bool, + "wait": int | float, + "timezone_id": str | None, + "page_action": Optional[Callable], + "proxy": Optional[str | Dict[str, str] | Tuple], + "extra_headers": Optional[Dict[str, str]], + "timeout": int | float, + "init_script": Optional[str], + "user_data_dir": str, "selector_config": Optional[Dict], + "additional_args": Optional[Dict], + "locale": Optional[str], + "real_chrome": bool, + "cdp_url": Optional[str], + "useragent": Optional[str], + "extra_flags": Optional[List[str]], } _STEALTHY_FETCH_PARAMS = { "headless": bool, - "block_images": bool, "disable_resources": bool, - "block_webrtc": bool, - "allow_webgl": bool, "network_idle": bool, "load_dom": bool, - "humanize": bool | float, - "solve_cloudflare": bool, - "wait": int | float, - "timeout": int | float, - "page_action": Optional[Any], "wait_selector": Optional[str], - "init_script": Optional[str], - "addons": Optional[List[str]], - "wait_selector_state": Any, - "cookies": Optional[List[Dict]], + "wait_selector_state": SelectorWaitStates, + "cookies": Sequence[SetCookieParam], "google_search": bool, + "wait": int | float, + "timezone_id": str | None, + "page_action": Optional[Callable], + "proxy": Optional[str | Dict[str, str] | Tuple], "extra_headers": Optional[Dict[str, str]], - "proxy": Optional[str | Dict], - "os_randomize": bool, - "disable_ads": bool, - "geoip": bool, + "timeout": int | float, + "init_script": Optional[str], + "user_data_dir": str, "selector_config": Optional[Dict], "additional_args": Optional[Dict], + "locale": Optional[str], + "real_chrome": bool, + "cdp_url": Optional[str], + "useragent": Optional[str], + "extra_flags": Optional[List[str]], + "allow_webgl": bool, + "hide_canvas": bool, + "block_webrtc": bool, + "solve_cloudflare": bool, } # Mapping of function names to their parameter definitions diff --git a/scrapling/engines/_browsers/_types.py b/scrapling/engines/_browsers/_types.py index 1cdbb9a..07cb90a 100644 --- a/scrapling/engines/_browsers/_types.py +++ b/scrapling/engines/_browsers/_types.py @@ -11,9 +11,10 @@ from scrapling.core._types import ( Mapping, Optional, Callable, - Iterable, + Sequence, TypedDict, TypeAlias, + SetCookieParam, SelectorWaitStates, TYPE_CHECKING, ) @@ -61,7 +62,7 @@ if TYPE_CHECKING: # pragma: no cover load_dom: bool wait_selector: Optional[str] wait_selector_state: SelectorWaitStates - cookies: Optional[Iterable[Dict]] + cookies: Sequence[SetCookieParam] google_search: bool wait: int | float timezone_id: str | None diff --git a/scrapling/engines/_browsers/_validators.py b/scrapling/engines/_browsers/_validators.py index cb3f48a..b148fe6 100644 --- a/scrapling/engines/_browsers/_validators.py +++ b/scrapling/engines/_browsers/_validators.py @@ -67,32 +67,29 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False, weakref=True): """Configuration struct for validation""" max_pages: PagesCount = 1 - cdp_url: Optional[str] = None headless: bool = True - google_search: bool = True - # hide_canvas: bool = False - # disable_webgl: bool = False - real_chrome: bool = False - # stealth: bool = False - wait: Seconds = 0 - page_action: Optional[Callable] = None - proxy: Optional[str | Dict[str, str] | Tuple] = None # The default value for proxy in Playwright's source is `None` - locale: str | None = None - extra_headers: Optional[Dict[str, str]] = None - useragent: Optional[str] = None - timeout: Seconds = 30000 - init_script: Optional[str] = None disable_resources: bool = False - wait_selector: Optional[str] = None - cookies: Sequence[SetCookieParam] | None = [] network_idle: bool = False load_dom: bool = True + wait_selector: Optional[str] = None wait_selector_state: SelectorWaitStates = "attached" - user_data_dir: str = "" + cookies: Sequence[SetCookieParam] | None = [] + google_search: bool = True + wait: Seconds = 0 timezone_id: str | None = "" - extra_flags: Optional[List[str]] = None + page_action: Optional[Callable] = None + proxy: Optional[str | Dict[str, str] | Tuple] = None # The default value for proxy in Playwright's source is `None` + extra_headers: Optional[Dict[str, str]] = None + timeout: Seconds = 30000 + init_script: Optional[str] = None + user_data_dir: str = "" selector_config: Optional[Dict] = {} additional_args: Optional[Dict] = {} + locale: str | None = None + real_chrome: bool = False + cdp_url: Optional[str] = None + useragent: Optional[str] = None + extra_flags: Optional[List[str]] = None def __post_init__(self): # pragma: no cover """Custom validation after msgspec validation"""