From d4a250766685e68849dcaa85578ff9463f968e3c Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Mon, 27 Oct 2025 04:48:45 +0300 Subject: [PATCH] refactor(validation): All browser based fetchers are ~8-15% faster now in most cases --- scrapling/engines/_browsers/_camoufox.py | 5 +- scrapling/engines/_browsers/_controllers.py | 4 +- scrapling/engines/_browsers/_validators.py | 130 +++++++++++--------- 3 files changed, 75 insertions(+), 64 deletions(-) diff --git a/scrapling/engines/_browsers/_camoufox.py b/scrapling/engines/_browsers/_camoufox.py index ac66870..c0b2f2c 100644 --- a/scrapling/engines/_browsers/_camoufox.py +++ b/scrapling/engines/_browsers/_camoufox.py @@ -13,9 +13,8 @@ from playwright.async_api import ( Playwright as AsyncPlaywright, BrowserContext as AsyncBrowserContext, ) -from playwright._impl._errors import Error as PlaywrightError -from ._validators import validate_fetch as _validate +from ._validators import validate_fetch as _validate, CamoufoxConfig from ._base import SyncSession, AsyncSession, StealthySessionMixin from scrapling.core.utils import log from scrapling.core._types import ( @@ -293,6 +292,7 @@ class StealthySession(StealthySessionMixin, SyncSession): ("solve_cloudflare", solve_cloudflare, self.solve_cloudflare), ("selector_config", selector_config, self.selector_config), ], + CamoufoxConfig, _UNSET, ) @@ -574,6 +574,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession): ("solve_cloudflare", solve_cloudflare, self.solve_cloudflare), ("selector_config", selector_config, self.selector_config), ], + CamoufoxConfig, _UNSET, ) diff --git a/scrapling/engines/_browsers/_controllers.py b/scrapling/engines/_browsers/_controllers.py index d8f841b..bcfd0ac 100644 --- a/scrapling/engines/_browsers/_controllers.py +++ b/scrapling/engines/_browsers/_controllers.py @@ -14,7 +14,7 @@ from patchright.async_api import async_playwright as async_patchright from scrapling.core.utils import log from ._base import SyncSession, AsyncSession, DynamicSessionMixin -from ._validators import validate_fetch as _validate +from ._validators import validate_fetch as _validate, PlaywrightConfig from scrapling.core._types import ( Any, Dict, @@ -225,6 +225,7 @@ class DynamicSession(DynamicSessionMixin, SyncSession): ("load_dom", load_dom, self.load_dom), ("selector_config", selector_config, self.selector_config), ], + PlaywrightConfig, _UNSET, ) @@ -442,6 +443,7 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession): ("load_dom", load_dom, self.load_dom), ("selector_config", selector_config, self.selector_config), ], + PlaywrightConfig, _UNSET, ) diff --git a/scrapling/engines/_browsers/_validators.py b/scrapling/engines/_browsers/_validators.py index 831aacb..edeaa2f 100644 --- a/scrapling/engines/_browsers/_validators.py +++ b/scrapling/engines/_browsers/_validators.py @@ -1,7 +1,8 @@ from pathlib import Path from typing import Annotated -from dataclasses import dataclass +from functools import lru_cache from urllib.parse import urlparse +from dataclasses import dataclass, fields from msgspec import Struct, Meta, convert, ValidationError @@ -19,18 +20,20 @@ from scrapling.engines.toolbelt.navigation import construct_proxy_dict # Custom validators for msgspec -def _validate_file_path(value: str): +@lru_cache(8) +def _is_invalid_file_path(value: str) -> bool | str: """Fast file path validation""" path = Path(value) if not path.exists(): - raise ValueError(f"Init script path not found: {value}") + return f"Init script path not found: {value}" if not path.is_file(): - raise ValueError(f"Init script is not a file: {value}") + return f"Init script is not a file: {value}" if not path.is_absolute(): - raise ValueError(f"Init script is not a absolute path: {value}") + return f"Init script is not a absolute path: {value}" + return False -def _validate_addon_path(value: str): +def _validate_addon_path(value: str) -> None: """Fast addon path validation""" path = Path(value) if not path.exists(): @@ -39,22 +42,16 @@ def _validate_addon_path(value: str): raise ValueError(f"Addon path must be a directory of the extracted addon: {value}") -def _validate_cdp_url(cdp_url: str): +@lru_cache(2) +def _is_invalid_cdp_url(cdp_url: str) -> bool | str: """Fast CDP URL validation""" - try: - # Check the scheme - if not cdp_url.startswith(("ws://", "wss://")): - raise ValueError("CDP URL must use 'ws://' or 'wss://' scheme") + if not cdp_url.startswith(("ws://", "wss://")): + return "CDP URL must use 'ws://' or 'wss://' scheme" - # Validate hostname and port - if not urlparse(cdp_url).netloc: - raise ValueError("Invalid hostname for the CDP URL") - - except AttributeError as e: - raise ValueError(f"Malformed CDP URL: {cdp_url}: {str(e)}") - - except Exception as e: - raise ValueError(f"Invalid CDP URL '{cdp_url}': {str(e)}") + netloc = urlparse(cdp_url).netloc + if not netloc: + return "Invalid hostname for the CDP URL" + return False # Type aliases for cleaner annotations @@ -62,7 +59,7 @@ PagesCount = Annotated[int, Meta(ge=1, le=50)] Seconds = Annotated[int, float, Meta(ge=0)] -class PlaywrightConfig(Struct, kw_only=True, frozen=False): +class PlaywrightConfig(Struct, kw_only=True, frozen=False, weakref=True): """Configuration struct for validation""" max_pages: PagesCount = 1 @@ -99,7 +96,9 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False): if self.proxy: self.proxy = construct_proxy_dict(self.proxy, as_tuple=True) if self.cdp_url: - _validate_cdp_url(self.cdp_url) + cdp_msg = _is_invalid_cdp_url(self.cdp_url) + if cdp_msg: + raise ValueError(cdp_msg) if not self.cookies: self.cookies = [] @@ -111,10 +110,12 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False): self.additional_args = {} if self.init_script is not None: - _validate_file_path(self.init_script) + validation_msg = _is_invalid_file_path(self.init_script) + if validation_msg: + raise ValueError(validation_msg) -class CamoufoxConfig(Struct, kw_only=True, frozen=False): +class CamoufoxConfig(Struct, kw_only=True, frozen=False, weakref=True): """Configuration struct for validation""" max_pages: PagesCount = 1 @@ -152,14 +153,16 @@ class CamoufoxConfig(Struct, kw_only=True, frozen=False): if self.proxy: self.proxy = construct_proxy_dict(self.proxy, as_tuple=True) - if self.addons and isinstance(self.addons, list): + if self.addons: for addon in self.addons: _validate_addon_path(addon) else: self.addons = [] if self.init_script is not None: - _validate_file_path(self.init_script) + validation_msg = _is_invalid_file_path(self.init_script) + if validation_msg: + raise ValueError(validation_msg) if not self.cookies: self.cookies = [] @@ -172,27 +175,6 @@ class CamoufoxConfig(Struct, kw_only=True, frozen=False): self.additional_args = {} -# Code parts to validate `fetch` in the least possible numbers of lines overall -class FetchConfig(Struct, kw_only=True): - """Configuration struct for `fetch` calls validation""" - - google_search: bool = True - timeout: Seconds = 30000 - wait: Seconds = 0 - page_action: Optional[Callable] = None - extra_headers: Optional[Dict[str, str]] = None - disable_resources: bool = False - wait_selector: Optional[str] = None - wait_selector_state: SelectorWaitStates = "attached" - network_idle: bool = False - load_dom: bool = True - solve_cloudflare: bool = False - selector_config: Dict = {} - - def to_dict(self): - return {f: getattr(self, f) for f in self.__struct_fields__} - - @dataclass class _fetch_params: """A dataclass of all parameters used by `fetch` calls""" @@ -211,7 +193,9 @@ class _fetch_params: selector_config: Dict -def validate_fetch(params: List[Tuple], sentinel=None) -> _fetch_params: +def validate_fetch( + params: List[Tuple], model: type[PlaywrightConfig] | type[CamoufoxConfig], sentinel=None +) -> _fetch_params: result = {} overrides = {} @@ -222,16 +206,44 @@ def validate_fetch(params: List[Tuple], sentinel=None) -> _fetch_params: result[arg] = session_value if overrides: - overrides = validate(overrides, FetchConfig).to_dict() - overrides.update(result) - return _fetch_params(**overrides) + validated_config = validate(overrides, model) + # Extract only the fields that _fetch_params needs from validated_config + validated_dict = { + f.name: getattr(validated_config, f.name) + for f in fields(_fetch_params) + if hasattr(validated_config, f.name) + } + # solve_cloudflare defaults to False for models that don't have it (PlaywrightConfig) + validated_dict.setdefault("solve_cloudflare", False) - if not result.get("solve_cloudflare"): - result["solve_cloudflare"] = False + validated_dict.update(result) + return _fetch_params(**validated_dict) + + result.setdefault("solve_cloudflare", False) return _fetch_params(**result) +# Cache default values for each model to reduce validation overhead +models_default_values = {} + +for _model in (CamoufoxConfig, PlaywrightConfig): + _defaults = {} + if hasattr(_model, "__struct_defaults__") and hasattr(_model, "__struct_fields__"): + for field_name, default_value in zip(_model.__struct_fields__, _model.__struct_defaults__): # type: ignore + # Skip factory defaults - these are msgspec._core.Factory instances + if type(default_value).__name__ != "Factory": + _defaults[field_name] = default_value + + models_default_values[_model.__name__] = _defaults.copy() + + +def _filter_defaults(params: Dict, model: str) -> Dict: + """Filter out parameters that match their default values to reduce validation overhead.""" + defaults = models_default_values[model] + return {k: v for k, v in params.items() if k not in defaults or v != defaults[k]} + + @overload def validate(params: Dict, model: type[PlaywrightConfig]) -> PlaywrightConfig: ... @@ -240,14 +252,10 @@ def validate(params: Dict, model: type[PlaywrightConfig]) -> PlaywrightConfig: . def validate(params: Dict, model: type[CamoufoxConfig]) -> CamoufoxConfig: ... -@overload -def validate(params: Dict, model: type[FetchConfig]) -> FetchConfig: ... - - -def validate( - params: Dict, model: type[PlaywrightConfig] | type[CamoufoxConfig] | type[FetchConfig] -) -> PlaywrightConfig | CamoufoxConfig | FetchConfig: +def validate(params: Dict, model: type[PlaywrightConfig] | type[CamoufoxConfig]) -> PlaywrightConfig | CamoufoxConfig: try: - return convert(params, model) + # Filter out params with the default values (no need to validate them) to speed up validation + filtered = _filter_defaults(params, model.__name__) + return convert(filtered, model) except ValidationError as e: raise TypeError(f"Invalid argument type: {e}") from e