perf: Improving all browser-based fetches by ~15%

scrapling/engines/_browsers/_base.py

@@ -60,11 +60,6 @@ class SyncSession:
 
         return self.page_pool.add_page(page)
 
-    @staticmethod
-    def _get_with_precedence(request_value: Any, session_value: Any, sentinel_value: object) -> Any:
-        """Get value with request-level priority over session-level"""
-        return request_value if request_value is not sentinel_value else session_value
-
     def get_pool_stats(self) -> Dict[str, int]:
         """Get statistics about the current page pool"""
         return {

scrapling/engines/_browsers/_camoufox.py

@@ -16,7 +16,7 @@ from playwright.async_api import (
 )
 from playwright._impl._errors import Error as PlaywrightError
 
-from ._validators import validate, CamoufoxConfig
+from ._validators import validate_fetch as _validate
 from ._base import SyncSession, AsyncSession, StealthySessionMixin
 from scrapling.core.utils import log
 from scrapling.core._types import (
@@ -297,23 +297,22 @@ class StealthySession(StealthySessionMixin, SyncSession):
         :param selector_config: The arguments that will be passed in the end while creating the final Selector's class.
         :return: A `Response` object.
         """
-        # Validate all resolved parameters
+        params = _validate(
-        params = validate(
+            [
-            dict(
+                ("google_search", google_search, self.google_search),
-                google_search=self._get_with_precedence(google_search, self.google_search, _UNSET),
+                ("timeout", timeout, self.timeout),
-                timeout=self._get_with_precedence(timeout, self.timeout, _UNSET),
+                ("wait", wait, self.wait),
-                wait=self._get_with_precedence(wait, self.wait, _UNSET),
+                ("page_action", page_action, self.page_action),
-                page_action=self._get_with_precedence(page_action, self.page_action, _UNSET),
+                ("extra_headers", extra_headers, self.extra_headers),
-                extra_headers=self._get_with_precedence(extra_headers, self.extra_headers, _UNSET),
+                ("disable_resources", disable_resources, self.disable_resources),
-                disable_resources=self._get_with_precedence(disable_resources, self.disable_resources, _UNSET),
+                ("wait_selector", wait_selector, self.wait_selector),
-                wait_selector=self._get_with_precedence(wait_selector, self.wait_selector, _UNSET),
+                ("wait_selector_state", wait_selector_state, self.wait_selector_state),
-                wait_selector_state=self._get_with_precedence(wait_selector_state, self.wait_selector_state, _UNSET),
+                ("network_idle", network_idle, self.network_idle),
-                network_idle=self._get_with_precedence(network_idle, self.network_idle, _UNSET),
+                ("load_dom", load_dom, self.load_dom),
-                load_dom=self._get_with_precedence(load_dom, self.load_dom, _UNSET),
+                ("solve_cloudflare", solve_cloudflare, self.solve_cloudflare),
-                solve_cloudflare=self._get_with_precedence(solve_cloudflare, self.solve_cloudflare, _UNSET),
+                ("selector_config", selector_config, self.selector_config),
-                selector_config=self._get_with_precedence(selector_config, self.selector_config, _UNSET),
+            ],
-            ),
+            _UNSET,
-            CamoufoxConfig,
         )
 
         if self._closed:  # pragma: no cover
@@ -617,22 +616,22 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):
         :param selector_config: The arguments that will be passed in the end while creating the final Selector's class.
         :return: A `Response` object.
         """
-        params = validate(
+        params = _validate(
-            dict(
+            [
-                google_search=self._get_with_precedence(google_search, self.google_search, _UNSET),
+                ("google_search", google_search, self.google_search),
-                timeout=self._get_with_precedence(timeout, self.timeout, _UNSET),
+                ("timeout", timeout, self.timeout),
-                wait=self._get_with_precedence(wait, self.wait, _UNSET),
+                ("wait", wait, self.wait),
-                page_action=self._get_with_precedence(page_action, self.page_action, _UNSET),
+                ("page_action", page_action, self.page_action),
-                extra_headers=self._get_with_precedence(extra_headers, self.extra_headers, _UNSET),
+                ("extra_headers", extra_headers, self.extra_headers),
-                disable_resources=self._get_with_precedence(disable_resources, self.disable_resources, _UNSET),
+                ("disable_resources", disable_resources, self.disable_resources),
-                wait_selector=self._get_with_precedence(wait_selector, self.wait_selector, _UNSET),
+                ("wait_selector", wait_selector, self.wait_selector),
-                wait_selector_state=self._get_with_precedence(wait_selector_state, self.wait_selector_state, _UNSET),
+                ("wait_selector_state", wait_selector_state, self.wait_selector_state),
-                network_idle=self._get_with_precedence(network_idle, self.network_idle, _UNSET),
+                ("network_idle", network_idle, self.network_idle),
-                load_dom=self._get_with_precedence(load_dom, self.load_dom, _UNSET),
+                ("load_dom", load_dom, self.load_dom),
-                solve_cloudflare=self._get_with_precedence(solve_cloudflare, self.solve_cloudflare, _UNSET),
+                ("solve_cloudflare", solve_cloudflare, self.solve_cloudflare),
-                selector_config=self._get_with_precedence(selector_config, self.selector_config, _UNSET),
+                ("selector_config", selector_config, self.selector_config),
-            ),
+            ],
-            CamoufoxConfig,
+            _UNSET,
         )
 
         if self._closed:  # pragma: no cover

scrapling/engines/_browsers/_controllers.py

@@ -16,7 +16,7 @@ from patchright.async_api import async_playwright as async_patchright
 
 from scrapling.core.utils import log
 from ._base import SyncSession, AsyncSession, DynamicSessionMixin
-from ._validators import validate, PlaywrightConfig
+from ._validators import validate_fetch as _validate
 from scrapling.core._types import (
     Dict,
     List,
@@ -224,22 +224,21 @@ class DynamicSession(DynamicSessionMixin, SyncSession):
         :param selector_config: The arguments that will be passed in the end while creating the final Selector's class.
         :return: A `Response` object.
         """
-        # Validate all resolved parameters
+        params = _validate(
-        params = validate(
+            [
-            dict(
+                ("google_search", google_search, self.google_search),
-                google_search=self._get_with_precedence(google_search, self.google_search, _UNSET),
+                ("timeout", timeout, self.timeout),
-                timeout=self._get_with_precedence(timeout, self.timeout, _UNSET),
+                ("wait", wait, self.wait),
-                wait=self._get_with_precedence(wait, self.wait, _UNSET),
+                ("page_action", page_action, self.page_action),
-                page_action=self._get_with_precedence(page_action, self.page_action, _UNSET),
+                ("extra_headers", extra_headers, self.extra_headers),
-                extra_headers=self._get_with_precedence(extra_headers, self.extra_headers, _UNSET),
+                ("disable_resources", disable_resources, self.disable_resources),
-                disable_resources=self._get_with_precedence(disable_resources, self.disable_resources, _UNSET),
+                ("wait_selector", wait_selector, self.wait_selector),
-                wait_selector=self._get_with_precedence(wait_selector, self.wait_selector, _UNSET),
+                ("wait_selector_state", wait_selector_state, self.wait_selector_state),
-                wait_selector_state=self._get_with_precedence(wait_selector_state, self.wait_selector_state, _UNSET),
+                ("network_idle", network_idle, self.network_idle),
-                network_idle=self._get_with_precedence(network_idle, self.network_idle, _UNSET),
+                ("load_dom", load_dom, self.load_dom),
-                load_dom=self._get_with_precedence(load_dom, self.load_dom, _UNSET),
+                ("selector_config", selector_config, self.selector_config),
-                selector_config=self._get_with_precedence(selector_config, self.selector_config, _UNSET),
+            ],
-            ),
+            _UNSET,
-            PlaywrightConfig,
         )
 
         if self._closed:  # pragma: no cover
@@ -471,22 +470,21 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession):
         :param selector_config: The arguments that will be passed in the end while creating the final Selector's class.
         :return: A `Response` object.
         """
-        # Validate all resolved parameters
+        params = _validate(
-        params = validate(
+            [
-            dict(
+                ("google_search", google_search, self.google_search),
-                google_search=self._get_with_precedence(google_search, self.google_search, _UNSET),
+                ("timeout", timeout, self.timeout),
-                timeout=self._get_with_precedence(timeout, self.timeout, _UNSET),
+                ("wait", wait, self.wait),
-                wait=self._get_with_precedence(wait, self.wait, _UNSET),
+                ("page_action", page_action, self.page_action),
-                page_action=self._get_with_precedence(page_action, self.page_action, _UNSET),
+                ("extra_headers", extra_headers, self.extra_headers),
-                extra_headers=self._get_with_precedence(extra_headers, self.extra_headers, _UNSET),
+                ("disable_resources", disable_resources, self.disable_resources),
-                disable_resources=self._get_with_precedence(disable_resources, self.disable_resources, _UNSET),
+                ("wait_selector", wait_selector, self.wait_selector),
-                wait_selector=self._get_with_precedence(wait_selector, self.wait_selector, _UNSET),
+                ("wait_selector_state", wait_selector_state, self.wait_selector_state),
-                wait_selector_state=self._get_with_precedence(wait_selector_state, self.wait_selector_state, _UNSET),
+                ("network_idle", network_idle, self.network_idle),
-                network_idle=self._get_with_precedence(network_idle, self.network_idle, _UNSET),
+                ("load_dom", load_dom, self.load_dom),
-                load_dom=self._get_with_precedence(load_dom, self.load_dom, _UNSET),
+                ("selector_config", selector_config, self.selector_config),
-                selector_config=self._get_with_precedence(selector_config, self.selector_config, _UNSET),
+            ],
-            ),
+            _UNSET,
-            PlaywrightConfig,
         )
 
         if self._closed:  # pragma: no cover

scrapling/engines/_browsers/_validators.py

@@ -1,72 +1,44 @@
-from msgspec import Struct, convert, ValidationError
-from urllib.parse import urlparse
 from pathlib import Path
+from typing import Annotated
+from dataclasses import dataclass
+from urllib.parse import urlparse
+
+from msgspec import Struct, Meta, convert, ValidationError
 
 from scrapling.core._types import (
-    Optional,
     Dict,
-    Callable,
     List,
+    Tuple,
+    Optional,
+    Callable,
     SelectorWaitStates,
 )
 from scrapling.engines.toolbelt.navigation import construct_proxy_dict
 
 
-class PlaywrightConfig(Struct, kw_only=True, frozen=False):
+# Custom validators for msgspec
-    """Configuration struct for validation"""
+def _validate_file_path(value: str):
+    """Fast file path validation"""
+    path = Path(value)
+    if not path.exists():
+        raise ValueError(f"Init script path not found: {value}")
+    if not path.is_file():
+        raise ValueError(f"Init script is not a file: {value}")
+    if not path.is_absolute():
+        raise ValueError(f"Init script is not a absolute path: {value}")
 
-    max_pages: int = 1
-    cdp_url: Optional[str] = None
-    headless: bool = True
-    google_search: bool = True
-    hide_canvas: bool = False
-    disable_webgl: bool = False
-    real_chrome: bool = False
-    stealth: bool = False
-    wait: int | float = 0
-    page_action: Optional[Callable] = None
-    proxy: Optional[str | Dict[str, str]] = None  # The default value for proxy in Playwright's source is `None`
-    locale: str = "en-US"
-    extra_headers: Optional[Dict[str, str]] = None
-    useragent: Optional[str] = None
-    timeout: int | float = 30000
-    init_script: Optional[str] = None
-    disable_resources: bool = False
-    wait_selector: Optional[str] = None
-    cookies: Optional[List[Dict]] = None
-    network_idle: bool = False
-    load_dom: bool = True
-    wait_selector_state: SelectorWaitStates = "attached"
-    selector_config: Optional[Dict] = None
 
-    def __post_init__(self):
+def _validate_addon_path(value: str):
-        """Custom validation after msgspec validation"""
+    """Fast addon path validation"""
-        if self.max_pages < 1 or self.max_pages > 50:
+    path = Path(value)
-            raise ValueError("max_pages must be between 1 and 50")
+    if not path.exists():
-        if self.timeout < 0:
+        raise FileNotFoundError(f"Addon path not found: {value}")
-            raise ValueError("timeout must be >= 0")
+    if not path.is_dir():
-        if self.page_action and not callable(self.page_action):
+        raise ValueError(f"Addon path must be a directory of the extracted addon: {value}")
-            raise TypeError(f"page_action must be callable, got {type(self.page_action).__name__}")
-        if self.proxy:
-            self.proxy = construct_proxy_dict(self.proxy, as_tuple=True)
-        if self.cdp_url:
-            self.__validate_cdp(self.cdp_url)
-        if not self.cookies:
-            self.cookies = []
-        if not self.selector_config:
-            self.selector_config = {}
 
-        if self.init_script is not None:
-            script_path = Path(self.init_script)
-            if not script_path.exists():
-                raise ValueError("Init script path not found")
-            elif not script_path.is_file():
-                raise ValueError("Init script is not a file")
-            elif not script_path.is_absolute():
-                raise ValueError("Init script is not a absolute path")
 
-    @staticmethod
+def _validate_cdp_url(cdp_url: str):
-    def __validate_cdp(cdp_url):
+    """Fast CDP URL validation"""
     try:
         # Check the scheme
         if not cdp_url.startswith(("ws://", "wss://")):
@@ -83,10 +55,60 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False):
         raise ValueError(f"Invalid CDP URL '{cdp_url}': {str(e)}")
 
 
+# Type aliases for cleaner annotations
+PagesCount = Annotated[int, Meta(ge=1, le=50)]
+Seconds = Annotated[int, float, Meta(ge=0)]
+
+
+class PlaywrightConfig(Struct, kw_only=True, frozen=False):
+    """Configuration struct for validation"""
+
+    max_pages: PagesCount = 1
+    cdp_url: Optional[str] = None
+    headless: bool = True
+    google_search: bool = True
+    hide_canvas: bool = False
+    disable_webgl: bool = False
+    real_chrome: bool = False
+    stealth: bool = False
+    wait: Seconds = 0
+    page_action: Optional[Callable] = None
+    proxy: Optional[str | Dict[str, str]] = None  # The default value for proxy in Playwright's source is `None`
+    locale: str = "en-US"
+    extra_headers: Optional[Dict[str, str]] = None
+    useragent: Optional[str] = None
+    timeout: Seconds = 30000
+    init_script: Optional[str] = None
+    disable_resources: bool = False
+    wait_selector: Optional[str] = None
+    cookies: Optional[List[Dict]] = None
+    network_idle: bool = False
+    load_dom: bool = True
+    wait_selector_state: SelectorWaitStates = "attached"
+    selector_config: Optional[Dict] = None
+
+    def __post_init__(self):
+        """Custom validation after msgspec validation"""
+        if self.page_action and not callable(self.page_action):
+            raise TypeError(f"page_action must be callable, got {type(self.page_action).__name__}")
+        if self.proxy:
+            self.proxy = construct_proxy_dict(self.proxy, as_tuple=True)
+        if self.cdp_url:
+            _validate_cdp_url(self.cdp_url)
+
+        if not self.cookies:
+            self.cookies = []
+        if not self.selector_config:
+            self.selector_config = {}
+
+        if self.init_script is not None:
+            _validate_file_path(self.init_script)
+
+
 class CamoufoxConfig(Struct, kw_only=True, frozen=False):
     """Configuration struct for validation"""
 
-    max_pages: int = 1
+    max_pages: PagesCount = 1
     headless: bool = True  # noqa: F821
     block_images: bool = False
     disable_resources: bool = False
@@ -96,8 +118,8 @@ class CamoufoxConfig(Struct, kw_only=True, frozen=False):
     load_dom: bool = True
     humanize: bool | float = True
     solve_cloudflare: bool = False
-    wait: int | float = 0
+    wait: Seconds = 0
-    timeout: int | float = 30000
+    timeout: Seconds = 30000
     init_script: Optional[str] = None
     page_action: Optional[Callable] = None
     wait_selector: Optional[str] = None
@@ -115,38 +137,23 @@ class CamoufoxConfig(Struct, kw_only=True, frozen=False):
 
     def __post_init__(self):
         """Custom validation after msgspec validation"""
-        if self.max_pages < 1 or self.max_pages > 50:
-            raise ValueError("max_pages must be between 1 and 50")
-        if self.timeout < 0:
-            raise ValueError("timeout must be >= 0")
         if self.page_action and not callable(self.page_action):
             raise TypeError(f"page_action must be callable, got {type(self.page_action).__name__}")
         if self.proxy:
             self.proxy = construct_proxy_dict(self.proxy, as_tuple=True)
 
-        if not self.addons:
+        if self.addons and isinstance(self.addons, list):
-            self.addons = []
-        else:
             for addon in self.addons:
-                addon_path = Path(addon)
+                _validate_addon_path(addon)
-                if not addon_path.exists():
+        else:
-                    raise FileNotFoundError(f"Addon's path not found: {addon}")
+            self.addons = []
-                elif not addon_path.is_dir():
-                    raise ValueError(
-                        f"Addon's path is not a folder, you need to pass a folder of the extracted addon: {addon}"
-                    )
 
         if self.init_script is not None:
-            script_path = Path(self.init_script)
+            _validate_file_path(self.init_script)
-            if not script_path.exists():
-                raise ValueError("Init script path not found")
-            elif not script_path.is_file():
-                raise ValueError("Init script is not a file")
-            elif not script_path.is_absolute():
-                raise ValueError("Init script is not a absolute path")
 
         if not self.cookies:
             self.cookies = []
+        # Cloudflare timeout adjustment
         if self.solve_cloudflare and self.timeout < 60_000:
             self.timeout = 60_000
         if not self.selector_config:
@@ -155,10 +162,68 @@ class CamoufoxConfig(Struct, kw_only=True, frozen=False):
             self.additional_args = {}
 
 
-def validate(params, model):
+# Code parts to validate `fetch` in the least possible numbers of lines overall
-    try:
+class FetchConfig(Struct, kw_only=True):
-        config = convert(params, model)
+    """Configuration struct for `fetch` calls validation"""
-    except ValidationError as e:
-        raise TypeError(f"Invalid argument type: {e}")
 
-    return config
+    google_search: bool = True
+    timeout: Seconds = 30000
+    wait: Seconds = 0
+    page_action: Optional[Callable] = None
+    extra_headers: Optional[Dict[str, str]] = None
+    disable_resources: bool = False
+    wait_selector: Optional[str] = None
+    wait_selector_state: SelectorWaitStates = "attached"
+    network_idle: bool = False
+    load_dom: bool = True
+    solve_cloudflare: bool = False
+    selector_config: Optional[Dict] = {}
+
+    def to_dict(self):
+        return {f: getattr(self, f) for f in self.__struct_fields__}
+
+
+@dataclass
+class _fetch_params:
+    """A dataclass of all parameters used by `fetch` calls"""
+
+    google_search: bool
+    timeout: Seconds
+    wait: Seconds
+    page_action: Optional[Callable]
+    extra_headers: Optional[Dict[str, str]]
+    disable_resources: bool
+    wait_selector: Optional[str]
+    wait_selector_state: SelectorWaitStates
+    network_idle: bool
+    load_dom: bool
+    solve_cloudflare: bool
+    selector_config: Optional[Dict]
+
+
+def validate_fetch(params: List[Tuple], sentinel=None) -> _fetch_params:
+    result = {}
+    overrides = {}
+
+    for arg, request_value, session_value in params:
+        if request_value is not sentinel:
+            overrides[arg] = request_value
+        else:
+            result[arg] = session_value
+
+    if overrides:
+        overrides = validate(overrides, FetchConfig).to_dict()
+        overrides.update(result)
+        return _fetch_params(**overrides)
+
+    if not result.get("solve_cloudflare"):
+        result["solve_cloudflare"] = False
+
+    return _fetch_params(**result)
+
+
+def validate(params: Dict, model) -> PlaywrightConfig | CamoufoxConfig | FetchConfig:
+    try:
+        return convert(params, model)
+    except ValidationError as e:
+        raise TypeError(f"Invalid argument type: {e}") from e