From c487faf4fdd32fb7576d22286a9654c7040ffa09 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Sun, 23 Nov 2025 21:50:54 +0200 Subject: [PATCH] pref(requests): Optimizing memory by specifiying slots --- scrapling/engines/_browsers/_validators.py | 10 ++--- scrapling/engines/static.py | 45 ++++++++++++++++++++++ 2 files changed, 50 insertions(+), 5 deletions(-) diff --git a/scrapling/engines/_browsers/_validators.py b/scrapling/engines/_browsers/_validators.py index edeaa2f..2b5ba0a 100644 --- a/scrapling/engines/_browsers/_validators.py +++ b/scrapling/engines/_browsers/_validators.py @@ -21,7 +21,7 @@ from scrapling.engines.toolbelt.navigation import construct_proxy_dict # Custom validators for msgspec @lru_cache(8) -def _is_invalid_file_path(value: str) -> bool | str: +def _is_invalid_file_path(value: str) -> bool | str: # pragma: no cover """Fast file path validation""" path = Path(value) if not path.exists(): @@ -33,7 +33,7 @@ def _is_invalid_file_path(value: str) -> bool | str: return False -def _validate_addon_path(value: str) -> None: +def _validate_addon_path(value: str) -> None: # pragma: no cover """Fast addon path validation""" path = Path(value) if not path.exists(): @@ -49,7 +49,7 @@ def _is_invalid_cdp_url(cdp_url: str) -> bool | str: return "CDP URL must use 'ws://' or 'wss://' scheme" netloc = urlparse(cdp_url).netloc - if not netloc: + if not netloc: # pragma: no cover return "Invalid hostname for the CDP URL" return False @@ -89,7 +89,7 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False, weakref=True): selector_config: Optional[Dict] = {} additional_args: Optional[Dict] = {} - def __post_init__(self): + def __post_init__(self): # pragma: no cover """Custom validation after msgspec validation""" if self.page_action and not callable(self.page_action): raise TypeError(f"page_action must be callable, got {type(self.page_action).__name__}") @@ -195,7 +195,7 @@ class _fetch_params: def validate_fetch( params: List[Tuple], model: type[PlaywrightConfig] | type[CamoufoxConfig], sentinel=None -) -> _fetch_params: +) -> _fetch_params: # pragma: no cover result = {} overrides = {} diff --git a/scrapling/engines/static.py b/scrapling/engines/static.py index 7251209..796df13 100644 --- a/scrapling/engines/static.py +++ b/scrapling/engines/static.py @@ -46,6 +46,24 @@ def _select_random_browser(impersonate: ImpersonateType) -> Optional[BrowserType class _ConfigurationLogic(ABC): # Core Logic Handler (Internal Engine) + __slots__ = ( + "_default_impersonate", + "_stealth", + "_default_proxies", + "_default_proxy", + "_default_proxy_auth", + "_default_timeout", + "_default_headers", + "_default_retries", + "_default_retry_delay", + "_default_follow_redirects", + "_default_max_redirects", + "_default_verify", + "_default_cert", + "_default_http3", + "selector_config", + ) + def __init__(self, **kwargs: Unpack[RequestsSession]): self._default_impersonate = kwargs.get("impersonate", "chrome") self._stealth = kwargs.get("stealthy_headers", True) @@ -157,6 +175,8 @@ class _ConfigurationLogic(ABC): class _SyncSessionLogic(_ConfigurationLogic): + __slots__ = ("_curl_session",) + def __init__(self, **kwargs: Unpack[RequestsSession]): super().__init__(**kwargs) self._curl_session: Optional[CurlSession] = None @@ -343,6 +363,8 @@ class _SyncSessionLogic(_ConfigurationLogic): class _ASyncSessionLogic(_ConfigurationLogic): + __slots__ = ("_async_curl_session",) + def __init__(self, **kwargs: Unpack[RequestsSession]): super().__init__(**kwargs) self._async_curl_session: Optional[AsyncCurlSession] = None @@ -549,6 +571,25 @@ class FetcherSession: same manager instance while a session is already active is disallowed. """ + __slots__ = ( + "_default_impersonate", + "_stealth", + "_default_proxies", + "_default_proxy", + "_default_proxy_auth", + "_default_timeout", + "_default_headers", + "_default_retries", + "_default_retry_delay", + "_default_follow_redirects", + "_default_max_redirects", + "_default_verify", + "_default_cert", + "_default_http3", + "selector_config", + "_client", + ) + def __init__( self, impersonate: ImpersonateType = "chrome", @@ -640,6 +681,8 @@ class FetcherSession: class FetcherClient(_SyncSessionLogic): + __slots__ = ("__enter__", "__exit__") + def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.__enter__: Any = None @@ -648,6 +691,8 @@ class FetcherClient(_SyncSessionLogic): class AsyncFetcherClient(_ASyncSessionLogic): + __slots__ = ("__aenter__", "__aexit__") + def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.__aenter__: Any = None