From 6a73f9dcd78ab81be81b01f3d09cf29602d777fe Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Thu, 8 Jan 2026 00:52:06 +0200 Subject: [PATCH] refactor: internal API changes to be easily used as indicators for spiders --- scrapling/engines/_browsers/_base.py | 12 ++++---- scrapling/engines/_browsers/_controllers.py | 9 ++++-- scrapling/engines/_browsers/_stealth.py | 5 ++- scrapling/engines/static.py | 34 ++++++++++++--------- 4 files changed, 34 insertions(+), 26 deletions(-) diff --git a/scrapling/engines/_browsers/_base.py b/scrapling/engines/_browsers/_base.py index bb14b5f..cbbdc24 100644 --- a/scrapling/engines/_browsers/_base.py +++ b/scrapling/engines/_browsers/_base.py @@ -37,14 +37,14 @@ class SyncSession: self._max_wait_for_page = 60 self.playwright: Playwright | Any = None self.context: BrowserContext | Any = None - self._closed = False + self._is_alive = False def start(self): pass def close(self): # pragma: no cover """Close all resources""" - if self._closed: + if not self._is_alive: return if self.context: @@ -55,7 +55,7 @@ class SyncSession: self.playwright.stop() self.playwright = None # pyright: ignore - self._closed = True + self._is_alive = False def __enter__(self): self.start() @@ -137,7 +137,7 @@ class AsyncSession: self._max_wait_for_page = 60 self.playwright: AsyncPlaywright | Any = None self.context: AsyncBrowserContext | Any = None - self._closed = False + self._is_alive = False self._lock = Lock() async def start(self): @@ -145,7 +145,7 @@ class AsyncSession: async def close(self): """Close all resources""" - if self._closed: # pragma: no cover + if not self._is_alive: # pragma: no cover return if self.context: @@ -156,7 +156,7 @@ class AsyncSession: await self.playwright.stop() self.playwright = None # pyright: ignore - self._closed = True + self._is_alive = False async def __aenter__(self): await self.start() diff --git a/scrapling/engines/_browsers/_controllers.py b/scrapling/engines/_browsers/_controllers.py index ff21438..285bb32 100644 --- a/scrapling/engines/_browsers/_controllers.py +++ b/scrapling/engines/_browsers/_controllers.py @@ -31,7 +31,6 @@ class DynamicSession(SyncSession, DynamicSessionMixin): "_max_wait_for_page", "playwright", "context", - "_closed", ) def __init__(self, **kwargs: Unpack[PlaywrightSession]): @@ -82,6 +81,8 @@ class DynamicSession(SyncSession, DynamicSessionMixin): if self._config.cookies: # pragma: no cover self.context.add_cookies(self._config.cookies) + + self._is_alive = True else: raise RuntimeError("Session has been already started") @@ -104,7 +105,7 @@ class DynamicSession(SyncSession, DynamicSessionMixin): :return: A `Response` object. """ params = _validate(kwargs, self, PlaywrightConfig) - if self._closed: # pragma: no cover + if not self._is_alive: # pragma: no cover raise RuntimeError("Context manager has been closed") referer = ( @@ -211,6 +212,8 @@ class AsyncDynamicSession(AsyncSession, DynamicSessionMixin): if self._config.cookies: await self.context.add_cookies(self._config.cookies) # pyright: ignore + + self._is_alive = True else: raise RuntimeError("Session has been already started") @@ -234,7 +237,7 @@ class AsyncDynamicSession(AsyncSession, DynamicSessionMixin): """ params = _validate(kwargs, self, PlaywrightConfig) - if self._closed: # pragma: no cover + if not self._is_alive: # pragma: no cover raise RuntimeError("Context manager has been closed") referer = ( diff --git a/scrapling/engines/_browsers/_stealth.py b/scrapling/engines/_browsers/_stealth.py index 912d38e..1cce675 100644 --- a/scrapling/engines/_browsers/_stealth.py +++ b/scrapling/engines/_browsers/_stealth.py @@ -39,7 +39,6 @@ class StealthySession(SyncSession, StealthySessionMixin): "_max_wait_for_page", "playwright", "context", - "_closed", ) def __init__(self, **kwargs: Unpack[StealthSession]): @@ -191,7 +190,7 @@ class StealthySession(SyncSession, StealthySessionMixin): :return: A `Response` object. """ params = _validate(kwargs, self, StealthConfig) - if self._closed: # pragma: no cover + if not self._is_alive: # pragma: no cover raise RuntimeError("Context manager has been closed") referer = ( @@ -404,7 +403,7 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin): """ params = _validate(kwargs, self, StealthConfig) - if self._closed: # pragma: no cover + if not self._is_alive: # pragma: no cover raise RuntimeError("Context manager has been closed") referer = ( diff --git a/scrapling/engines/static.py b/scrapling/engines/static.py index c7afa79..2386820 100644 --- a/scrapling/engines/static.py +++ b/scrapling/engines/static.py @@ -62,6 +62,7 @@ class _ConfigurationLogic(ABC): "_default_cert", "_default_http3", "selector_config", + "_is_alive", ) def __init__(self, **kwargs: Unpack[RequestsSession]): @@ -80,6 +81,7 @@ class _ConfigurationLogic(ABC): self._default_cert = kwargs.get("cert") or None self._default_http3 = kwargs.get("http3", False) self.selector_config = kwargs.get("selector_config") or {} + self._is_alive = False @staticmethod def _get_param(kwargs: Dict, key: str, default: Any) -> Any: @@ -183,10 +185,11 @@ class _SyncSessionLogic(_ConfigurationLogic): def __enter__(self): """Creates and returns a new synchronous Fetcher Session""" - if self._curl_session: + if self._is_alive: raise RuntimeError("This FetcherSession instance already has an active synchronous session.") self._curl_session = CurlSession() + self._is_alive = True return self def __exit__(self, exc_type, exc_val, exc_tb): @@ -201,7 +204,9 @@ class _SyncSessionLogic(_ConfigurationLogic): self._curl_session.close() self._curl_session = None - def __make_request(self, method: SUPPORTED_HTTP_METHODS, stealth: Optional[bool] = None, **kwargs) -> Response: + self._is_alive = False + + def _make_request(self, method: SUPPORTED_HTTP_METHODS, stealth: Optional[bool] = None, **kwargs) -> Response: """ Perform an HTTP request using the configured session. """ @@ -267,7 +272,7 @@ class _SyncSessionLogic(_ConfigurationLogic): :return: A `Response` object. """ stealthy_headers = kwargs.pop("stealthy_headers", None) - return self.__make_request("GET", stealth=stealthy_headers, url=url, **kwargs) + return self._make_request("GET", stealth=stealthy_headers, url=url, **kwargs) def post(self, url: str, **kwargs: Unpack[DataRequestParams]) -> Response: """ @@ -299,7 +304,7 @@ class _SyncSessionLogic(_ConfigurationLogic): :return: A `Response` object. """ stealthy_headers = kwargs.pop("stealthy_headers", None) - return self.__make_request("POST", stealth=stealthy_headers, url=url, **kwargs) + return self._make_request("POST", stealth=stealthy_headers, url=url, **kwargs) def put(self, url: str, **kwargs: Unpack[DataRequestParams]) -> Response: """ @@ -331,7 +336,7 @@ class _SyncSessionLogic(_ConfigurationLogic): :return: A `Response` object. """ stealthy_headers = kwargs.pop("stealthy_headers", None) - return self.__make_request("PUT", stealth=stealthy_headers, url=url, **kwargs) + return self._make_request("PUT", stealth=stealthy_headers, url=url, **kwargs) def delete(self, url: str, **kwargs: Unpack[DataRequestParams]) -> Response: """ @@ -365,7 +370,7 @@ class _SyncSessionLogic(_ConfigurationLogic): # Careful of sending a body in a DELETE request, it might cause some websites to reject the request as per https://www.rfc-editor.org/rfc/rfc7231#section-4.3.5, # But some websites accept it, it depends on the implementation used. stealthy_headers = kwargs.pop("stealthy_headers", None) - return self.__make_request("DELETE", stealth=stealthy_headers, url=url, **kwargs) + return self._make_request("DELETE", stealth=stealthy_headers, url=url, **kwargs) class _ASyncSessionLogic(_ConfigurationLogic): @@ -377,10 +382,11 @@ class _ASyncSessionLogic(_ConfigurationLogic): async def __aenter__(self): # pragma: no cover """Creates and returns a new asynchronous Session.""" - if self._async_curl_session: + if self._is_alive: raise RuntimeError("This FetcherSession instance already has an active asynchronous session.") self._async_curl_session = AsyncCurlSession() + self._is_alive = True return self async def __aexit__(self, exc_type, exc_val, exc_tb): @@ -395,9 +401,9 @@ class _ASyncSessionLogic(_ConfigurationLogic): await self._async_curl_session.close() self._async_curl_session = None - async def __make_request( - self, method: SUPPORTED_HTTP_METHODS, stealth: Optional[bool] = None, **kwargs - ) -> Response: + self._is_alive = False + + async def _make_request(self, method: SUPPORTED_HTTP_METHODS, stealth: Optional[bool] = None, **kwargs) -> Response: """ Perform an HTTP request using the configured session. """ @@ -465,7 +471,7 @@ class _ASyncSessionLogic(_ConfigurationLogic): :return: A `Response` object. """ stealthy_headers = kwargs.pop("stealthy_headers", None) - return self.__make_request("GET", stealth=stealthy_headers, url=url, **kwargs) + return self._make_request("GET", stealth=stealthy_headers, url=url, **kwargs) def post(self, url: str, **kwargs: Unpack[DataRequestParams]) -> Awaitable[Response]: """ @@ -497,7 +503,7 @@ class _ASyncSessionLogic(_ConfigurationLogic): :return: A `Response` object. """ stealthy_headers = kwargs.pop("stealthy_headers", None) - return self.__make_request("POST", stealth=stealthy_headers, url=url, **kwargs) + return self._make_request("POST", stealth=stealthy_headers, url=url, **kwargs) def put(self, url: str, **kwargs: Unpack[DataRequestParams]) -> Awaitable[Response]: """ @@ -529,7 +535,7 @@ class _ASyncSessionLogic(_ConfigurationLogic): :return: A `Response` object. """ stealthy_headers = kwargs.pop("stealthy_headers", None) - return self.__make_request("PUT", stealth=stealthy_headers, url=url, **kwargs) + return self._make_request("PUT", stealth=stealthy_headers, url=url, **kwargs) def delete(self, url: str, **kwargs: Unpack[DataRequestParams]) -> Awaitable[Response]: """ @@ -563,7 +569,7 @@ class _ASyncSessionLogic(_ConfigurationLogic): # Careful of sending a body in a DELETE request, it might cause some websites to reject the request as per https://www.rfc-editor.org/rfc/rfc7231#section-4.3.5, # But some websites accept it, it depends on the implementation used. stealthy_headers = kwargs.pop("stealthy_headers", None) - return self.__make_request("DELETE", stealth=stealthy_headers, url=url, **kwargs) + return self._make_request("DELETE", stealth=stealthy_headers, url=url, **kwargs) class FetcherSession: