From a9d05cc0ef495b329618be2824554e69856e5b27 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Fri, 19 Sep 2025 04:49:30 +0300 Subject: [PATCH] style: Removing dead code/docstrings and correcting type hints --- scrapling/core/custom_types.py | 4 ++-- scrapling/engines/_browsers/_page.py | 8 +------- scrapling/engines/static.py | 6 ++---- scrapling/engines/toolbelt/navigation.py | 2 +- scrapling/parser.py | 4 ++-- 5 files changed, 8 insertions(+), 16 deletions(-) diff --git a/scrapling/core/custom_types.py b/scrapling/core/custom_types.py index eb7fa34..e9ece7d 100644 --- a/scrapling/core/custom_types.py +++ b/scrapling/core/custom_types.py @@ -145,7 +145,7 @@ class TextHandler(str): clean_match: bool = False, case_sensitive: bool = True, check_match: Literal[False] = False, - ) -> "TextHandlers[TextHandler]": ... + ) -> "TextHandlers": ... def re( self, @@ -241,7 +241,7 @@ class TextHandlers(List[TextHandler]): replace_entities: bool = True, clean_match: bool = False, case_sensitive: bool = True, - ) -> "TextHandlers[TextHandler]": + ) -> "TextHandlers": """Call the ``.re()`` method for each element in this list and return their results flattened as TextHandlers. diff --git a/scrapling/engines/_browsers/_page.py b/scrapling/engines/_browsers/_page.py index 8c80944..821fbbb 100644 --- a/scrapling/engines/_browsers/_page.py +++ b/scrapling/engines/_browsers/_page.py @@ -6,7 +6,7 @@ from playwright.async_api import Page as AsyncPage from scrapling.core._types import Optional, List, Literal -PageState = Literal["finished", "ready", "busy", "error"] # States that a page can be in +PageState = Literal["ready", "busy", "error"] # States that a page can be in @dataclass @@ -62,12 +62,6 @@ class PagePool: """Get the total number of pages""" return len(self.pages) - @property - def finished_count(self) -> int: - """Get the number of finished pages""" - with self._lock: - return sum(1 for p in self.pages if p.state == "finished") - @property def busy_count(self) -> int: """Get the number of busy pages""" diff --git a/scrapling/engines/static.py b/scrapling/engines/static.py index 3f6cb79..6e5c0de 100644 --- a/scrapling/engines/static.py +++ b/scrapling/engines/static.py @@ -94,8 +94,8 @@ class FetcherSession: self.default_http3 = http3 self.selector_config = selector_config or {} - self._curl_session: Optional[CurlSession] = None - self._async_curl_session: Optional[AsyncCurlSession] = None + self._curl_session: Optional[CurlSession] | bool = None + self._async_curl_session: Optional[AsyncCurlSession] | bool = None def _merge_request_args(self, **kwargs) -> Dict[str, Any]: """Merge request-specific arguments with default session arguments.""" @@ -239,7 +239,6 @@ class FetcherSession: Perform an HTTP request using the configured session. :param method: HTTP method to be used, supported methods are ["GET", "POST", "PUT", "DELETE"] - :param url: Target URL for the request. :param request_args: Arguments to be passed to the session's `request()` method. :param max_retries: Maximum number of retries for the request. :param retry_delay: Number of seconds to wait between retries. @@ -280,7 +279,6 @@ class FetcherSession: Perform an HTTP request using the configured session. :param method: HTTP method to be used, supported methods are ["GET", "POST", "PUT", "DELETE"] - :param url: Target URL for the request. :param request_args: Arguments to be passed to the session's `request()` method. :param max_retries: Maximum number of retries for the request. :param retry_delay: Number of seconds to wait between retries. diff --git a/scrapling/engines/toolbelt/navigation.py b/scrapling/engines/toolbelt/navigation.py index f2f445c..ea5991b 100644 --- a/scrapling/engines/toolbelt/navigation.py +++ b/scrapling/engines/toolbelt/navigation.py @@ -4,7 +4,7 @@ Functions related to files and URLs from pathlib import Path from functools import lru_cache -from urllib.parse import urlencode, urlparse +from urllib.parse import urlparse from playwright.async_api import Route as async_Route from msgspec import Struct, structs, convert, ValidationError diff --git a/scrapling/parser.py b/scrapling/parser.py index 6260988..f0b9e54 100644 --- a/scrapling/parser.py +++ b/scrapling/parser.py @@ -239,7 +239,7 @@ class Selector(SelectorsGeneration): ) def __handle_element( - self, element: HtmlElement | _ElementUnicodeResult + self, element: Optional[HtmlElement | _ElementUnicodeResult] ) -> Optional[Union[TextHandler, "Selector"]]: """Used internally in all functions to convert a single element to type (Selector|TextHandler) when possible""" if element is None: @@ -345,7 +345,7 @@ class Selector(SelectorsGeneration): return TextHandler(content) @property - def body(self): + def body(self) -> str | bytes: """Return the raw body of the current `Selector` without any processing. Useful for binary and non-HTML requests.""" return self._raw_body