From 9bcb9e9d9308a6be438922703f32bda7dd840adc Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Tue, 29 Jul 2025 23:43:06 +0300 Subject: [PATCH] style: General type hints fixes and imports optimizing --- ruff.toml | 2 +- scrapling/core/_types.py | 1 + scrapling/core/custom_types.py | 17 ++++++++------ scrapling/core/storage.py | 6 ++--- scrapling/core/utils.py | 12 +++++----- scrapling/engines/_browsers/_validators.py | 2 +- scrapling/parser.py | 26 +++++++++++----------- 7 files changed, 35 insertions(+), 31 deletions(-) diff --git a/ruff.toml b/ruff.toml index 04dadf0..a579697 100644 --- a/ruff.toml +++ b/ruff.toml @@ -15,7 +15,7 @@ target-version = "py39" [lint] select = ["E", "F", "W"] -ignore = ["E501", "F401"] +ignore = ["E501", "F401", "F811"] [format] # Like Black, use double quotes for strings. diff --git a/scrapling/core/_types.py b/scrapling/core/_types.py index 2ed107f..a41e077 100644 --- a/scrapling/core/_types.py +++ b/scrapling/core/_types.py @@ -4,6 +4,7 @@ Type definitions for type checking purposes. from typing import ( TYPE_CHECKING, + overload, Any, Callable, Dict, diff --git a/scrapling/core/custom_types.py b/scrapling/core/custom_types.py index 4afc926..52dfe2e 100644 --- a/scrapling/core/custom_types.py +++ b/scrapling/core/custom_types.py @@ -7,14 +7,15 @@ from orjson import dumps, loads from scrapling.core._types import ( Dict, - Iterable, List, - Literal, - Optional, - Pattern, - SupportsIndex, - TypeVar, Union, + TypeVar, + Literal, + Pattern, + Iterable, + Optional, + Generator, + SupportsIndex, ) from scrapling.core.utils import _is_iterable, flatten from scrapling.core._html_utils import _replace_entities @@ -341,7 +342,9 @@ class AttributesHandler(Mapping[str, _TextHandlerType]): """Acts like the standard dictionary `.get()` method""" return self._data.get(key, default) - def search_values(self, keyword, partial=False): + def search_values( + self, keyword: str, partial: bool = False + ) -> Generator["AttributesHandler", None, None]: """Search current attributes by values and return a dictionary of each matching item :param keyword: The keyword to search for in the attribute values :param partial: If True, the function will search if keyword in each value instead of perfect match diff --git a/scrapling/core/storage.py b/scrapling/core/storage.py index 5821707..03ca612 100644 --- a/scrapling/core/storage.py +++ b/scrapling/core/storage.py @@ -9,7 +9,7 @@ from orjson import dumps, loads from tldextract import extract as tld from scrapling.core.utils import _StorageTools, log -from scrapling.core._types import Dict, Optional, Union +from scrapling.core._types import Dict, Optional, Union, Any class StorageSystemMixin(ABC): @@ -106,7 +106,7 @@ class SQLiteStorageSystem(StorageSystemMixin): """) self.connection.commit() - def save(self, element: HtmlElement, identifier: str): + def save(self, element: HtmlElement, identifier: str) -> None: """Saves the elements unique properties to the storage for retrieval and relocation later :param element: The element itself which we want to save to storage. @@ -126,7 +126,7 @@ class SQLiteStorageSystem(StorageSystemMixin): self.cursor.fetchall() self.connection.commit() - def retrieve(self, identifier: str) -> Optional[Dict]: + def retrieve(self, identifier: str) -> Optional[Dict[str, Any]]: """Using the identifier, we search the storage and return the unique properties of the element :param identifier: This is the identifier that will be used to retrieve the element from the storage. See diff --git a/scrapling/core/utils.py b/scrapling/core/utils.py index e33c914..0219cb0 100644 --- a/scrapling/core/utils.py +++ b/scrapling/core/utils.py @@ -5,7 +5,7 @@ from itertools import chain import orjson from lxml import html -from scrapling.core._types import Any, Dict, Iterable, Union +from scrapling.core._types import Any, Dict, Iterable, Union, List # Using cache on top of a class is a brilliant way to achieve a Singleton design pattern without much code from functools import lru_cache # isort:skip @@ -41,8 +41,8 @@ def setup_logger(): log = setup_logger() -def is_jsonable(content: Union[bytes, str]) -> bool: - if type(content) is bytes: +def is_jsonable(content: bytes | str) -> bool: + if isinstance(content, bytes): content = content.decode() try: @@ -52,14 +52,14 @@ def is_jsonable(content: Union[bytes, str]) -> bool: return False -def flatten(lst: Iterable): +def flatten(lst: Iterable[Any]) -> List[Any]: return list(chain.from_iterable(lst)) -def _is_iterable(s: Any): +def _is_iterable(obj: Any) -> bool: # This will be used only in regex functions to make sure it's iterable but not string/bytes return isinstance( - s, + obj, ( list, tuple, diff --git a/scrapling/engines/_browsers/_validators.py b/scrapling/engines/_browsers/_validators.py index 60a8dcf..e6557ff 100644 --- a/scrapling/engines/_browsers/_validators.py +++ b/scrapling/engines/_browsers/_validators.py @@ -82,7 +82,7 @@ class CamoufoxConfig(Struct, kw_only=True, frozen=False): """Configuration struct for validation""" max_pages: int = 1 - headless: Union[bool] = True # noqa: F821 + headless: bool = True # noqa: F821 block_images: bool = False disable_resources: bool = False block_webrtc: bool = False diff --git a/scrapling/parser.py b/scrapling/parser.py index daddc9d..05d2384 100644 --- a/scrapling/parser.py +++ b/scrapling/parser.py @@ -1,7 +1,6 @@ -import inspect import os import re -import typing +from inspect import signature from difflib import SequenceMatcher from urllib.parse import urljoin @@ -18,16 +17,17 @@ from lxml.etree import ( from scrapling.core._types import ( Any, - Callable, Dict, - Generator, - Iterable, List, - Optional, - Pattern, - SupportsIndex, Tuple, Union, + Pattern, + Callable, + Optional, + Iterable, + overload, + Generator, + SupportsIndex, ) from scrapling.core.custom_types import AttributesHandler, TextHandler, TextHandlers from scrapling.core.mixins import SelectorsGeneration @@ -248,7 +248,7 @@ class Selector(SelectorsGeneration): def __handle_elements( self, result: List[Union[HtmlElement, _ElementUnicodeResult]] - ) -> Union["Selectors", "TextHandlers", List]: + ) -> Union["Selectors", "TextHandlers"]: """Used internally in all functions to convert results to type (Selectors|TextHandlers) in bulk when possible""" if not len( result @@ -761,7 +761,7 @@ class Selector(SelectorsGeneration): patterns.add(arg) elif callable(arg): - if len(inspect.signature(arg).parameters) > 0: + if len(signature(arg).parameters) > 0: functions.append(arg) else: raise TypeError( @@ -914,7 +914,7 @@ class Selector(SelectorsGeneration): return round((score / checks) * 100, 2) @staticmethod - def __calculate_dict_diff(dict1: dict, dict2: dict) -> float: + def __calculate_dict_diff(dict1: Dict, dict2: Dict) -> float: """Used internally to calculate similarity between two dictionaries as SequenceMatcher doesn't accept dictionaries""" score = ( SequenceMatcher(None, tuple(dict1.keys()), tuple(dict2.keys())).ratio() @@ -1210,11 +1210,11 @@ class Selectors(List[Selector]): __slots__ = () - @typing.overload + @overload def __getitem__(self, pos: SupportsIndex) -> Selector: pass - @typing.overload + @overload def __getitem__(self, pos: slice) -> "Selectors": pass