style: General type hints fixes and imports optimizing

ruff.toml

@@ -15,7 +15,7 @@ target-version = "py39"
 
 [lint]
 select = ["E", "F", "W"]
-ignore = ["E501", "F401"]
+ignore = ["E501", "F401", "F811"]
 
 [format]
 # Like Black, use double quotes for strings.

scrapling/core/_types.py

@@ -4,6 +4,7 @@ Type definitions for type checking purposes.
 
 from typing import (
     TYPE_CHECKING,
+    overload,
     Any,
     Callable,
     Dict,

scrapling/core/custom_types.py

@@ -7,14 +7,15 @@ from orjson import dumps, loads
 
 from scrapling.core._types import (
     Dict,
-    Iterable,
     List,
-    Literal,
-    Optional,
-    Pattern,
-    SupportsIndex,
-    TypeVar,
     Union,
+    TypeVar,
+    Literal,
+    Pattern,
+    Iterable,
+    Optional,
+    Generator,
+    SupportsIndex,
 )
 from scrapling.core.utils import _is_iterable, flatten
 from scrapling.core._html_utils import _replace_entities
@@ -341,7 +342,9 @@ class AttributesHandler(Mapping[str, _TextHandlerType]):
         """Acts like the standard dictionary `.get()` method"""
         return self._data.get(key, default)
 
-    def search_values(self, keyword, partial=False):
+    def search_values(
+        self, keyword: str, partial: bool = False
+    ) -> Generator["AttributesHandler", None, None]:
         """Search current attributes by values and return a dictionary of each matching item
         :param keyword: The keyword to search for in the attribute values
         :param partial: If True, the function will search if keyword in each value instead of perfect match

scrapling/core/storage.py

@@ -9,7 +9,7 @@ from orjson import dumps, loads
 from tldextract import extract as tld
 
 from scrapling.core.utils import _StorageTools, log
-from scrapling.core._types import Dict, Optional, Union
+from scrapling.core._types import Dict, Optional, Union, Any
 
 
 class StorageSystemMixin(ABC):
@@ -106,7 +106,7 @@ class SQLiteStorageSystem(StorageSystemMixin):
         """)
         self.connection.commit()
 
-    def save(self, element: HtmlElement, identifier: str):
+    def save(self, element: HtmlElement, identifier: str) -> None:
         """Saves the elements unique properties to the storage for retrieval and relocation later
 
         :param element: The element itself which we want to save to storage.
@@ -126,7 +126,7 @@ class SQLiteStorageSystem(StorageSystemMixin):
             self.cursor.fetchall()
             self.connection.commit()
 
-    def retrieve(self, identifier: str) -> Optional[Dict]:
+    def retrieve(self, identifier: str) -> Optional[Dict[str, Any]]:
         """Using the identifier, we search the storage and return the unique properties of the element
 
         :param identifier: This is the identifier that will be used to retrieve the element from the storage. See

scrapling/core/utils.py

@@ -5,7 +5,7 @@ from itertools import chain
 import orjson
 from lxml import html
 
-from scrapling.core._types import Any, Dict, Iterable, Union
+from scrapling.core._types import Any, Dict, Iterable, Union, List
 
 # Using cache on top of a class is a brilliant way to achieve a Singleton design pattern without much code
 from functools import lru_cache  # isort:skip
@@ -41,8 +41,8 @@ def setup_logger():
 log = setup_logger()
 
 
-def is_jsonable(content: Union[bytes, str]) -> bool:
-    if type(content) is bytes:
+def is_jsonable(content: bytes | str) -> bool:
+    if isinstance(content, bytes):
         content = content.decode()
 
     try:
@@ -52,14 +52,14 @@ def is_jsonable(content: Union[bytes, str]) -> bool:
         return False
 
 
-def flatten(lst: Iterable):
+def flatten(lst: Iterable[Any]) -> List[Any]:
     return list(chain.from_iterable(lst))
 
 
-def _is_iterable(s: Any):
+def _is_iterable(obj: Any) -> bool:
     # This will be used only in regex functions to make sure it's iterable but not string/bytes
     return isinstance(
-        s,
+        obj,
         (
             list,
             tuple,

scrapling/engines/_browsers/_validators.py

@@ -82,7 +82,7 @@ class CamoufoxConfig(Struct, kw_only=True, frozen=False):
     """Configuration struct for validation"""
 
     max_pages: int = 1
-    headless: Union[bool] = True  # noqa: F821
+    headless: bool = True  # noqa: F821
     block_images: bool = False
     disable_resources: bool = False
     block_webrtc: bool = False

scrapling/parser.py

@@ -1,7 +1,6 @@
-import inspect
 import os
 import re
-import typing
+from inspect import signature
 from difflib import SequenceMatcher
 from urllib.parse import urljoin
 
@@ -18,16 +17,17 @@ from lxml.etree import (
 
 from scrapling.core._types import (
     Any,
-    Callable,
     Dict,
-    Generator,
-    Iterable,
     List,
-    Optional,
-    Pattern,
-    SupportsIndex,
     Tuple,
     Union,
+    Pattern,
+    Callable,
+    Optional,
+    Iterable,
+    overload,
+    Generator,
+    SupportsIndex,
 )
 from scrapling.core.custom_types import AttributesHandler, TextHandler, TextHandlers
 from scrapling.core.mixins import SelectorsGeneration
@@ -248,7 +248,7 @@ class Selector(SelectorsGeneration):
 
     def __handle_elements(
         self, result: List[Union[HtmlElement, _ElementUnicodeResult]]
-    ) -> Union["Selectors", "TextHandlers", List]:
+    ) -> Union["Selectors", "TextHandlers"]:
         """Used internally in all functions to convert results to type (Selectors|TextHandlers) in bulk when possible"""
         if not len(
             result
@@ -761,7 +761,7 @@ class Selector(SelectorsGeneration):
                 patterns.add(arg)
 
             elif callable(arg):
-                if len(inspect.signature(arg).parameters) > 0:
+                if len(signature(arg).parameters) > 0:
                     functions.append(arg)
                 else:
                     raise TypeError(
@@ -914,7 +914,7 @@ class Selector(SelectorsGeneration):
         return round((score / checks) * 100, 2)
 
     @staticmethod
-    def __calculate_dict_diff(dict1: dict, dict2: dict) -> float:
+    def __calculate_dict_diff(dict1: Dict, dict2: Dict) -> float:
         """Used internally to calculate similarity between two dictionaries as SequenceMatcher doesn't accept dictionaries"""
         score = (
             SequenceMatcher(None, tuple(dict1.keys()), tuple(dict2.keys())).ratio()
@@ -1210,11 +1210,11 @@ class Selectors(List[Selector]):
 
     __slots__ = ()
 
-    @typing.overload
+    @overload
     def __getitem__(self, pos: SupportsIndex) -> Selector:
         pass
 
-    @typing.overload
+    @overload
     def __getitem__(self, pos: slice) -> "Selectors":
         pass