freedak/Scrapling
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

style: A lot of type hints correction

Browse Source 
Since we are using Py3.10 as minimum version now, we remove Union when possible
This commit is contained in:
Karim shoair
2025-07-30 02:46:57 +03:00
parent ca12a11b7e
commit ae9ccaec79
17 changed files with 179 additions and 253 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scrapling/core/_html_utils.py
+2 -2
View File
@@ -6,7 +6,7 @@ Repo source code: https://github.com/scrapy/w3lib/blob/master/w3lib/html.py
from re import compile as _re_compile, IGNORECASE
from scrapling.core._types import Iterable, Union, Match, StrOrBytes
from scrapling.core._types import Iterable, Optional, Match, StrOrBytes
_ent_re = _re_compile(
r"&((?P<named>[a-z\d]+)|#(?P<dec>\d+)|#x(?P<hex>[a-f\d]+))(?P<semicolon>;?)",
@@ -270,7 +270,7 @@ name2codepoint = {
def to_unicode(
text: StrOrBytes, encoding: Union[str, None] = None, errors: str = "strict"
text: StrOrBytes, encoding: Optional[str] = None, errors: str = "strict"
) -> str:
"""Return the Unicode representation of a bytes object `text`. If `text`
is already a Unicode object, return it as-is."""
scrapling/core/_types.py
-1
View File
@@ -16,7 +16,6 @@ from typing import (
Optional,
Pattern,
Tuple,
Type,
TypeVar,
Union,
Match,
scrapling/core/ai.py
+22 -23
View File
@@ -17,7 +17,6 @@ from scrapling.core._types import (
Optional,
Tuple,
extraction_types,
Union,
Mapping,
Dict,
List,
@@ -61,10 +60,10 @@ class ScraplingMCPServer:
extraction_type: extraction_types = "markdown",
css_selector: Optional[str] = None,
main_content_only: bool = True,
params: Optional[Union[Dict, List, Tuple]] = None,
params: Optional[Dict | List | Tuple] = None,
headers: Optional[Mapping[str, Optional[str]]] = None,
cookies: Optional[Union[Dict[str, str], list[tuple[str, str]]]] = None,
timeout: Optional[Union[int, float]] = 30,
cookies: Optional[Dict[str, str] | list[tuple[str, str]]] = None,
timeout: Optional[int | float] = 30,
follow_redirects: bool = True,
max_redirects: int = 30,
retries: Optional[int] = 3,
@@ -140,10 +139,10 @@ class ScraplingMCPServer:
extraction_type: extraction_types = "markdown",
css_selector: Optional[str] = None,
main_content_only: bool = True,
params: Optional[Union[Dict, List, Tuple]] = None,
params: Optional[Dict | List | Tuple] = None,
headers: Optional[Mapping[str, Optional[str]]] = None,
cookies: Optional[Union[Dict[str, str], list[tuple[str, str]]]] = None,
timeout: Optional[Union[int, float]] = 30,
cookies: Optional[Dict[str, str] | list[tuple[str, str]]] = None,
timeout: Optional[int | float] = 30,
follow_redirects: bool = True,
max_redirects: int = 30,
retries: Optional[int] = 3,
@@ -232,13 +231,13 @@ class ScraplingMCPServer:
disable_webgl: bool = False,
real_chrome: bool = False,
stealth: bool = False,
wait: Union[int, float] = 0,
proxy: Optional[Union[str, Dict[str, str]]] = None,
wait: int | float = 0,
proxy: Optional[str | Dict[str, str]] = None,
locale: str = "en-US",
extra_headers: Optional[Dict[str, str]] = None,
useragent: Optional[str] = None,
cdp_url: Optional[str] = None,
timeout: Union[int, float] = 30000,
timeout: int | float = 30000,
disable_resources: bool = False,
wait_selector: Optional[str] = None,
cookies: Optional[List[Dict]] = None,
@@ -321,13 +320,13 @@ class ScraplingMCPServer:
disable_webgl: bool = False,
real_chrome: bool = False,
stealth: bool = False,
wait: Union[int, float] = 0,
proxy: Optional[Union[str, Dict[str, str]]] = None,
wait: int | float = 0,
proxy: Optional[str | Dict[str, str]] = None,
locale: str = "en-US",
extra_headers: Optional[Dict[str, str]] = None,
useragent: Optional[str] = None,
cdp_url: Optional[str] = None,
timeout: Union[int, float] = 30000,
timeout: int | float = 30000,
disable_resources: bool = False,
wait_selector: Optional[str] = None,
cookies: Optional[List[Dict]] = None,
@@ -409,23 +408,23 @@ class ScraplingMCPServer:
extraction_type: extraction_types = "markdown",
css_selector: Optional[str] = None,
main_content_only: bool = True,
headless: Union[bool] = True, # noqa: F821
headless: bool = True, # noqa: F821
block_images: bool = False,
disable_resources: bool = False,
block_webrtc: bool = False,
allow_webgl: bool = True,
network_idle: bool = False,
humanize: Union[bool, float] = True,
humanize: bool | float = True,
solve_cloudflare: bool = False,
wait: Union[int, float] = 0,
timeout: Union[int, float] = 30000,
wait: int | float = 0,
timeout: int | float = 30000,
wait_selector: Optional[str] = None,
addons: Optional[List[str]] = None,
wait_selector_state: SelectorWaitStates = "attached",
cookies: Optional[List[Dict]] = None,
google_search: bool = True,
extra_headers: Optional[Dict[str, str]] = None,
proxy: Optional[Union[str, Dict[str, str]]] = None,
proxy: Optional[str | Dict[str, str]] = None,
os_randomize: bool = False,
disable_ads: bool = False,
geoip: bool = False,
@@ -509,23 +508,23 @@ class ScraplingMCPServer:
extraction_type: extraction_types = "markdown",
css_selector: Optional[str] = None,
main_content_only: bool = True,
headless: Union[bool] = True, # noqa: F821
headless: bool = True, # noqa: F821
block_images: bool = False,
disable_resources: bool = False,
block_webrtc: bool = False,
allow_webgl: bool = True,
network_idle: bool = False,
humanize: Union[bool, float] = True,
humanize: bool | float = True,
solve_cloudflare: bool = False,
wait: Union[int, float] = 0,
timeout: Union[int, float] = 30000,
wait: int | float = 0,
timeout: int | float = 30000,
wait_selector: Optional[str] = None,
addons: Optional[List[str]] = None,
wait_selector_state: SelectorWaitStates = "attached",
cookies: Optional[List[Dict]] = None,
google_search: bool = True,
extra_headers: Optional[Dict[str, str]] = None,
proxy: Optional[Union[str, Dict[str, str]]] = None,
proxy: Optional[str | Dict[str, str]] = None,
os_randomize: bool = False,
disable_ads: bool = False,
geoip: bool = False,
scrapling/core/custom_types.py
+31 -40
View File
@@ -8,7 +8,6 @@ from scrapling.core._types import (
cast,
Dict,
List,
Union,
overload,
TypeVar,
Literal,
@@ -34,7 +33,7 @@ class TextHandler(str):
def __new__(cls, string):
return super().__new__(cls, str(string))
def __getitem__(self, key: Union[SupportsIndex, slice]) -> "TextHandler":
def __getitem__(self, key: SupportsIndex | slice) -> "TextHandler":
lst = super().__getitem__(key)
return cast(_TextHandlerType, TextHandler(lst))
@@ -46,78 +45,72 @@ class TextHandler(str):
)
)
def strip(self, chars: str = None) -> Union[str, "TextHandler"]:
def strip(self, chars: str = None) -> str | "TextHandler":
return TextHandler(super().strip(chars))
def lstrip(self, chars: str = None) -> Union[str, "TextHandler"]:
def lstrip(self, chars: str = None) -> str | "TextHandler":
return TextHandler(super().lstrip(chars))
def rstrip(self, chars: str = None) -> Union[str, "TextHandler"]:
def rstrip(self, chars: str = None) -> str | "TextHandler":
return TextHandler(super().rstrip(chars))
def capitalize(self) -> Union[str, "TextHandler"]:
def capitalize(self) -> str | "TextHandler":
return TextHandler(super().capitalize())
def casefold(self) -> Union[str, "TextHandler"]:
def casefold(self) -> str | "TextHandler":
return TextHandler(super().casefold())
def center(
self, width: SupportsIndex, fillchar: str = " "
) -> Union[str, "TextHandler"]:
def center(self, width: SupportsIndex, fillchar: str = " ") -> str | "TextHandler":
return TextHandler(super().center(width, fillchar))
def expandtabs(self, tabsize: SupportsIndex = 8) -> Union[str, "TextHandler"]:
def expandtabs(self, tabsize: SupportsIndex = 8) -> str | "TextHandler":
return TextHandler(super().expandtabs(tabsize))
def format(self, *args: str, **kwargs: str) -> Union[str, "TextHandler"]:
def format(self, *args: str, **kwargs: str) -> str | "TextHandler":
return TextHandler(super().format(*args, **kwargs))
def format_map(self, mapping) -> Union[str, "TextHandler"]:
def format_map(self, mapping) -> str | "TextHandler":
return TextHandler(super().format_map(mapping))
def join(self, iterable: Iterable[str]) -> Union[str, "TextHandler"]:
def join(self, iterable: Iterable[str]) -> str | "TextHandler":
return TextHandler(super().join(iterable))
def ljust(
self, width: SupportsIndex, fillchar: str = " "
) -> Union[str, "TextHandler"]:
def ljust(self, width: SupportsIndex, fillchar: str = " ") -> str | "TextHandler":
return TextHandler(super().ljust(width, fillchar))
def rjust(
self, width: SupportsIndex, fillchar: str = " "
) -> Union[str, "TextHandler"]:
def rjust(self, width: SupportsIndex, fillchar: str = " ") -> str | "TextHandler":
return TextHandler(super().rjust(width, fillchar))
def swapcase(self) -> Union[str, "TextHandler"]:
def swapcase(self) -> str | "TextHandler":
return TextHandler(super().swapcase())
def title(self) -> Union[str, "TextHandler"]:
def title(self) -> str | "TextHandler":
return TextHandler(super().title())
def translate(self, table) -> Union[str, "TextHandler"]:
def translate(self, table) -> str | "TextHandler":
return TextHandler(super().translate(table))
def zfill(self, width: SupportsIndex) -> Union[str, "TextHandler"]:
def zfill(self, width: SupportsIndex) -> str | "TextHandler":
return TextHandler(super().zfill(width))
def replace(
self, old: str, new: str, count: SupportsIndex = -1
) -> Union[str, "TextHandler"]:
) -> str | "TextHandler":
return TextHandler(super().replace(old, new, count))
def upper(self) -> Union[str, "TextHandler"]:
def upper(self) -> str | "TextHandler":
return TextHandler(super().upper())
def lower(self) -> Union[str, "TextHandler"]:
def lower(self) -> str | "TextHandler":
return TextHandler(super().lower())
##############
def sort(self, reverse: bool = False) -> Union[str, "TextHandler"]:
def sort(self, reverse: bool = False) -> str | "TextHandler":
"""Return a sorted version of the string"""
return self.__class__("".join(sorted(self, reverse=reverse)))
def clean(self) -> Union[str, "TextHandler"]:
def clean(self) -> str | "TextHandler":
"""Return a new version of the string after removing all white spaces and consecutive spaces"""
data = self.translate(__CLEANING_TABLE__)
return self.__class__(__CONSECUTIVE_SPACES_REGEX__.sub(" ", data).strip())
@@ -141,7 +134,7 @@ class TextHandler(str):
@overload
def re(
self,
regex: Union[str, Pattern[str]],
regex: str | Pattern,
check_match: Literal[True],
replace_entities: bool = True,
clean_match: bool = False,
@@ -151,7 +144,7 @@ class TextHandler(str):
@overload
def re(
self,
regex: Union[str, Pattern[str]],
regex: str | Pattern,
replace_entities: bool = True,
clean_match: bool = False,
case_sensitive: bool = True,
@@ -160,12 +153,12 @@ class TextHandler(str):
def re(
self,
regex: Union[str, Pattern[str]],
regex: str | Pattern,
replace_entities: bool = True,
clean_match: bool = False,
case_sensitive: bool = True,
check_match: bool = False,
) -> Union["TextHandlers[TextHandler]", bool]:
) -> "TextHandlers" | bool:
"""Apply the given regex to the current text and return a list of strings with the matches.
:param regex: Can be either a compiled regular expression or a string.
@@ -205,7 +198,7 @@ class TextHandler(str):
def re_first(
self,
regex: Union[str, Pattern[str]],
regex: str | Pattern,
default=None,
replace_entities: bool = True,
clean_match: bool = False,
@@ -244,9 +237,7 @@ class TextHandlers(List[TextHandler]):
def __getitem__(self, pos: slice) -> "TextHandlers":
pass
def __getitem__(
self, pos: Union[SupportsIndex, slice]
) -> Union[TextHandler, "TextHandlers"]:
def __getitem__(self, pos: SupportsIndex | slice) -> TextHandler | "TextHandlers":
lst = super().__getitem__(pos)
if isinstance(pos, slice):
lst = [TextHandler(s) for s in lst]
@@ -255,7 +246,7 @@ class TextHandlers(List[TextHandler]):
def re(
self,
regex: Union[str, Pattern[str]],
regex: str | Pattern,
replace_entities: bool = True,
clean_match: bool = False,
case_sensitive: bool = True,
@@ -275,7 +266,7 @@ class TextHandlers(List[TextHandler]):
def re_first(
self,
regex: Union[str, Pattern[str]],
regex: str | Pattern,
default=None,
replace_entities: bool = True,
clean_match: bool = False,
@@ -339,7 +330,7 @@ class AttributesHandler(Mapping[str, _TextHandlerType]):
def get(
self, key: str, default: Optional[str] = None
) -> Union[_TextHandlerType, None]:
) -> Optional[_TextHandlerType]:
"""Acts like the standard dictionary `.get()` method"""
return self._data.get(key, default)
scrapling/core/shell.py
+2 -3
View File
@@ -33,7 +33,6 @@ from scrapling.core._types import (
Dict,
Tuple,
Any,
Union,
extraction_types,
Generator,
)
@@ -254,7 +253,7 @@ class CurlParser:
# --- Process Data Payload ---
params = dict()
data_payload: Union[str, bytes, Dict, None] = None
data_payload: Optional[str | bytes | Dict] = None
json_payload: Optional[Any] = None
# DevTools often uses --data-raw for JSON bodies
@@ -358,7 +357,7 @@ class CurlParser:
follow_redirects=True, # Scrapling default is True
)
def convert2fetcher(self, curl_command: Union[Request, str]) -> Optional[Response]:
def convert2fetcher(self, curl_command: Request | str) -> Optional[Response]:
if isinstance(curl_command, (Request, str)):
request = (
self.parse(curl_command)
scrapling/core/storage.py
+7 -7
View File
@@ -1,20 +1,20 @@
from sqlite3 import connect as db_connect
from threading import RLock
from abc import ABC, abstractmethod
from hashlib import sha256
from threading import RLock
from functools import lru_cache
from abc import ABC, abstractmethod
from sqlite3 import connect as db_connect
from lxml.html import HtmlElement
from orjson import dumps, loads
from lxml.html import HtmlElement
from tldextract import extract as tld
from scrapling.core.utils import _StorageTools, log
from scrapling.core._types import Dict, Optional, Union, Any
from scrapling.core._types import Dict, Optional, Any
class StorageSystemMixin(ABC):
# If you want to make your own storage system, you have to inherit from this
def __init__(self, url: Union[str, None] = None):
def __init__(self, url: Optional[str] = None):
"""
:param url: URL of the website we are working on to separate it from other websites data
"""
@@ -74,7 +74,7 @@ class SQLiteStorageSystem(StorageSystemMixin):
Mainly built, so the library can run in threaded frameworks like scrapy or threaded tools
> It's optimized for threaded applications, but running it without threads shouldn't make it slow."""
def __init__(self, storage_file: str, url: Union[str, None] = None):
def __init__(self, storage_file: str, url: Optional[str] = None):
"""
:param storage_file: File to be used to store elements' data.
:param url: URL of the website we are working on to separate it from other websites data