freedak/Scrapling
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

style: A lot of type hints correction

Browse Source 
Since we are using Py3.10 as minimum version now, we remove Union when possible
This commit is contained in:
Karim shoair
2025-07-30 02:46:57 +03:00
parent ca12a11b7e
commit ae9ccaec79
17 changed files with 179 additions and 253 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scrapling/engines/_browsers/_camoufox.py
+12 -13
View File
@@ -26,10 +26,9 @@ from ._page import PageInfo, PagePool
from ._validators import validate, CamoufoxConfig
from scrapling.core._types import (
Dict,
Optional,
Union,
Callable,
List,
Optional,
Callable,
SelectorWaitStates,
)
from scrapling.engines.toolbelt import (
@@ -84,16 +83,16 @@ class StealthySession:
def __init__(
self,
max_pages: int = 1,
headless: Union[bool] = True, # noqa: F821
headless: bool = True, # noqa: F821
block_images: bool = False,
disable_resources: bool = False,
block_webrtc: bool = False,
allow_webgl: bool = True,
network_idle: bool = False,
humanize: Union[bool, float] = True,
humanize: bool | float = True,
solve_cloudflare: bool = False,
wait: Union[int, float] = 0,
timeout: Union[int, float] = 30000,
wait: int | float = 0,
timeout: int | float = 30000,
page_action: Optional[Callable] = None,
wait_selector: Optional[str] = None,
addons: Optional[List[str]] = None,
@@ -101,7 +100,7 @@ class StealthySession:
cookies: Optional[List[Dict]] = None,
google_search: bool = True,
extra_headers: Optional[Dict[str, str]] = None,
proxy: Optional[Union[str, Dict[str, str]]] = None,
proxy: Optional[str | Dict[str, str]] = None,
os_randomize: bool = False,
disable_ads: bool = False,
geoip: bool = False,
@@ -461,16 +460,16 @@ class AsyncStealthySession(StealthySession):
def __init__(
self,
max_pages: int = 1,
headless: Union[bool] = True, # noqa: F821
headless: bool = True, # noqa: F821
block_images: bool = False,
disable_resources: bool = False,
block_webrtc: bool = False,
allow_webgl: bool = True,
network_idle: bool = False,
humanize: Union[bool, float] = True,
humanize: bool | float = True,
solve_cloudflare: bool = False,
wait: Union[int, float] = 0,
timeout: Union[int, float] = 30000,
wait: int | float = 0,
timeout: int | float = 30000,
page_action: Optional[Callable] = None,
wait_selector: Optional[str] = None,
addons: Optional[List[str]] = None,
@@ -478,7 +477,7 @@ class AsyncStealthySession(StealthySession):
cookies: Optional[List[Dict]] = None,
google_search: bool = True,
extra_headers: Optional[Dict[str, str]] = None,
proxy: Optional[Union[str, Dict[str, str]]] = None,
proxy: Optional[str | Dict[str, str]] = None,
os_randomize: bool = False,
disable_ads: bool = False,
geoip: bool = False,
scrapling/engines/_browsers/_controllers.py
+7 -8
View File
@@ -28,9 +28,8 @@ from ._validators import validate, PlaywrightConfig
from ._config_tools import _compiled_stealth_scripts, _launch_kwargs, _context_kwargs
from scrapling.core._types import (
Dict,
Optional,
Union,
List,
Optional,
Callable,
SelectorWaitStates,
)
@@ -87,14 +86,14 @@ class DynamicSession:
disable_webgl: bool = False,
real_chrome: bool = False,
stealth: bool = False,
wait: Union[int, float] = 0,
wait: int | float = 0,
page_action: Optional[Callable] = None,
proxy: Optional[Union[str, Dict[str, str]]] = None,
proxy: Optional[str | Dict[str, str]] = None,
locale: str = "en-US",
extra_headers: Optional[Dict[str, str]] = None,
useragent: Optional[str] = None,
cdp_url: Optional[str] = None,
timeout: Union[int, float] = 30000,
timeout: int | float = 30000,
disable_resources: bool = False,
wait_selector: Optional[str] = None,
cookies: Optional[List[Dict]] = None,
@@ -404,14 +403,14 @@ class AsyncDynamicSession(DynamicSession):
disable_webgl: bool = False,
real_chrome: bool = False,
stealth: bool = False,
wait: Union[int, float] = 0,
wait: int | float = 0,
page_action: Optional[Callable] = None,
proxy: Optional[Union[str, Dict[str, str]]] = None,
proxy: Optional[str | Dict[str, str]] = None,
locale: str = "en-US",
extra_headers: Optional[Dict[str, str]] = None,
useragent: Optional[str] = None,
cdp_url: Optional[str] = None,
timeout: Union[int, float] = 30000,
timeout: int | float = 30000,
disable_resources: bool = False,
wait_selector: Optional[str] = None,
cookies: Optional[List[Dict]] = None,
scrapling/engines/_browsers/_page.py
+3 -3
View File
@@ -4,7 +4,7 @@ from dataclasses import dataclass
from playwright.sync_api import Page as SyncPage
from playwright.async_api import Page as AsyncPage
from scrapling.core._types import Optional, Union, List, Literal
from scrapling.core._types import Optional, List, Literal
PageState = Literal["ready", "busy", "error"] # States that a page can be in
@@ -14,7 +14,7 @@ class PageInfo:
"""Information about the page and its current state"""
__slots__ = ("page", "state", "url")
page: Union[SyncPage, AsyncPage]
page: SyncPage | AsyncPage
state: PageState
url: Optional[str]
@@ -52,7 +52,7 @@ class PagePool:
self.pages: List[PageInfo] = []
self._lock = RLock()
def add_page(self, page: Union[SyncPage, AsyncPage]) -> PageInfo:
def add_page(self, page: SyncPage | AsyncPage) -> PageInfo:
"""Add a new page to the pool"""
with self._lock:
if len(self.pages) >= self.max_pages:
scrapling/engines/_browsers/_validators.py
+7 -8
View File
@@ -4,7 +4,6 @@ from pathlib import Path
from scrapling.core._types import (
Optional,
Union,
Dict,
Callable,
List,
@@ -24,15 +23,15 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False):
disable_webgl: bool = False
real_chrome: bool = False
stealth: bool = False
wait: Union[int, float] = 0
wait: int | float = 0
page_action: Optional[Callable] = None
proxy: Optional[Union[str, Dict[str, str]]] = (
proxy: Optional[str | Dict[str, str]] = (
None # The default value for proxy in Playwright's source is `None`
)
locale: str = "en-US"
extra_headers: Optional[Dict[str, str]] = None
useragent: Optional[str] = None
timeout: Union[int, float] = 30000
timeout: int | float = 30000
disable_resources: bool = False
wait_selector: Optional[str] = None
cookies: Optional[List[Dict]] = None
@@ -87,10 +86,10 @@ class CamoufoxConfig(Struct, kw_only=True, frozen=False):
block_webrtc: bool = False
allow_webgl: bool = True
network_idle: bool = False
humanize: Union[bool, float] = True
humanize: bool | float = True
solve_cloudflare: bool = False
wait: Union[int, float] = 0
timeout: Union[int, float] = 30000
wait: int | float = 0
timeout: int | float = 30000
page_action: Optional[Callable] = None
wait_selector: Optional[str] = None
addons: Optional[List[str]] = None
@@ -98,7 +97,7 @@ class CamoufoxConfig(Struct, kw_only=True, frozen=False):
cookies: Optional[List[Dict]] = None
google_search: bool = True
extra_headers: Optional[Dict[str, str]] = None
proxy: Optional[Union[str, Dict[str, str]]] = (
proxy: Optional[str | Dict[str, str]] = (
None # The default value for proxy in Playwright's source is `None`
)
os_randomize: bool = False
scrapling/engines/static.py
+25 -26
View File
@@ -17,7 +17,6 @@ from scrapling.core._types import (
Dict,
Optional,
Tuple,
Union,
Mapping,
SUPPORTED_HTTP_METHODS,
Awaitable,
@@ -55,14 +54,14 @@ class FetcherSession:
proxies: Optional[Dict[str, str]] = None,
proxy: Optional[str] = None,
proxy_auth: Optional[Tuple[str, str]] = None,
timeout: Optional[Union[int, float]] = 30,
timeout: Optional[int | float] = 30,
headers: Optional[Dict[str, str]] = None,
retries: Optional[int] = 3,
retry_delay: Optional[int] = 1,
follow_redirects: bool = True,
max_redirects: int = 30,
verify: bool = True,
cert: Optional[Union[str, Tuple[str, str]]] = None,
cert: Optional[str | Tuple[str, str]] = None,
selector_config: Optional[Dict] = None,
):
"""
@@ -357,7 +356,7 @@ class FetcherSession:
method: SUPPORTED_HTTP_METHODS,
stealth: Optional[bool] = None,
**kwargs,
) -> Union[Response, Awaitable[Response]]:
) -> Response | Awaitable[Response]:
"""
Internal dispatcher. Prepares arguments and calls sync or async request helper.
@@ -390,10 +389,10 @@ class FetcherSession:
def get(
self,
url: str,
params: Optional[Union[Dict, List, Tuple]] = None,
params: Optional[Dict | List | Tuple] = None,
headers: Optional[Mapping[str, Optional[str]]] = _UNSET,
cookies: Optional[CookieTypes] = None,
timeout: Optional[Union[int, float]] = _UNSET,
timeout: Optional[int | float] = _UNSET,
follow_redirects: Optional[bool] = _UNSET,
max_redirects: Optional[int] = _UNSET,
retries: Optional[int] = _UNSET,
@@ -403,12 +402,12 @@ class FetcherSession:
proxy_auth: Optional[Tuple[str, str]] = _UNSET,
auth: Optional[Tuple[str, str]] = None,
verify: Optional[bool] = _UNSET,
cert: Optional[Union[str, Tuple[str, str]]] = _UNSET,
cert: Optional[str | Tuple[str, str]] = _UNSET,
impersonate: Optional[BrowserTypeLiteral] = _UNSET,
http3: Optional[bool] = _UNSET,
stealthy_headers: Optional[bool] = _UNSET,
**kwargs,
) -> Union[Response, Awaitable[Response]]:
) -> Response | Awaitable[Response]:
"""
Perform a GET request.
@@ -461,12 +460,12 @@ class FetcherSession:
def post(
self,
url: str,
data: Optional[Union[Dict, str]] = None,
json: Optional[Union[Dict, List]] = None,
data: Optional[Dict | str] = None,
json: Optional[Dict | List] = None,
headers: Optional[Mapping[str, Optional[str]]] = _UNSET,
params: Optional[Union[Dict, List, Tuple]] = None,
params: Optional[Dict | List | Tuple] = None,
cookies: Optional[CookieTypes] = None,
timeout: Optional[Union[int, float]] = _UNSET,
timeout: Optional[int | float] = _UNSET,
follow_redirects: Optional[bool] = _UNSET,
max_redirects: Optional[int] = _UNSET,
retries: Optional[int] = _UNSET,
@@ -476,12 +475,12 @@ class FetcherSession:
proxy_auth: Optional[Tuple[str, str]] = _UNSET,
auth: Optional[Tuple[str, str]] = None,
verify: Optional[bool] = _UNSET,
cert: Optional[Union[str, Tuple[str, str]]] = _UNSET,
cert: Optional[str | Tuple[str, str]] = _UNSET,
impersonate: Optional[BrowserTypeLiteral] = _UNSET,
http3: Optional[bool] = _UNSET,
stealthy_headers: Optional[bool] = _UNSET,
**kwargs,
) -> Union[Response, Awaitable[Response]]:
) -> Response | Awaitable[Response]:
"""
Perform a POST request.
@@ -538,12 +537,12 @@ class FetcherSession:
def put(
self,
url: str,
data: Optional[Union[Dict, str]] = None,
json: Optional[Union[Dict, List]] = None,
data: Optional[Dict | str] = None,
json: Optional[Dict | List] = None,
headers: Optional[Mapping[str, Optional[str]]] = _UNSET,
params: Optional[Union[Dict, List, Tuple]] = None,
params: Optional[Dict | List | Tuple] = None,
cookies: Optional[CookieTypes] = None,
timeout: Optional[Union[int, float]] = _UNSET,
timeout: Optional[int | float] = _UNSET,
follow_redirects: Optional[bool] = _UNSET,
max_redirects: Optional[int] = _UNSET,
retries: Optional[int] = _UNSET,
@@ -553,12 +552,12 @@ class FetcherSession:
proxy_auth: Optional[Tuple[str, str]] = _UNSET,
auth: Optional[Tuple[str, str]] = None,
verify: Optional[bool] = _UNSET,
cert: Optional[Union[str, Tuple[str, str]]] = _UNSET,
cert: Optional[str | Tuple[str, str]] = _UNSET,
impersonate: Optional[BrowserTypeLiteral] = _UNSET,
http3: Optional[bool] = _UNSET,
stealthy_headers: Optional[bool] = _UNSET,
**kwargs,
) -> Union[Response, Awaitable[Response]]:
) -> Response | Awaitable[Response]:
"""
Perform a PUT request.
@@ -615,12 +614,12 @@ class FetcherSession:
def delete(
self,
url: str,
data: Optional[Union[Dict, str]] = None,
json: Optional[Union[Dict, List]] = None,
data: Optional[Dict | str] = None,
json: Optional[Dict | List] = None,
headers: Optional[Mapping[str, Optional[str]]] = _UNSET,
params: Optional[Union[Dict, List, Tuple]] = None,
params: Optional[Dict | List | Tuple] = None,
cookies: Optional[CookieTypes] = None,
timeout: Optional[Union[int, float]] = _UNSET,
timeout: Optional[int | float] = _UNSET,
follow_redirects: Optional[bool] = _UNSET,
max_redirects: Optional[int] = _UNSET,
retries: Optional[int] = _UNSET,
@@ -630,12 +629,12 @@ class FetcherSession:
proxy_auth: Optional[Tuple[str, str]] = _UNSET,
auth: Optional[Tuple[str, str]] = None,
verify: Optional[bool] = _UNSET,
cert: Optional[Union[str, Tuple[str, str]]] = _UNSET,
cert: Optional[str | Tuple[str, str]] = _UNSET,
impersonate: Optional[BrowserTypeLiteral] = _UNSET,
http3: Optional[bool] = _UNSET,
stealthy_headers: Optional[bool] = _UNSET,
**kwargs,
) -> Union[Response, Awaitable[Response]]:
) -> Response | Awaitable[Response]:
"""
Perform a DELETE request.
scrapling/engines/toolbelt/__init__.py
-1
View File
@@ -2,7 +2,6 @@ from .custom import (
BaseFetcher,
Response,
StatusText,
check_type_validity,
get_variable_name,
)
from .fingerprints import (
scrapling/engines/toolbelt/custom.py
+1 -51
View File
@@ -10,8 +10,6 @@ from scrapling.core._types import (
List,
Optional,
Tuple,
Type,
Union,
)
from scrapling.core.custom_types import MappingProxyType
from scrapling.core.utils import log, lru_cache
@@ -106,7 +104,7 @@ class Response(Selector):
content: str | bytes,
status: int,
reason: str,
cookies: Union[Tuple[Dict[str, str], ...], Dict[str, str]],
cookies: Tuple[Dict[str, str], ...] | Dict[str, str],
headers: Dict,
request_headers: Dict,
encoding: str = "utf-8",
@@ -318,51 +316,3 @@ def get_variable_name(var: Any) -> Optional[str]:
if value is var:
return name
return None
def check_type_validity(
variable: Any,
valid_types: Union[List[Type], None],
default_value: Any = None,
critical: bool = False,
param_name: Optional[str] = None,
) -> Any:
"""Check if a variable matches the specified type constraints.
:param variable: The variable to check
:param valid_types: List of valid types for the variable
:param default_value: Value to return if type check fails
:param critical: If True, raises TypeError instead of logging error
:param param_name: Optional parameter name for error messages
:return: The original variable if valid, default_value if invalid
:raise TypeError: If critical=True and type check fails
"""
# Use provided param_name or try to get it automatically
var_name = param_name or get_variable_name(variable) or "Unknown"
# Convert valid_types to a list if None
valid_types = valid_types or []
# Handle None value
if variable is None:
if type(None) in valid_types:
return variable
error_msg = f'Argument "{var_name}" cannot be None'
if critical:
raise TypeError(error_msg)
log.error(f"[Ignored] {error_msg}")
return default_value
# If no valid_types specified and variable has a value, return it
if not valid_types:
return variable
# Check if variable type matches any of the valid types
if not any(isinstance(variable, t) for t in valid_types):
type_names = [t.__name__ for t in valid_types]
error_msg = f'Argument "{var_name}" must be of type {" or ".join(type_names)}'
if critical:
raise TypeError(error_msg)
log.error(f"[Ignored] {error_msg}")
return default_value
return variable
scrapling/engines/toolbelt/fingerprints.py
+2 -2
View File
@@ -7,7 +7,7 @@ from platform import system as platform_system
from tldextract import extract
from browserforge.headers import Browser, HeaderGenerator
from scrapling.core._types import Dict, Union
from scrapling.core._types import Dict, Optional
from scrapling.core.utils import lru_cache
__OS_NAME__ = platform_system()
@@ -28,7 +28,7 @@ def generate_convincing_referer(url: str) -> str:
@lru_cache(1, typed=True)
def get_os_name() -> Union[str, None]:
def get_os_name() -> Optional[str]:
"""Get the current OS name in the same format needed for browserforge
:return: Current OS name or `None` otherwise
scrapling/engines/toolbelt/navigation.py
+3 -3
View File
@@ -11,7 +11,7 @@ from msgspec import Struct, structs, convert, ValidationError
from playwright.sync_api import Route
from scrapling.core.utils import log
from scrapling.core._types import Dict, Optional, Union, Tuple
from scrapling.core._types import Dict, Optional, Tuple
from scrapling.engines.constants import DEFAULT_DISABLED_RESOURCES
__BYPASSES_DIR__ = Path(__file__).parent / "bypasses"
@@ -54,8 +54,8 @@ async def async_intercept_route(route: async_Route):
def construct_proxy_dict(
proxy_string: Union[str, Dict[str, str]], as_tuple=False
) -> Union[Dict, Tuple, None]:
proxy_string: str | Dict[str, str], as_tuple=False
) -> Optional[Dict | Tuple]:
"""Validate a proxy and return it in the acceptable format for Playwright
Reference: https://playwright.dev/python/docs/network#http-proxy