freedak/Scrapling
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: Making all the codebase acceptable by PyRight

Browse Source 
Also fixes #97
This commit is contained in:
Karim shoair
2025-10-05 04:03:39 +03:00
parent e149c715dd
commit debe03256b
21 changed files with 306 additions and 205 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scrapling/engines/_browsers/_validators.py
+28 -11
View File
@@ -11,7 +11,10 @@ from scrapling.core._types import (
Tuple,
Optional,
Callable,
Iterable,
SelectorWaitStates,
cast,
overload,
)
from scrapling.engines.toolbelt.navigation import construct_proxy_dict
@@ -73,7 +76,7 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False):
stealth: bool = False
wait: Seconds = 0
page_action: Optional[Callable] = None
proxy: Optional[str | Dict[str, str]] = None # The default value for proxy in Playwright's source is `None`
proxy: Optional[str | Dict[str, str] | Tuple] = None # The default value for proxy in Playwright's source is `None`
locale: str = "en-US"
extra_headers: Optional[Dict[str, str]] = None
useragent: Optional[str] = None
@@ -81,11 +84,11 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False):
init_script: Optional[str] = None
disable_resources: bool = False
wait_selector: Optional[str] = None
cookies: Optional[List[Dict]] = None
cookies: Optional[Iterable[Dict]] = None
network_idle: bool = False
load_dom: bool = True
wait_selector_state: SelectorWaitStates = "attached"
selector_config: Optional[Dict] = None
selector_config: Optional[Dict] = {}
def __post_init__(self):
"""Custom validation after msgspec validation"""
@@ -125,15 +128,15 @@ class CamoufoxConfig(Struct, kw_only=True, frozen=False):
wait_selector: Optional[str] = None
addons: Optional[List[str]] = None
wait_selector_state: SelectorWaitStates = "attached"
cookies: Optional[List[Dict]] = None
cookies: Optional[Iterable[Dict]] = None
google_search: bool = True
extra_headers: Optional[Dict[str, str]] = None
proxy: Optional[str | Dict[str, str]] = None # The default value for proxy in Playwright's source is `None`
proxy: Optional[str | Dict[str, str] | Tuple] = None # The default value for proxy in Playwright's source is `None`
os_randomize: bool = False
disable_ads: bool = False
geoip: bool = False
selector_config: Optional[Dict] = None
additional_args: Optional[Dict] = None
selector_config: Optional[Dict] = {}
additional_args: Optional[Dict] = {}
def __post_init__(self):
"""Custom validation after msgspec validation"""
@@ -177,7 +180,7 @@ class FetchConfig(Struct, kw_only=True):
network_idle: bool = False
load_dom: bool = True
solve_cloudflare: bool = False
selector_config: Optional[Dict] = {}
selector_config: Dict = {}
def to_dict(self):
return {f: getattr(self, f) for f in self.__struct_fields__}
@@ -198,7 +201,7 @@ class _fetch_params:
network_idle: bool
load_dom: bool
solve_cloudflare: bool
selector_config: Optional[Dict]
selector_config: Dict
def validate_fetch(params: List[Tuple], sentinel=None) -> _fetch_params:
@@ -212,7 +215,7 @@ def validate_fetch(params: List[Tuple], sentinel=None) -> _fetch_params:
result[arg] = session_value
if overrides:
overrides = validate(overrides, FetchConfig).to_dict()
overrides = cast(FetchConfig, validate(overrides, FetchConfig)).to_dict()
overrides.update(result)
return _fetch_params(**overrides)
@@ -222,7 +225,21 @@ def validate_fetch(params: List[Tuple], sentinel=None) -> _fetch_params:
return _fetch_params(**result)
def validate(params: Dict, model) -> PlaywrightConfig | CamoufoxConfig | FetchConfig:
@overload
def validate(params: Dict, model: type[PlaywrightConfig]) -> PlaywrightConfig: ...
@overload
def validate(params: Dict, model: type[CamoufoxConfig]) -> CamoufoxConfig: ...
@overload
def validate(params: Dict, model: type[FetchConfig]) -> FetchConfig: ...
def validate(
params: Dict, model: type[PlaywrightConfig] | type[CamoufoxConfig] | type[FetchConfig]
) -> PlaywrightConfig | CamoufoxConfig | FetchConfig:
try:
return convert(params, model)
except ValidationError as e: