freedak/Scrapling
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: make browser fetchers type hints always present

Browse Source
This commit is contained in:
Karim shoair
2026-02-15 01:03:15 +02:00
parent 77e825e7be
commit ba4d33337f
1 changed files with 83 additions and 88 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scrapling/engines/_browsers/_types.py
+83 -88
View File
@@ -19,7 +19,6 @@ from scrapling.core._types import (
TypeAlias, TypeAlias,
SetCookieParam, SetCookieParam,
SelectorWaitStates, SelectorWaitStates,
TYPE_CHECKING,
) )
from scrapling.engines.toolbelt.proxy_rotation import ProxyRotator from scrapling.engines.toolbelt.proxy_rotation import ProxyRotator
@@ -27,97 +26,93 @@ from scrapling.engines.toolbelt.proxy_rotation import ProxyRotator
ImpersonateType: TypeAlias = BrowserTypeLiteral | List[BrowserTypeLiteral] | None ImpersonateType: TypeAlias = BrowserTypeLiteral | List[BrowserTypeLiteral] | None
if TYPE_CHECKING: # pragma: no cover # Types for session initialization
# Types for session initialization class RequestsSession(TypedDict, total=False):
class RequestsSession(TypedDict, total=False): impersonate: ImpersonateType
impersonate: ImpersonateType http3: Optional[bool]
http3: Optional[bool] stealthy_headers: Optional[bool]
stealthy_headers: Optional[bool] proxies: Optional[ProxySpec]
proxies: Optional[ProxySpec] proxy: Optional[str]
proxy: Optional[str] proxy_auth: Optional[Tuple[str, str]]
proxy_auth: Optional[Tuple[str, str]] proxy_rotator: Optional[ProxyRotator]
proxy_rotator: Optional[ProxyRotator] timeout: Optional[int | float]
timeout: Optional[int | float] headers: Optional[Mapping[str, Optional[str]]]
headers: Optional[Mapping[str, Optional[str]]] retries: Optional[int]
retries: Optional[int] retry_delay: Optional[int]
retry_delay: Optional[int] follow_redirects: Optional[bool]
follow_redirects: Optional[bool] max_redirects: Optional[int]
max_redirects: Optional[int] verify: Optional[bool]
verify: Optional[bool] cert: Optional[str | Tuple[str, str]]
cert: Optional[str | Tuple[str, str]] selector_config: Optional[Dict]
selector_config: Optional[Dict]
# Types for GET request method parameters
class GetRequestParams(RequestsSession, total=False):
params: Optional[Dict | List | Tuple]
cookies: Optional[CookieTypes]
auth: Optional[Tuple[str, str]]
# Types for POST/PUT/DELETE request method parameters # Types for GET request method parameters
class DataRequestParams(GetRequestParams, total=False): class GetRequestParams(RequestsSession, total=False):
data: Optional[Dict[str, str] | List[Tuple] | str | BytesIO | bytes] params: Optional[Dict | List | Tuple]
json: Optional[Dict | List] cookies: Optional[CookieTypes]
auth: Optional[Tuple[str, str]]
# Types for browser session
class PlaywrightSession(TypedDict, total=False):
max_pages: int
headless: bool
disable_resources: bool
network_idle: bool
load_dom: bool
wait_selector: Optional[str]
wait_selector_state: SelectorWaitStates
cookies: Sequence[SetCookieParam] | None
google_search: bool
wait: int | float
timezone_id: str | None
page_action: Optional[Callable]
proxy: Optional[str | Dict[str, str] | Tuple]
proxy_rotator: Optional[ProxyRotator]
extra_headers: Optional[Dict[str, str]]
timeout: int | float
init_script: Optional[str]
user_data_dir: str
selector_config: Optional[Dict]
additional_args: Optional[Dict]
locale: Optional[str]
real_chrome: bool
cdp_url: Optional[str]
useragent: Optional[str]
extra_flags: Optional[List[str]]
blocked_domains: Optional[Set[str]]
retries: int
retry_delay: int | float
class PlaywrightFetchParams(TypedDict, total=False): # Types for POST/PUT/DELETE request method parameters
load_dom: bool class DataRequestParams(GetRequestParams, total=False):
wait: int | float data: Optional[Dict[str, str] | List[Tuple] | str | BytesIO | bytes]
network_idle: bool json: Optional[Dict | List]
google_search: bool
timeout: int | float
disable_resources: bool
wait_selector: Optional[str]
page_action: Optional[Callable]
selector_config: Optional[Dict]
extra_headers: Optional[Dict[str, str]]
wait_selector_state: SelectorWaitStates
blocked_domains: Optional[Set[str]]
proxy: Optional[str | Dict[str, str]]
class StealthSession(PlaywrightSession, total=False):
allow_webgl: bool
hide_canvas: bool
block_webrtc: bool
solve_cloudflare: bool
class StealthFetchParams(PlaywrightFetchParams, total=False): # Types for browser session
solve_cloudflare: bool class PlaywrightSession(TypedDict, total=False):
max_pages: int
headless: bool
disable_resources: bool
network_idle: bool
load_dom: bool
wait_selector: Optional[str]
wait_selector_state: SelectorWaitStates
cookies: Sequence[SetCookieParam] | None
google_search: bool
wait: int | float
timezone_id: str | None
page_action: Optional[Callable]
proxy: Optional[str | Dict[str, str] | Tuple]
proxy_rotator: Optional[ProxyRotator]
extra_headers: Optional[Dict[str, str]]
timeout: int | float
init_script: Optional[str]
user_data_dir: str
selector_config: Optional[Dict]
additional_args: Optional[Dict]
locale: Optional[str]
real_chrome: bool
cdp_url: Optional[str]
useragent: Optional[str]
extra_flags: Optional[List[str]]
blocked_domains: Optional[Set[str]]
retries: int
retry_delay: int | float
else: # pragma: no cover
RequestsSession = TypedDict class PlaywrightFetchParams(TypedDict, total=False):
GetRequestParams = TypedDict load_dom: bool
DataRequestParams = TypedDict wait: int | float
PlaywrightSession = TypedDict network_idle: bool
PlaywrightFetchParams = TypedDict google_search: bool
StealthSession = TypedDict timeout: int | float
StealthFetchParams = TypedDict disable_resources: bool
wait_selector: Optional[str]
page_action: Optional[Callable]
selector_config: Optional[Dict]
extra_headers: Optional[Dict[str, str]]
wait_selector_state: SelectorWaitStates
blocked_domains: Optional[Set[str]]
proxy: Optional[str | Dict[str, str]]
class StealthSession(PlaywrightSession, total=False):
allow_webgl: bool
hide_canvas: bool
block_webrtc: bool
solve_cloudflare: bool
class StealthFetchParams(PlaywrightFetchParams, total=False):
solve_cloudflare: bool