refactor(requests): Make all the type hints dynamic

Made the code shorter by about ~500 lines and easier to maintain in return for making the arguments autocompletion bad for shells that don't check for dynamic type hints like IPython.
This commit is contained in:
Karim shoair
2025-11-23 20:48:51 +02:00
parent 2bae3261a0
commit 357b170e1c
3 changed files with 340 additions and 711 deletions
+13
View File
@@ -4,6 +4,8 @@ Type definitions for type checking purposes.
from typing import (
TYPE_CHECKING,
TypedDict,
TypeAlias,
cast,
overload,
Any,
@@ -34,6 +36,17 @@ PageLoadStates = Literal["commit", "domcontentloaded", "load", "networkidle"]
extraction_types = Literal["text", "html", "markdown"]
StrOrBytes = Union[str, bytes]
if TYPE_CHECKING: # pragma: no cover
from typing_extensions import Unpack
else: # pragma: no cover
class _Unpack:
@staticmethod
def __getitem__(*args, **kwargs):
pass
Unpack = _Unpack()
try:
# Python 3.11+
+55
View File
@@ -0,0 +1,55 @@
from curl_cffi.requests import (
ProxySpec,
CookieTypes,
BrowserTypeLiteral,
)
from scrapling.core._types import (
Dict,
List,
Tuple,
Mapping,
Optional,
TypedDict,
TypeAlias,
TYPE_CHECKING,
)
# Type alias for `impersonate` parameter - accepts a single browser or list of browsers
ImpersonateType: TypeAlias = BrowserTypeLiteral | List[BrowserTypeLiteral] | None
if TYPE_CHECKING: # pragma: no cover
# Types for session initialization
class RequestsSession(TypedDict, total=False):
impersonate: ImpersonateType
http3: Optional[bool]
stealthy_headers: Optional[bool]
proxies: Optional[ProxySpec]
proxy: Optional[str]
proxy_auth: Optional[Tuple[str, str]]
timeout: Optional[int | float]
headers: Optional[Mapping[str, Optional[str]]]
retries: Optional[int]
retry_delay: Optional[int]
follow_redirects: Optional[bool]
max_redirects: Optional[int]
verify: Optional[bool]
cert: Optional[str | Tuple[str, str]]
selector_config: Optional[Dict]
# Types for GET request method parameters
class GetRequestParams(RequestsSession, total=False):
params: Optional[Dict | List | Tuple]
cookies: Optional[CookieTypes]
auth: Optional[Tuple[str, str]]
# Types for POST/PUT/DELETE request method parameters
class DataRequestParams(GetRequestParams, total=False):
data: Optional[Dict | str]
json: Optional[Dict | List]
else: # pragma: no cover
RequestsSession = TypedDict
GetRequestParams = TypedDict
DataRequestParams = TypedDict
File diff suppressed because it is too large Load Diff