refactor(requests): Make all the type hints dynamic
Made the code shorter by about ~500 lines and easier to maintain in return for making the arguments autocompletion bad for shells that don't check for dynamic type hints like IPython.
This commit is contained in:
@@ -4,6 +4,8 @@ Type definitions for type checking purposes.
|
||||
|
||||
from typing import (
|
||||
TYPE_CHECKING,
|
||||
TypedDict,
|
||||
TypeAlias,
|
||||
cast,
|
||||
overload,
|
||||
Any,
|
||||
@@ -34,6 +36,17 @@ PageLoadStates = Literal["commit", "domcontentloaded", "load", "networkidle"]
|
||||
extraction_types = Literal["text", "html", "markdown"]
|
||||
StrOrBytes = Union[str, bytes]
|
||||
|
||||
if TYPE_CHECKING: # pragma: no cover
|
||||
from typing_extensions import Unpack
|
||||
else: # pragma: no cover
|
||||
|
||||
class _Unpack:
|
||||
@staticmethod
|
||||
def __getitem__(*args, **kwargs):
|
||||
pass
|
||||
|
||||
Unpack = _Unpack()
|
||||
|
||||
|
||||
try:
|
||||
# Python 3.11+
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
from curl_cffi.requests import (
|
||||
ProxySpec,
|
||||
CookieTypes,
|
||||
BrowserTypeLiteral,
|
||||
)
|
||||
|
||||
from scrapling.core._types import (
|
||||
Dict,
|
||||
List,
|
||||
Tuple,
|
||||
Mapping,
|
||||
Optional,
|
||||
TypedDict,
|
||||
TypeAlias,
|
||||
TYPE_CHECKING,
|
||||
)
|
||||
|
||||
# Type alias for `impersonate` parameter - accepts a single browser or list of browsers
|
||||
ImpersonateType: TypeAlias = BrowserTypeLiteral | List[BrowserTypeLiteral] | None
|
||||
|
||||
|
||||
if TYPE_CHECKING: # pragma: no cover
|
||||
# Types for session initialization
|
||||
class RequestsSession(TypedDict, total=False):
|
||||
impersonate: ImpersonateType
|
||||
http3: Optional[bool]
|
||||
stealthy_headers: Optional[bool]
|
||||
proxies: Optional[ProxySpec]
|
||||
proxy: Optional[str]
|
||||
proxy_auth: Optional[Tuple[str, str]]
|
||||
timeout: Optional[int | float]
|
||||
headers: Optional[Mapping[str, Optional[str]]]
|
||||
retries: Optional[int]
|
||||
retry_delay: Optional[int]
|
||||
follow_redirects: Optional[bool]
|
||||
max_redirects: Optional[int]
|
||||
verify: Optional[bool]
|
||||
cert: Optional[str | Tuple[str, str]]
|
||||
selector_config: Optional[Dict]
|
||||
|
||||
# Types for GET request method parameters
|
||||
class GetRequestParams(RequestsSession, total=False):
|
||||
params: Optional[Dict | List | Tuple]
|
||||
cookies: Optional[CookieTypes]
|
||||
auth: Optional[Tuple[str, str]]
|
||||
|
||||
# Types for POST/PUT/DELETE request method parameters
|
||||
class DataRequestParams(GetRequestParams, total=False):
|
||||
data: Optional[Dict | str]
|
||||
json: Optional[Dict | List]
|
||||
|
||||
else: # pragma: no cover
|
||||
RequestsSession = TypedDict
|
||||
GetRequestParams = TypedDict
|
||||
DataRequestParams = TypedDict
|
||||
+272
-711
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user