refactor(requests): Make all the type hints dynamic
Made the code shorter by about ~500 lines and easier to maintain in return for making the arguments autocompletion bad for shells that don't check for dynamic type hints like IPython.
This commit is contained in:
@@ -4,6 +4,8 @@ Type definitions for type checking purposes.
|
|||||||
|
|
||||||
from typing import (
|
from typing import (
|
||||||
TYPE_CHECKING,
|
TYPE_CHECKING,
|
||||||
|
TypedDict,
|
||||||
|
TypeAlias,
|
||||||
cast,
|
cast,
|
||||||
overload,
|
overload,
|
||||||
Any,
|
Any,
|
||||||
@@ -34,6 +36,17 @@ PageLoadStates = Literal["commit", "domcontentloaded", "load", "networkidle"]
|
|||||||
extraction_types = Literal["text", "html", "markdown"]
|
extraction_types = Literal["text", "html", "markdown"]
|
||||||
StrOrBytes = Union[str, bytes]
|
StrOrBytes = Union[str, bytes]
|
||||||
|
|
||||||
|
if TYPE_CHECKING: # pragma: no cover
|
||||||
|
from typing_extensions import Unpack
|
||||||
|
else: # pragma: no cover
|
||||||
|
|
||||||
|
class _Unpack:
|
||||||
|
@staticmethod
|
||||||
|
def __getitem__(*args, **kwargs):
|
||||||
|
pass
|
||||||
|
|
||||||
|
Unpack = _Unpack()
|
||||||
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
# Python 3.11+
|
# Python 3.11+
|
||||||
|
|||||||
@@ -0,0 +1,55 @@
|
|||||||
|
from curl_cffi.requests import (
|
||||||
|
ProxySpec,
|
||||||
|
CookieTypes,
|
||||||
|
BrowserTypeLiteral,
|
||||||
|
)
|
||||||
|
|
||||||
|
from scrapling.core._types import (
|
||||||
|
Dict,
|
||||||
|
List,
|
||||||
|
Tuple,
|
||||||
|
Mapping,
|
||||||
|
Optional,
|
||||||
|
TypedDict,
|
||||||
|
TypeAlias,
|
||||||
|
TYPE_CHECKING,
|
||||||
|
)
|
||||||
|
|
||||||
|
# Type alias for `impersonate` parameter - accepts a single browser or list of browsers
|
||||||
|
ImpersonateType: TypeAlias = BrowserTypeLiteral | List[BrowserTypeLiteral] | None
|
||||||
|
|
||||||
|
|
||||||
|
if TYPE_CHECKING: # pragma: no cover
|
||||||
|
# Types for session initialization
|
||||||
|
class RequestsSession(TypedDict, total=False):
|
||||||
|
impersonate: ImpersonateType
|
||||||
|
http3: Optional[bool]
|
||||||
|
stealthy_headers: Optional[bool]
|
||||||
|
proxies: Optional[ProxySpec]
|
||||||
|
proxy: Optional[str]
|
||||||
|
proxy_auth: Optional[Tuple[str, str]]
|
||||||
|
timeout: Optional[int | float]
|
||||||
|
headers: Optional[Mapping[str, Optional[str]]]
|
||||||
|
retries: Optional[int]
|
||||||
|
retry_delay: Optional[int]
|
||||||
|
follow_redirects: Optional[bool]
|
||||||
|
max_redirects: Optional[int]
|
||||||
|
verify: Optional[bool]
|
||||||
|
cert: Optional[str | Tuple[str, str]]
|
||||||
|
selector_config: Optional[Dict]
|
||||||
|
|
||||||
|
# Types for GET request method parameters
|
||||||
|
class GetRequestParams(RequestsSession, total=False):
|
||||||
|
params: Optional[Dict | List | Tuple]
|
||||||
|
cookies: Optional[CookieTypes]
|
||||||
|
auth: Optional[Tuple[str, str]]
|
||||||
|
|
||||||
|
# Types for POST/PUT/DELETE request method parameters
|
||||||
|
class DataRequestParams(GetRequestParams, total=False):
|
||||||
|
data: Optional[Dict | str]
|
||||||
|
json: Optional[Dict | List]
|
||||||
|
|
||||||
|
else: # pragma: no cover
|
||||||
|
RequestsSession = TypedDict
|
||||||
|
GetRequestParams = TypedDict
|
||||||
|
DataRequestParams = TypedDict
|
||||||
+272
-711
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user