style(Fetcher): correcting type hints and annotation
This commit is contained in:
+49
-24
@@ -1,7 +1,7 @@
|
|||||||
from time import sleep as time_sleep
|
from time import sleep as time_sleep
|
||||||
from asyncio import sleep as asyncio_sleep
|
from asyncio import sleep as asyncio_sleep
|
||||||
|
|
||||||
from curl_cffi.requests.session import CurlError
|
from curl_cffi.curl import CurlError
|
||||||
from curl_cffi import CurlHttpVersion
|
from curl_cffi import CurlHttpVersion
|
||||||
from curl_cffi.requests.impersonate import DEFAULT_CHROME
|
from curl_cffi.requests.impersonate import DEFAULT_CHROME
|
||||||
from curl_cffi.requests import (
|
from curl_cffi.requests import (
|
||||||
@@ -22,13 +22,14 @@ from scrapling.core._types import (
|
|||||||
Awaitable,
|
Awaitable,
|
||||||
List,
|
List,
|
||||||
Any,
|
Any,
|
||||||
|
cast,
|
||||||
)
|
)
|
||||||
|
|
||||||
from .toolbelt.custom import Response
|
from .toolbelt.custom import Response
|
||||||
from .toolbelt.convertor import ResponseFactory
|
from .toolbelt.convertor import ResponseFactory
|
||||||
from .toolbelt.fingerprints import generate_convincing_referer, generate_headers, __default_useragent__
|
from .toolbelt.fingerprints import generate_convincing_referer, generate_headers, __default_useragent__
|
||||||
|
|
||||||
_UNSET = object()
|
_UNSET: Any = object()
|
||||||
|
|
||||||
|
|
||||||
class FetcherSession:
|
class FetcherSession:
|
||||||
@@ -94,8 +95,8 @@ class FetcherSession:
|
|||||||
self.default_http3 = http3
|
self.default_http3 = http3
|
||||||
self.selector_config = selector_config or {}
|
self.selector_config = selector_config or {}
|
||||||
|
|
||||||
self._curl_session: Optional[CurlSession] | bool = None
|
self._curl_session: Optional[CurlSession] = None
|
||||||
self._async_curl_session: Optional[AsyncCurlSession] | bool = None
|
self._async_curl_session: Optional[AsyncCurlSession] = None
|
||||||
|
|
||||||
def _merge_request_args(self, **kwargs) -> Dict[str, Any]:
|
def _merge_request_args(self, **kwargs) -> Dict[str, Any]:
|
||||||
"""Merge request-specific arguments with default session arguments."""
|
"""Merge request-specific arguments with default session arguments."""
|
||||||
@@ -233,7 +234,7 @@ class FetcherSession:
|
|||||||
request_args: Dict[str, Any],
|
request_args: Dict[str, Any],
|
||||||
max_retries: int,
|
max_retries: int,
|
||||||
retry_delay: int,
|
retry_delay: int,
|
||||||
selector_config: Optional[Dict] = None,
|
selector_config: Dict,
|
||||||
) -> Response:
|
) -> Response:
|
||||||
"""
|
"""
|
||||||
Perform an HTTP request using the configured session.
|
Perform an HTTP request using the configured session.
|
||||||
@@ -273,7 +274,7 @@ class FetcherSession:
|
|||||||
request_args: Dict[str, Any],
|
request_args: Dict[str, Any],
|
||||||
max_retries: int,
|
max_retries: int,
|
||||||
retry_delay: int,
|
retry_delay: int,
|
||||||
selector_config: Optional[Dict] = None,
|
selector_config: Dict,
|
||||||
) -> Response:
|
) -> Response:
|
||||||
"""
|
"""
|
||||||
Perform an HTTP request using the configured session.
|
Perform an HTTP request using the configured session.
|
||||||
@@ -644,11 +645,11 @@ class FetcherSession:
|
|||||||
class FetcherClient(FetcherSession):
|
class FetcherClient(FetcherSession):
|
||||||
def __init__(self, *args, **kwargs):
|
def __init__(self, *args, **kwargs):
|
||||||
super().__init__(*args, **kwargs)
|
super().__init__(*args, **kwargs)
|
||||||
self.__enter__ = None
|
self.__enter__: Any = None
|
||||||
self.__exit__ = None
|
self.__exit__: Any = None
|
||||||
self.__aenter__ = None
|
self.__aenter__: Any = None
|
||||||
self.__aexit__ = None
|
self.__aexit__: Any = None
|
||||||
self._curl_session = True
|
self._curl_session: Any = True
|
||||||
|
|
||||||
# Setting the correct return types for the type checking/autocompletion
|
# Setting the correct return types for the type checking/autocompletion
|
||||||
def get(
|
def get(
|
||||||
@@ -673,7 +674,9 @@ class FetcherClient(FetcherSession):
|
|||||||
stealthy_headers: Optional[bool] = _UNSET,
|
stealthy_headers: Optional[bool] = _UNSET,
|
||||||
**kwargs,
|
**kwargs,
|
||||||
) -> Response:
|
) -> Response:
|
||||||
return super().get(
|
return cast(
|
||||||
|
Response,
|
||||||
|
super().get(
|
||||||
url,
|
url,
|
||||||
params,
|
params,
|
||||||
headers,
|
headers,
|
||||||
@@ -693,6 +696,7 @@ class FetcherClient(FetcherSession):
|
|||||||
http3,
|
http3,
|
||||||
stealthy_headers,
|
stealthy_headers,
|
||||||
**kwargs,
|
**kwargs,
|
||||||
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
def post(
|
def post(
|
||||||
@@ -719,7 +723,9 @@ class FetcherClient(FetcherSession):
|
|||||||
stealthy_headers: Optional[bool] = _UNSET,
|
stealthy_headers: Optional[bool] = _UNSET,
|
||||||
**kwargs,
|
**kwargs,
|
||||||
) -> Response:
|
) -> Response:
|
||||||
return super().post(
|
return cast(
|
||||||
|
Response,
|
||||||
|
super().post(
|
||||||
url,
|
url,
|
||||||
data,
|
data,
|
||||||
json,
|
json,
|
||||||
@@ -741,6 +747,7 @@ class FetcherClient(FetcherSession):
|
|||||||
http3,
|
http3,
|
||||||
stealthy_headers,
|
stealthy_headers,
|
||||||
**kwargs,
|
**kwargs,
|
||||||
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
def put(
|
def put(
|
||||||
@@ -767,7 +774,9 @@ class FetcherClient(FetcherSession):
|
|||||||
stealthy_headers: Optional[bool] = _UNSET,
|
stealthy_headers: Optional[bool] = _UNSET,
|
||||||
**kwargs,
|
**kwargs,
|
||||||
) -> Response:
|
) -> Response:
|
||||||
return super().put(
|
return cast(
|
||||||
|
Response,
|
||||||
|
super().put(
|
||||||
url,
|
url,
|
||||||
data,
|
data,
|
||||||
json,
|
json,
|
||||||
@@ -789,6 +798,7 @@ class FetcherClient(FetcherSession):
|
|||||||
http3,
|
http3,
|
||||||
stealthy_headers,
|
stealthy_headers,
|
||||||
**kwargs,
|
**kwargs,
|
||||||
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
def delete(
|
def delete(
|
||||||
@@ -815,7 +825,9 @@ class FetcherClient(FetcherSession):
|
|||||||
stealthy_headers: Optional[bool] = _UNSET,
|
stealthy_headers: Optional[bool] = _UNSET,
|
||||||
**kwargs,
|
**kwargs,
|
||||||
) -> Response:
|
) -> Response:
|
||||||
return super().delete(
|
return cast(
|
||||||
|
Response,
|
||||||
|
super().delete(
|
||||||
url,
|
url,
|
||||||
data,
|
data,
|
||||||
json,
|
json,
|
||||||
@@ -837,17 +849,18 @@ class FetcherClient(FetcherSession):
|
|||||||
http3,
|
http3,
|
||||||
stealthy_headers,
|
stealthy_headers,
|
||||||
**kwargs,
|
**kwargs,
|
||||||
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
class AsyncFetcherClient(FetcherSession):
|
class AsyncFetcherClient(FetcherSession):
|
||||||
def __init__(self, *args, **kwargs):
|
def __init__(self, *args, **kwargs):
|
||||||
super().__init__(*args, **kwargs)
|
super().__init__(*args, **kwargs)
|
||||||
self.__enter__ = None
|
self.__enter__: Any = None
|
||||||
self.__exit__ = None
|
self.__exit__: Any = None
|
||||||
self.__aenter__ = None
|
self.__aenter__: Any = None
|
||||||
self.__aexit__ = None
|
self.__aexit__: Any = None
|
||||||
self._async_curl_session = True
|
self._async_curl_session: Any = True
|
||||||
|
|
||||||
# Setting the correct return types for the type checking/autocompletion
|
# Setting the correct return types for the type checking/autocompletion
|
||||||
def get(
|
def get(
|
||||||
@@ -872,7 +885,9 @@ class AsyncFetcherClient(FetcherSession):
|
|||||||
stealthy_headers: Optional[bool] = _UNSET,
|
stealthy_headers: Optional[bool] = _UNSET,
|
||||||
**kwargs,
|
**kwargs,
|
||||||
) -> Awaitable[Response]:
|
) -> Awaitable[Response]:
|
||||||
return super().get(
|
return cast(
|
||||||
|
Awaitable[Response],
|
||||||
|
super().get(
|
||||||
url,
|
url,
|
||||||
params,
|
params,
|
||||||
headers,
|
headers,
|
||||||
@@ -892,6 +907,7 @@ class AsyncFetcherClient(FetcherSession):
|
|||||||
http3,
|
http3,
|
||||||
stealthy_headers,
|
stealthy_headers,
|
||||||
**kwargs,
|
**kwargs,
|
||||||
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
def post(
|
def post(
|
||||||
@@ -918,7 +934,9 @@ class AsyncFetcherClient(FetcherSession):
|
|||||||
stealthy_headers: Optional[bool] = _UNSET,
|
stealthy_headers: Optional[bool] = _UNSET,
|
||||||
**kwargs,
|
**kwargs,
|
||||||
) -> Awaitable[Response]:
|
) -> Awaitable[Response]:
|
||||||
return super().post(
|
return cast(
|
||||||
|
Awaitable[Response],
|
||||||
|
super().post(
|
||||||
url,
|
url,
|
||||||
data,
|
data,
|
||||||
json,
|
json,
|
||||||
@@ -940,6 +958,7 @@ class AsyncFetcherClient(FetcherSession):
|
|||||||
http3,
|
http3,
|
||||||
stealthy_headers,
|
stealthy_headers,
|
||||||
**kwargs,
|
**kwargs,
|
||||||
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
def put(
|
def put(
|
||||||
@@ -966,7 +985,9 @@ class AsyncFetcherClient(FetcherSession):
|
|||||||
stealthy_headers: Optional[bool] = _UNSET,
|
stealthy_headers: Optional[bool] = _UNSET,
|
||||||
**kwargs,
|
**kwargs,
|
||||||
) -> Awaitable[Response]:
|
) -> Awaitable[Response]:
|
||||||
return super().put(
|
return cast(
|
||||||
|
Awaitable[Response],
|
||||||
|
super().put(
|
||||||
url,
|
url,
|
||||||
data,
|
data,
|
||||||
json,
|
json,
|
||||||
@@ -988,6 +1009,7 @@ class AsyncFetcherClient(FetcherSession):
|
|||||||
http3,
|
http3,
|
||||||
stealthy_headers,
|
stealthy_headers,
|
||||||
**kwargs,
|
**kwargs,
|
||||||
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
def delete(
|
def delete(
|
||||||
@@ -1014,7 +1036,9 @@ class AsyncFetcherClient(FetcherSession):
|
|||||||
stealthy_headers: Optional[bool] = _UNSET,
|
stealthy_headers: Optional[bool] = _UNSET,
|
||||||
**kwargs,
|
**kwargs,
|
||||||
) -> Awaitable[Response]:
|
) -> Awaitable[Response]:
|
||||||
return super().delete(
|
return cast(
|
||||||
|
Awaitable[Response],
|
||||||
|
super().delete(
|
||||||
url,
|
url,
|
||||||
data,
|
data,
|
||||||
json,
|
json,
|
||||||
@@ -1036,4 +1060,5 @@ class AsyncFetcherClient(FetcherSession):
|
|||||||
http3,
|
http3,
|
||||||
stealthy_headers,
|
stealthy_headers,
|
||||||
**kwargs,
|
**kwargs,
|
||||||
|
),
|
||||||
)
|
)
|
||||||
|
|||||||
Reference in New Issue
Block a user