freedak/Scrapling
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

style(Fetcher): correcting type hints and annotation

Browse Source
This commit is contained in:
Karim shoair
2025-10-01 06:18:23 +03:00
parent 849913ad77
commit 5127d386c9
1 changed files with 213 additions and 188 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scrapling/engines/static.py
+49 -24
View File
@@ -1,7 +1,7 @@
from time import sleep as time_sleep
from asyncio import sleep as asyncio_sleep
from curl_cffi.requests.session import CurlError
from curl_cffi.curl import CurlError
from curl_cffi import CurlHttpVersion
from curl_cffi.requests.impersonate import DEFAULT_CHROME
from curl_cffi.requests import (
@@ -22,13 +22,14 @@ from scrapling.core._types import (
Awaitable,
List,
Any,
cast,
)
from .toolbelt.custom import Response
from .toolbelt.convertor import ResponseFactory
from .toolbelt.fingerprints import generate_convincing_referer, generate_headers, __default_useragent__
_UNSET = object()
_UNSET: Any = object()
class FetcherSession:
@@ -94,8 +95,8 @@ class FetcherSession:
self.default_http3 = http3
self.selector_config = selector_config or {}
self._curl_session: Optional[CurlSession] | bool = None
self._async_curl_session: Optional[AsyncCurlSession] | bool = None
self._curl_session: Optional[CurlSession] = None
self._async_curl_session: Optional[AsyncCurlSession] = None
def _merge_request_args(self, **kwargs) -> Dict[str, Any]:
"""Merge request-specific arguments with default session arguments."""
@@ -233,7 +234,7 @@ class FetcherSession:
request_args: Dict[str, Any],
max_retries: int,
retry_delay: int,
selector_config: Optional[Dict] = None,
selector_config: Dict,
) -> Response:
"""
Perform an HTTP request using the configured session.
@@ -273,7 +274,7 @@ class FetcherSession:
request_args: Dict[str, Any],
max_retries: int,
retry_delay: int,
selector_config: Optional[Dict] = None,
selector_config: Dict,
) -> Response:
"""
Perform an HTTP request using the configured session.
@@ -644,11 +645,11 @@ class FetcherSession:
class FetcherClient(FetcherSession):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.__enter__ = None
self.__exit__ = None
self.__aenter__ = None
self.__aexit__ = None
self._curl_session = True
self.__enter__: Any = None
self.__exit__: Any = None
self.__aenter__: Any = None
self.__aexit__: Any = None
self._curl_session: Any = True
# Setting the correct return types for the type checking/autocompletion
def get(
@@ -673,7 +674,9 @@ class FetcherClient(FetcherSession):
stealthy_headers: Optional[bool] = _UNSET,
**kwargs,
) -> Response:
return super().get(
return cast(
Response,
super().get(
url,
params,
headers,
@@ -693,6 +696,7 @@ class FetcherClient(FetcherSession):
http3,
stealthy_headers,
**kwargs,
),
)
def post(
@@ -719,7 +723,9 @@ class FetcherClient(FetcherSession):
stealthy_headers: Optional[bool] = _UNSET,
**kwargs,
) -> Response:
return super().post(
return cast(
Response,
super().post(
url,
data,
json,
@@ -741,6 +747,7 @@ class FetcherClient(FetcherSession):
http3,
stealthy_headers,
**kwargs,
),
)
def put(
@@ -767,7 +774,9 @@ class FetcherClient(FetcherSession):
stealthy_headers: Optional[bool] = _UNSET,
**kwargs,
) -> Response:
return super().put(
return cast(
Response,
super().put(
url,
data,
json,
@@ -789,6 +798,7 @@ class FetcherClient(FetcherSession):
http3,
stealthy_headers,
**kwargs,
),
)
def delete(
@@ -815,7 +825,9 @@ class FetcherClient(FetcherSession):
stealthy_headers: Optional[bool] = _UNSET,
**kwargs,
) -> Response:
return super().delete(
return cast(
Response,
super().delete(
url,
data,
json,
@@ -837,17 +849,18 @@ class FetcherClient(FetcherSession):
http3,
stealthy_headers,
**kwargs,
),
)
class AsyncFetcherClient(FetcherSession):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.__enter__ = None
self.__exit__ = None
self.__aenter__ = None
self.__aexit__ = None
self._async_curl_session = True
self.__enter__: Any = None
self.__exit__: Any = None
self.__aenter__: Any = None
self.__aexit__: Any = None
self._async_curl_session: Any = True
# Setting the correct return types for the type checking/autocompletion
def get(
@@ -872,7 +885,9 @@ class AsyncFetcherClient(FetcherSession):
stealthy_headers: Optional[bool] = _UNSET,
**kwargs,
) -> Awaitable[Response]:
return super().get(
return cast(
Awaitable[Response],
super().get(
url,
params,
headers,
@@ -892,6 +907,7 @@ class AsyncFetcherClient(FetcherSession):
http3,
stealthy_headers,
**kwargs,
),
)
def post(
@@ -918,7 +934,9 @@ class AsyncFetcherClient(FetcherSession):
stealthy_headers: Optional[bool] = _UNSET,
**kwargs,
) -> Awaitable[Response]:
return super().post(
return cast(
Awaitable[Response],
super().post(
url,
data,
json,
@@ -940,6 +958,7 @@ class AsyncFetcherClient(FetcherSession):
http3,
stealthy_headers,
**kwargs,
),
)
def put(
@@ -966,7 +985,9 @@ class AsyncFetcherClient(FetcherSession):
stealthy_headers: Optional[bool] = _UNSET,
**kwargs,
) -> Awaitable[Response]:
return super().put(
return cast(
Awaitable[Response],
super().put(
url,
data,
json,
@@ -988,6 +1009,7 @@ class AsyncFetcherClient(FetcherSession):
http3,
stealthy_headers,
**kwargs,
),
)
def delete(
@@ -1014,7 +1036,9 @@ class AsyncFetcherClient(FetcherSession):
stealthy_headers: Optional[bool] = _UNSET,
**kwargs,
) -> Awaitable[Response]:
return super().delete(
return cast(
Awaitable[Response],
super().delete(
url,
data,
json,
@@ -1036,4 +1060,5 @@ class AsyncFetcherClient(FetcherSession):
http3,
stealthy_headers,
**kwargs,
),
)