freedak/Scrapling
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

style(Fetcher): correcting type hints and annotation

Browse Source
This commit is contained in:
Karim shoair
2025-10-01 06:18:23 +03:00
parent 849913ad77
commit 5127d386c9
1 changed files with 213 additions and 188 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scrapling/engines/static.py
+49 -24
View File
@@ -1,7 +1,7 @@
from time import sleep as time_sleep from time import sleep as time_sleep
from asyncio import sleep as asyncio_sleep from asyncio import sleep as asyncio_sleep
from curl_cffi.requests.session import CurlError from curl_cffi.curl import CurlError
from curl_cffi import CurlHttpVersion from curl_cffi import CurlHttpVersion
from curl_cffi.requests.impersonate import DEFAULT_CHROME from curl_cffi.requests.impersonate import DEFAULT_CHROME
from curl_cffi.requests import ( from curl_cffi.requests import (
@@ -22,13 +22,14 @@ from scrapling.core._types import (
Awaitable, Awaitable,
List, List,
Any, Any,
cast,
) )
from .toolbelt.custom import Response from .toolbelt.custom import Response
from .toolbelt.convertor import ResponseFactory from .toolbelt.convertor import ResponseFactory
from .toolbelt.fingerprints import generate_convincing_referer, generate_headers, __default_useragent__ from .toolbelt.fingerprints import generate_convincing_referer, generate_headers, __default_useragent__
_UNSET = object() _UNSET: Any = object()
class FetcherSession: class FetcherSession:
@@ -94,8 +95,8 @@ class FetcherSession:
self.default_http3 = http3 self.default_http3 = http3
self.selector_config = selector_config or {} self.selector_config = selector_config or {}
self._curl_session: Optional[CurlSession] | bool = None self._curl_session: Optional[CurlSession] = None
self._async_curl_session: Optional[AsyncCurlSession] | bool = None self._async_curl_session: Optional[AsyncCurlSession] = None
def _merge_request_args(self, **kwargs) -> Dict[str, Any]: def _merge_request_args(self, **kwargs) -> Dict[str, Any]:
"""Merge request-specific arguments with default session arguments.""" """Merge request-specific arguments with default session arguments."""
@@ -233,7 +234,7 @@ class FetcherSession:
request_args: Dict[str, Any], request_args: Dict[str, Any],
max_retries: int, max_retries: int,
retry_delay: int, retry_delay: int,
selector_config: Optional[Dict] = None, selector_config: Dict,
) -> Response: ) -> Response:
""" """
Perform an HTTP request using the configured session. Perform an HTTP request using the configured session.
@@ -273,7 +274,7 @@ class FetcherSession:
request_args: Dict[str, Any], request_args: Dict[str, Any],
max_retries: int, max_retries: int,
retry_delay: int, retry_delay: int,
selector_config: Optional[Dict] = None, selector_config: Dict,
) -> Response: ) -> Response:
""" """
Perform an HTTP request using the configured session. Perform an HTTP request using the configured session.
@@ -644,11 +645,11 @@ class FetcherSession:
class FetcherClient(FetcherSession): class FetcherClient(FetcherSession):
def __init__(self, *args, **kwargs): def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs) super().__init__(*args, **kwargs)
self.__enter__ = None self.__enter__: Any = None
self.__exit__ = None self.__exit__: Any = None
self.__aenter__ = None self.__aenter__: Any = None
self.__aexit__ = None self.__aexit__: Any = None
self._curl_session = True self._curl_session: Any = True
# Setting the correct return types for the type checking/autocompletion # Setting the correct return types for the type checking/autocompletion
def get( def get(
@@ -673,7 +674,9 @@ class FetcherClient(FetcherSession):
stealthy_headers: Optional[bool] = _UNSET, stealthy_headers: Optional[bool] = _UNSET,
**kwargs, **kwargs,
) -> Response: ) -> Response:
return super().get( return cast(
Response,
super().get(
url, url,
params, params,
headers, headers,
@@ -693,6 +696,7 @@ class FetcherClient(FetcherSession):
http3, http3,
stealthy_headers, stealthy_headers,
**kwargs, **kwargs,
),
) )
def post( def post(
@@ -719,7 +723,9 @@ class FetcherClient(FetcherSession):
stealthy_headers: Optional[bool] = _UNSET, stealthy_headers: Optional[bool] = _UNSET,
**kwargs, **kwargs,
) -> Response: ) -> Response:
return super().post( return cast(
Response,
super().post(
url, url,
data, data,
json, json,
@@ -741,6 +747,7 @@ class FetcherClient(FetcherSession):
http3, http3,
stealthy_headers, stealthy_headers,
**kwargs, **kwargs,
),
) )
def put( def put(
@@ -767,7 +774,9 @@ class FetcherClient(FetcherSession):
stealthy_headers: Optional[bool] = _UNSET, stealthy_headers: Optional[bool] = _UNSET,
**kwargs, **kwargs,
) -> Response: ) -> Response:
return super().put( return cast(
Response,
super().put(
url, url,
data, data,
json, json,
@@ -789,6 +798,7 @@ class FetcherClient(FetcherSession):
http3, http3,
stealthy_headers, stealthy_headers,
**kwargs, **kwargs,
),
) )
def delete( def delete(
@@ -815,7 +825,9 @@ class FetcherClient(FetcherSession):
stealthy_headers: Optional[bool] = _UNSET, stealthy_headers: Optional[bool] = _UNSET,
**kwargs, **kwargs,
) -> Response: ) -> Response:
return super().delete( return cast(
Response,
super().delete(
url, url,
data, data,
json, json,
@@ -837,17 +849,18 @@ class FetcherClient(FetcherSession):
http3, http3,
stealthy_headers, stealthy_headers,
**kwargs, **kwargs,
),
) )
class AsyncFetcherClient(FetcherSession): class AsyncFetcherClient(FetcherSession):
def __init__(self, *args, **kwargs): def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs) super().__init__(*args, **kwargs)
self.__enter__ = None self.__enter__: Any = None
self.__exit__ = None self.__exit__: Any = None
self.__aenter__ = None self.__aenter__: Any = None
self.__aexit__ = None self.__aexit__: Any = None
self._async_curl_session = True self._async_curl_session: Any = True
# Setting the correct return types for the type checking/autocompletion # Setting the correct return types for the type checking/autocompletion
def get( def get(
@@ -872,7 +885,9 @@ class AsyncFetcherClient(FetcherSession):
stealthy_headers: Optional[bool] = _UNSET, stealthy_headers: Optional[bool] = _UNSET,
**kwargs, **kwargs,
) -> Awaitable[Response]: ) -> Awaitable[Response]:
return super().get( return cast(
Awaitable[Response],
super().get(
url, url,
params, params,
headers, headers,
@@ -892,6 +907,7 @@ class AsyncFetcherClient(FetcherSession):
http3, http3,
stealthy_headers, stealthy_headers,
**kwargs, **kwargs,
),
) )
def post( def post(
@@ -918,7 +934,9 @@ class AsyncFetcherClient(FetcherSession):
stealthy_headers: Optional[bool] = _UNSET, stealthy_headers: Optional[bool] = _UNSET,
**kwargs, **kwargs,
) -> Awaitable[Response]: ) -> Awaitable[Response]:
return super().post( return cast(
Awaitable[Response],
super().post(
url, url,
data, data,
json, json,
@@ -940,6 +958,7 @@ class AsyncFetcherClient(FetcherSession):
http3, http3,
stealthy_headers, stealthy_headers,
**kwargs, **kwargs,
),
) )
def put( def put(
@@ -966,7 +985,9 @@ class AsyncFetcherClient(FetcherSession):
stealthy_headers: Optional[bool] = _UNSET, stealthy_headers: Optional[bool] = _UNSET,
**kwargs, **kwargs,
) -> Awaitable[Response]: ) -> Awaitable[Response]:
return super().put( return cast(
Awaitable[Response],
super().put(
url, url,
data, data,
json, json,
@@ -988,6 +1009,7 @@ class AsyncFetcherClient(FetcherSession):
http3, http3,
stealthy_headers, stealthy_headers,
**kwargs, **kwargs,
),
) )
def delete( def delete(
@@ -1014,7 +1036,9 @@ class AsyncFetcherClient(FetcherSession):
stealthy_headers: Optional[bool] = _UNSET, stealthy_headers: Optional[bool] = _UNSET,
**kwargs, **kwargs,
) -> Awaitable[Response]: ) -> Awaitable[Response]:
return super().delete( return cast(
Awaitable[Response],
super().delete(
url, url,
data, data,
json, json,
@@ -1036,4 +1060,5 @@ class AsyncFetcherClient(FetcherSession):
http3, http3,
stealthy_headers, stealthy_headers,
**kwargs, **kwargs,
),
) )