freedak/Scrapling
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

style(Fetcher): correcting type hints and annotation

Browse Source
This commit is contained in:
Karim shoair
2025-10-01 06:18:23 +03:00
parent 849913ad77
commit 5127d386c9
1 changed files with 213 additions and 188 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scrapling/engines/static.py
+213 -188
View File
@@ -1,7 +1,7 @@
from time import sleep as time_sleep from time import sleep as time_sleep
from asyncio import sleep as asyncio_sleep from asyncio import sleep as asyncio_sleep
from curl_cffi.requests.session import CurlError from curl_cffi.curl import CurlError
from curl_cffi import CurlHttpVersion from curl_cffi import CurlHttpVersion
from curl_cffi.requests.impersonate import DEFAULT_CHROME from curl_cffi.requests.impersonate import DEFAULT_CHROME
from curl_cffi.requests import ( from curl_cffi.requests import (
@@ -22,13 +22,14 @@ from scrapling.core._types import (
Awaitable, Awaitable,
List, List,
Any, Any,
cast,
) )
from .toolbelt.custom import Response from .toolbelt.custom import Response
from .toolbelt.convertor import ResponseFactory from .toolbelt.convertor import ResponseFactory
from .toolbelt.fingerprints import generate_convincing_referer, generate_headers, __default_useragent__ from .toolbelt.fingerprints import generate_convincing_referer, generate_headers, __default_useragent__
_UNSET = object() _UNSET: Any = object()
class FetcherSession: class FetcherSession:
@@ -94,8 +95,8 @@ class FetcherSession:
self.default_http3 = http3 self.default_http3 = http3
self.selector_config = selector_config or {} self.selector_config = selector_config or {}
self._curl_session: Optional[CurlSession] | bool = None self._curl_session: Optional[CurlSession] = None
self._async_curl_session: Optional[AsyncCurlSession] | bool = None self._async_curl_session: Optional[AsyncCurlSession] = None
def _merge_request_args(self, **kwargs) -> Dict[str, Any]: def _merge_request_args(self, **kwargs) -> Dict[str, Any]:
"""Merge request-specific arguments with default session arguments.""" """Merge request-specific arguments with default session arguments."""
@@ -233,7 +234,7 @@ class FetcherSession:
request_args: Dict[str, Any], request_args: Dict[str, Any],
max_retries: int, max_retries: int,
retry_delay: int, retry_delay: int,
selector_config: Optional[Dict] = None, selector_config: Dict,
) -> Response: ) -> Response:
""" """
Perform an HTTP request using the configured session. Perform an HTTP request using the configured session.
@@ -273,7 +274,7 @@ class FetcherSession:
request_args: Dict[str, Any], request_args: Dict[str, Any],
max_retries: int, max_retries: int,
retry_delay: int, retry_delay: int,
selector_config: Optional[Dict] = None, selector_config: Dict,
) -> Response: ) -> Response:
""" """
Perform an HTTP request using the configured session. Perform an HTTP request using the configured session.
@@ -644,11 +645,11 @@ class FetcherSession:
class FetcherClient(FetcherSession): class FetcherClient(FetcherSession):
def __init__(self, *args, **kwargs): def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs) super().__init__(*args, **kwargs)
self.__enter__ = None self.__enter__: Any = None
self.__exit__ = None self.__exit__: Any = None
self.__aenter__ = None self.__aenter__: Any = None
self.__aexit__ = None self.__aexit__: Any = None
self._curl_session = True self._curl_session: Any = True
# Setting the correct return types for the type checking/autocompletion # Setting the correct return types for the type checking/autocompletion
def get( def get(
@@ -673,26 +674,29 @@ class FetcherClient(FetcherSession):
stealthy_headers: Optional[bool] = _UNSET, stealthy_headers: Optional[bool] = _UNSET,
**kwargs, **kwargs,
) -> Response: ) -> Response:
return super().get( return cast(
url, Response,
params, super().get(
headers, url,
cookies, params,
timeout, headers,
follow_redirects, cookies,
max_redirects, timeout,
retries, follow_redirects,
retry_delay, max_redirects,
proxies, retries,
proxy, retry_delay,
proxy_auth, proxies,
auth, proxy,
verify, proxy_auth,
cert, auth,
impersonate, verify,
http3, cert,
stealthy_headers, impersonate,
**kwargs, http3,
stealthy_headers,
**kwargs,
),
) )
def post( def post(
@@ -719,28 +723,31 @@ class FetcherClient(FetcherSession):
stealthy_headers: Optional[bool] = _UNSET, stealthy_headers: Optional[bool] = _UNSET,
**kwargs, **kwargs,
) -> Response: ) -> Response:
return super().post( return cast(
url, Response,
data, super().post(
json, url,
headers, data,
params, json,
cookies, headers,
timeout, params,
follow_redirects, cookies,
max_redirects, timeout,
retries, follow_redirects,
retry_delay, max_redirects,
proxies, retries,
proxy, retry_delay,
proxy_auth, proxies,
auth, proxy,
verify, proxy_auth,
cert, auth,
impersonate, verify,
http3, cert,
stealthy_headers, impersonate,
**kwargs, http3,
stealthy_headers,
**kwargs,
),
) )
def put( def put(
@@ -767,28 +774,31 @@ class FetcherClient(FetcherSession):
stealthy_headers: Optional[bool] = _UNSET, stealthy_headers: Optional[bool] = _UNSET,
**kwargs, **kwargs,
) -> Response: ) -> Response:
return super().put( return cast(
url, Response,
data, super().put(
json, url,
headers, data,
params, json,
cookies, headers,
timeout, params,
follow_redirects, cookies,
max_redirects, timeout,
retries, follow_redirects,
retry_delay, max_redirects,
proxies, retries,
proxy, retry_delay,
proxy_auth, proxies,
auth, proxy,
verify, proxy_auth,
cert, auth,
impersonate, verify,
http3, cert,
stealthy_headers, impersonate,
**kwargs, http3,
stealthy_headers,
**kwargs,
),
) )
def delete( def delete(
@@ -815,39 +825,42 @@ class FetcherClient(FetcherSession):
stealthy_headers: Optional[bool] = _UNSET, stealthy_headers: Optional[bool] = _UNSET,
**kwargs, **kwargs,
) -> Response: ) -> Response:
return super().delete( return cast(
url, Response,
data, super().delete(
json, url,
headers, data,
params, json,
cookies, headers,
timeout, params,
follow_redirects, cookies,
max_redirects, timeout,
retries, follow_redirects,
retry_delay, max_redirects,
proxies, retries,
proxy, retry_delay,
proxy_auth, proxies,
auth, proxy,
verify, proxy_auth,
cert, auth,
impersonate, verify,
http3, cert,
stealthy_headers, impersonate,
**kwargs, http3,
stealthy_headers,
**kwargs,
),
) )
class AsyncFetcherClient(FetcherSession): class AsyncFetcherClient(FetcherSession):
def __init__(self, *args, **kwargs): def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs) super().__init__(*args, **kwargs)
self.__enter__ = None self.__enter__: Any = None
self.__exit__ = None self.__exit__: Any = None
self.__aenter__ = None self.__aenter__: Any = None
self.__aexit__ = None self.__aexit__: Any = None
self._async_curl_session = True self._async_curl_session: Any = True
# Setting the correct return types for the type checking/autocompletion # Setting the correct return types for the type checking/autocompletion
def get( def get(
@@ -872,26 +885,29 @@ class AsyncFetcherClient(FetcherSession):
stealthy_headers: Optional[bool] = _UNSET, stealthy_headers: Optional[bool] = _UNSET,
**kwargs, **kwargs,
) -> Awaitable[Response]: ) -> Awaitable[Response]:
return super().get( return cast(
url, Awaitable[Response],
params, super().get(
headers, url,
cookies, params,
timeout, headers,
follow_redirects, cookies,
max_redirects, timeout,
retries, follow_redirects,
retry_delay, max_redirects,
proxies, retries,
proxy, retry_delay,
proxy_auth, proxies,
auth, proxy,
verify, proxy_auth,
cert, auth,
impersonate, verify,
http3, cert,
stealthy_headers, impersonate,
**kwargs, http3,
stealthy_headers,
**kwargs,
),
) )
def post( def post(
@@ -918,28 +934,31 @@ class AsyncFetcherClient(FetcherSession):
stealthy_headers: Optional[bool] = _UNSET, stealthy_headers: Optional[bool] = _UNSET,
**kwargs, **kwargs,
) -> Awaitable[Response]: ) -> Awaitable[Response]:
return super().post( return cast(
url, Awaitable[Response],
data, super().post(
json, url,
headers, data,
params, json,
cookies, headers,
timeout, params,
follow_redirects, cookies,
max_redirects, timeout,
retries, follow_redirects,
retry_delay, max_redirects,
proxies, retries,
proxy, retry_delay,
proxy_auth, proxies,
auth, proxy,
verify, proxy_auth,
cert, auth,
impersonate, verify,
http3, cert,
stealthy_headers, impersonate,
**kwargs, http3,
stealthy_headers,
**kwargs,
),
) )
def put( def put(
@@ -966,28 +985,31 @@ class AsyncFetcherClient(FetcherSession):
stealthy_headers: Optional[bool] = _UNSET, stealthy_headers: Optional[bool] = _UNSET,
**kwargs, **kwargs,
) -> Awaitable[Response]: ) -> Awaitable[Response]:
return super().put( return cast(
url, Awaitable[Response],
data, super().put(
json, url,
headers, data,
params, json,
cookies, headers,
timeout, params,
follow_redirects, cookies,
max_redirects, timeout,
retries, follow_redirects,
retry_delay, max_redirects,
proxies, retries,
proxy, retry_delay,
proxy_auth, proxies,
auth, proxy,
verify, proxy_auth,
cert, auth,
impersonate, verify,
http3, cert,
stealthy_headers, impersonate,
**kwargs, http3,
stealthy_headers,
**kwargs,
),
) )
def delete( def delete(
@@ -1014,26 +1036,29 @@ class AsyncFetcherClient(FetcherSession):
stealthy_headers: Optional[bool] = _UNSET, stealthy_headers: Optional[bool] = _UNSET,
**kwargs, **kwargs,
) -> Awaitable[Response]: ) -> Awaitable[Response]:
return super().delete( return cast(
url, Awaitable[Response],
data, super().delete(
json, url,
headers, data,
params, json,
cookies, headers,
timeout, params,
follow_redirects, cookies,
max_redirects, timeout,
retries, follow_redirects,
retry_delay, max_redirects,
proxies, retries,
proxy, retry_delay,
proxy_auth, proxies,
auth, proxy,
verify, proxy_auth,
cert, auth,
impersonate, verify,
http3, cert,
stealthy_headers, impersonate,
**kwargs, http3,
stealthy_headers,
**kwargs,
),
) )