fix: solving a bug with using configure on Fetcher
This commit is contained in:
@@ -1,28 +1,65 @@
|
|||||||
|
from scrapling.core._types import Any, Awaitable, Unpack
|
||||||
|
from scrapling.engines._browsers._types import DataRequestParams, GetRequestParams
|
||||||
from scrapling.engines.static import (
|
from scrapling.engines.static import (
|
||||||
FetcherSession,
|
FetcherSession,
|
||||||
FetcherClient as _FetcherClient,
|
FetcherClient as _FetcherClient,
|
||||||
AsyncFetcherClient as _AsyncFetcherClient,
|
AsyncFetcherClient as _AsyncFetcherClient,
|
||||||
)
|
)
|
||||||
from scrapling.engines.toolbelt.custom import BaseFetcher
|
from scrapling.engines.toolbelt.custom import BaseFetcher, Response
|
||||||
|
|
||||||
|
__all__ = ["Fetcher", "AsyncFetcher", "FetcherSession"]
|
||||||
|
|
||||||
|
|
||||||
__FetcherClientInstance__ = _FetcherClient()
|
__FetcherClientInstance__ = _FetcherClient()
|
||||||
__AsyncFetcherClientInstance__ = _AsyncFetcherClient()
|
__AsyncFetcherClientInstance__ = _AsyncFetcherClient()
|
||||||
|
|
||||||
|
|
||||||
|
def _merge_selector_config(cls: type[BaseFetcher], kwargs: Any) -> Any:
|
||||||
|
"""Merge class-level parser arguments into per-request ``selector_config``.
|
||||||
|
|
||||||
|
Values from ``Fetcher.configure(...)`` act as the base; any explicit
|
||||||
|
``selector_config`` passed on the call overrides them.
|
||||||
|
"""
|
||||||
|
selector_config = kwargs.get("selector_config") or {}
|
||||||
|
kwargs["selector_config"] = {**cls._generate_parser_arguments(), **selector_config}
|
||||||
|
return kwargs
|
||||||
|
|
||||||
|
|
||||||
class Fetcher(BaseFetcher):
|
class Fetcher(BaseFetcher):
|
||||||
"""A basic `Fetcher` class type that can only do basic GET, POST, PUT, and DELETE HTTP requests based on `curl_cffi`."""
|
"""A basic `Fetcher` class type that can only do basic GET, POST, PUT, and DELETE HTTP requests based on `curl_cffi`."""
|
||||||
|
|
||||||
get = __FetcherClientInstance__.get
|
@classmethod
|
||||||
post = __FetcherClientInstance__.post
|
def get(cls, url: str, **kwargs: Unpack[GetRequestParams]) -> Response:
|
||||||
put = __FetcherClientInstance__.put
|
return __FetcherClientInstance__.get(url, **_merge_selector_config(cls, kwargs))
|
||||||
delete = __FetcherClientInstance__.delete
|
|
||||||
|
@classmethod
|
||||||
|
def post(cls, url: str, **kwargs: Unpack[DataRequestParams]) -> Response:
|
||||||
|
return __FetcherClientInstance__.post(url, **_merge_selector_config(cls, kwargs))
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def put(cls, url: str, **kwargs: Unpack[DataRequestParams]) -> Response:
|
||||||
|
return __FetcherClientInstance__.put(url, **_merge_selector_config(cls, kwargs))
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def delete(cls, url: str, **kwargs: Unpack[DataRequestParams]) -> Response:
|
||||||
|
return __FetcherClientInstance__.delete(url, **_merge_selector_config(cls, kwargs))
|
||||||
|
|
||||||
|
|
||||||
class AsyncFetcher(BaseFetcher):
|
class AsyncFetcher(BaseFetcher):
|
||||||
"""A basic `Fetcher` class type that can only do basic GET, POST, PUT, and DELETE HTTP requests based on `curl_cffi`."""
|
"""A basic `Fetcher` class type that can only do basic GET, POST, PUT, and DELETE HTTP requests based on `curl_cffi`."""
|
||||||
|
|
||||||
get = __AsyncFetcherClientInstance__.get
|
@classmethod
|
||||||
post = __AsyncFetcherClientInstance__.post
|
def get(cls, url: str, **kwargs: Unpack[GetRequestParams]) -> Awaitable[Response]:
|
||||||
put = __AsyncFetcherClientInstance__.put
|
return __AsyncFetcherClientInstance__.get(url, **_merge_selector_config(cls, kwargs))
|
||||||
delete = __AsyncFetcherClientInstance__.delete
|
|
||||||
|
@classmethod
|
||||||
|
def post(cls, url: str, **kwargs: Unpack[DataRequestParams]) -> Awaitable[Response]:
|
||||||
|
return __AsyncFetcherClientInstance__.post(url, **_merge_selector_config(cls, kwargs))
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def put(cls, url: str, **kwargs: Unpack[DataRequestParams]) -> Awaitable[Response]:
|
||||||
|
return __AsyncFetcherClientInstance__.put(url, **_merge_selector_config(cls, kwargs))
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def delete(cls, url: str, **kwargs: Unpack[DataRequestParams]) -> Awaitable[Response]:
|
||||||
|
return __AsyncFetcherClientInstance__.delete(url, **_merge_selector_config(cls, kwargs))
|
||||||
|
|||||||
Reference in New Issue
Block a user