From 35032d6f5fef4c8989262bf7497ce1f38d1af885 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Wed, 22 Apr 2026 15:31:18 +0200 Subject: [PATCH] fix: solving a bug with using `configure` on Fetcher --- scrapling/fetchers/requests.py | 55 ++++++++++++++++++++++++++++------ 1 file changed, 46 insertions(+), 9 deletions(-) diff --git a/scrapling/fetchers/requests.py b/scrapling/fetchers/requests.py index b559cd6..ec629a4 100644 --- a/scrapling/fetchers/requests.py +++ b/scrapling/fetchers/requests.py @@ -1,28 +1,65 @@ +from scrapling.core._types import Any, Awaitable, Unpack +from scrapling.engines._browsers._types import DataRequestParams, GetRequestParams from scrapling.engines.static import ( FetcherSession, FetcherClient as _FetcherClient, AsyncFetcherClient as _AsyncFetcherClient, ) -from scrapling.engines.toolbelt.custom import BaseFetcher +from scrapling.engines.toolbelt.custom import BaseFetcher, Response + +__all__ = ["Fetcher", "AsyncFetcher", "FetcherSession"] __FetcherClientInstance__ = _FetcherClient() __AsyncFetcherClientInstance__ = _AsyncFetcherClient() +def _merge_selector_config(cls: type[BaseFetcher], kwargs: Any) -> Any: + """Merge class-level parser arguments into per-request ``selector_config``. + + Values from ``Fetcher.configure(...)`` act as the base; any explicit + ``selector_config`` passed on the call overrides them. + """ + selector_config = kwargs.get("selector_config") or {} + kwargs["selector_config"] = {**cls._generate_parser_arguments(), **selector_config} + return kwargs + + class Fetcher(BaseFetcher): """A basic `Fetcher` class type that can only do basic GET, POST, PUT, and DELETE HTTP requests based on `curl_cffi`.""" - get = __FetcherClientInstance__.get - post = __FetcherClientInstance__.post - put = __FetcherClientInstance__.put - delete = __FetcherClientInstance__.delete + @classmethod + def get(cls, url: str, **kwargs: Unpack[GetRequestParams]) -> Response: + return __FetcherClientInstance__.get(url, **_merge_selector_config(cls, kwargs)) + + @classmethod + def post(cls, url: str, **kwargs: Unpack[DataRequestParams]) -> Response: + return __FetcherClientInstance__.post(url, **_merge_selector_config(cls, kwargs)) + + @classmethod + def put(cls, url: str, **kwargs: Unpack[DataRequestParams]) -> Response: + return __FetcherClientInstance__.put(url, **_merge_selector_config(cls, kwargs)) + + @classmethod + def delete(cls, url: str, **kwargs: Unpack[DataRequestParams]) -> Response: + return __FetcherClientInstance__.delete(url, **_merge_selector_config(cls, kwargs)) class AsyncFetcher(BaseFetcher): """A basic `Fetcher` class type that can only do basic GET, POST, PUT, and DELETE HTTP requests based on `curl_cffi`.""" - get = __AsyncFetcherClientInstance__.get - post = __AsyncFetcherClientInstance__.post - put = __AsyncFetcherClientInstance__.put - delete = __AsyncFetcherClientInstance__.delete + @classmethod + def get(cls, url: str, **kwargs: Unpack[GetRequestParams]) -> Awaitable[Response]: + return __AsyncFetcherClientInstance__.get(url, **_merge_selector_config(cls, kwargs)) + + @classmethod + def post(cls, url: str, **kwargs: Unpack[DataRequestParams]) -> Awaitable[Response]: + return __AsyncFetcherClientInstance__.post(url, **_merge_selector_config(cls, kwargs)) + + @classmethod + def put(cls, url: str, **kwargs: Unpack[DataRequestParams]) -> Awaitable[Response]: + return __AsyncFetcherClientInstance__.put(url, **_merge_selector_config(cls, kwargs)) + + @classmethod + def delete(cls, url: str, **kwargs: Unpack[DataRequestParams]) -> Awaitable[Response]: + return __AsyncFetcherClientInstance__.delete(url, **_merge_selector_config(cls, kwargs))