feat(Fetcher): Replacing httpx + Adding FetcherSession

Check out the discord server for details

scrapling/core/_types.py

@@ -18,8 +18,11 @@ from typing import (
     TypeVar,
     Union,
     Match,
+    Mapping,
+    Awaitable,
 )
 
+SUPPORTED_HTTP_METHODS = Literal["GET", "POST", "PUT", "DELETE"]
 SelectorWaitStates = Literal["attached", "detached", "hidden", "visible"]
 StrOrBytes = Union[str, bytes]
 

scrapling/engines/__init__.py

@@ -1,7 +1,7 @@
 from .camo import CamoufoxEngine
 from .constants import DEFAULT_DISABLED_RESOURCES, DEFAULT_STEALTH_FLAGS
 from .pw import PlaywrightEngine
-from .static import StaticEngine
+from .static import FetcherSession, FetcherClient, AsyncFetcherClient
 from .toolbelt import check_if_engine_usable
 
 __all__ = ["CamoufoxEngine", "PlaywrightEngine"]

scrapling/fetchers.py

@@ -9,462 +9,34 @@ from scrapling.core._types import (
     Iterable,
 )
 from scrapling.engines import (
+    FetcherSession,
     CamoufoxEngine,
     PlaywrightEngine,
-    StaticEngine,
     check_if_engine_usable,
+    FetcherClient as _FetcherClient,
+    AsyncFetcherClient as _AsyncFetcherClient,
 )
 from scrapling.engines.toolbelt import BaseFetcher, Response
 
+__FetcherClientInstance__ = _FetcherClient()
+
 
 class Fetcher(BaseFetcher):
-    """A basic `Fetcher` class type that can only do basic GET, POST, PUT, and DELETE HTTP requests based on httpx.
+    """A basic `Fetcher` class type that can only do basic GET, POST, PUT, and DELETE HTTP requests based on `curl_cffi`."""
 
-    Any additional keyword arguments passed to the methods below are passed to the respective httpx's method directly.
+    get = __FetcherClientInstance__.get
-    """
+    post = __FetcherClientInstance__.post
-
+    put = __FetcherClientInstance__.put
-    @classmethod
+    delete = __FetcherClientInstance__.delete
-    def get(
-        cls,
-        url: str,
-        follow_redirects: bool = True,
-        timeout: Optional[Union[int, float]] = 10,
-        stealthy_headers: bool = True,
-        proxy: Optional[str] = None,
-        retries: Optional[int] = 3,
-        cookies: Optional[Dict] = None,
-        custom_config: Dict = None,
-        **kwargs: Dict,
-    ) -> Response:
-        """Make basic HTTP GET request for you but with some added flavors.
-
-        :param url: Target url.
-        :param follow_redirects: As the name says -- if enabled (default), redirects will be followed.
-        :param timeout: The time to wait for the request to finish in seconds. The default is 10 seconds.
-        :param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers and
-            create a referer header as if this request had came from Google's search of this URL's domain.
-        :param proxy: A string of a proxy to use for http and https requests, the format accepted is `http://username:password@localhost:8030`
-        :param retries: The number of retries to do through httpx if the request failed for any reason. The default is 3 retries.
-        :param cookies: Set cookies for the next request.
-        :param custom_config: A dictionary of custom parser arguments to use with this request. Any argument passed will override any class parameters values.
-        :param kwargs: Any additional keyword arguments are passed directly to `httpx.get()` function so check httpx documentation for details.
-        :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
-        """
-        if not custom_config:
-            custom_config = {}
-        elif not isinstance(custom_config, dict):
-            ValueError(
-                f"The custom parser config must be of type dictionary, got {cls.__class__}"
-            )
-
-        adaptor_arguments = tuple(
-            {**cls._generate_parser_arguments(), **custom_config}.items()
-        )
-
-        if not cookies:
-            cookies = {}
-        elif not isinstance(cookies, dict):
-            ValueError(f"The cookies must be of type dictionary, got {cls.__class__}")
-
-        response_object = StaticEngine(
-            url,
-            proxy,
-            stealthy_headers,
-            follow_redirects,
-            timeout,
-            retries,
-            tuple(cookies.items()),
-            adaptor_arguments=adaptor_arguments,
-        ).get(**kwargs)
-        return response_object
-
-    @classmethod
-    def post(
-        cls,
-        url: str,
-        follow_redirects: bool = True,
-        timeout: Optional[Union[int, float]] = 10,
-        stealthy_headers: bool = True,
-        proxy: Optional[str] = None,
-        retries: Optional[int] = 3,
-        cookies: Optional[Dict] = None,
-        custom_config: Dict = None,
-        **kwargs: Dict,
-    ) -> Response:
-        """Make basic HTTP POST request for you but with some added flavors.
-
-        :param url: Target url.
-        :param follow_redirects: As the name says -- if enabled (default), redirects will be followed.
-        :param timeout: The time to wait for the request to finish in seconds. The default is 10 seconds.
-        :param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers and
-            create a referer header as if this request came from Google's search of this URL's domain.
-        :param proxy: A string of a proxy to use for http and https requests, the format accepted is `http://username:password@localhost:8030`
-        :param retries: The number of retries to do through httpx if the request failed for any reason. The default is 3 retries.
-        :param cookies: Set cookies for the next request.
-        :param custom_config: A dictionary of custom parser arguments to use with this request. Any argument passed will override any class parameters values.
-        :param kwargs: Any additional keyword arguments are passed directly to `httpx.post()` function so check httpx documentation for details.
-        :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
-        """
-        if not custom_config:
-            custom_config = {}
-        elif not isinstance(custom_config, dict):
-            ValueError(
-                f"The custom parser config must be of type dictionary, got {cls.__class__}"
-            )
-
-        adaptor_arguments = tuple(
-            {**cls._generate_parser_arguments(), **custom_config}.items()
-        )
-
-        if not cookies:
-            cookies = {}
-        elif not isinstance(cookies, dict):
-            ValueError(f"The cookies must be of type dictionary, got {cls.__class__}")
-
-        response_object = StaticEngine(
-            url,
-            proxy,
-            stealthy_headers,
-            follow_redirects,
-            timeout,
-            retries,
-            tuple(cookies.items()),
-            adaptor_arguments=adaptor_arguments,
-        ).post(**kwargs)
-        return response_object
-
-    @classmethod
-    def put(
-        cls,
-        url: str,
-        follow_redirects: bool = True,
-        timeout: Optional[Union[int, float]] = 10,
-        stealthy_headers: bool = True,
-        proxy: Optional[str] = None,
-        retries: Optional[int] = 3,
-        cookies: Optional[Dict] = None,
-        custom_config: Dict = None,
-        **kwargs: Dict,
-    ) -> Response:
-        """Make basic HTTP PUT request for you but with some added flavors.
-
-        :param url: Target url
-        :param follow_redirects: As the name says -- if enabled (default), redirects will be followed.
-        :param timeout: The time to wait for the request to finish in seconds. The default is 10 seconds.
-        :param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers and
-            create a referer header as if this request came from Google's search of this URL's domain.
-        :param proxy: A string of a proxy to use for http and https requests, the format accepted is `http://username:password@localhost:8030`
-        :param retries: The number of retries to do through httpx if the request failed for any reason. The default is 3 retries.
-        :param cookies: Set cookies for the next request.
-        :param custom_config: A dictionary of custom parser arguments to use with this request. Any argument passed will override any class parameters values.
-        :param kwargs: Any additional keyword arguments are passed directly to `httpx.put()` function so check httpx documentation for details.
-
-        :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
-        """
-        if not custom_config:
-            custom_config = {}
-        elif not isinstance(custom_config, dict):
-            ValueError(
-                f"The custom parser config must be of type dictionary, got {cls.__class__}"
-            )
-
-        adaptor_arguments = tuple(
-            {**cls._generate_parser_arguments(), **custom_config}.items()
-        )
-
-        if not cookies:
-            cookies = {}
-        elif not isinstance(cookies, dict):
-            ValueError(f"The cookies must be of type dictionary, got {cls.__class__}")
-
-        response_object = StaticEngine(
-            url,
-            proxy,
-            stealthy_headers,
-            follow_redirects,
-            timeout,
-            retries,
-            tuple(cookies.items()),
-            adaptor_arguments=adaptor_arguments,
-        ).put(**kwargs)
-        return response_object
-
-    @classmethod
-    def delete(
-        cls,
-        url: str,
-        follow_redirects: bool = True,
-        timeout: Optional[Union[int, float]] = 10,
-        stealthy_headers: bool = True,
-        proxy: Optional[str] = None,
-        retries: Optional[int] = 3,
-        cookies: Optional[Dict] = None,
-        custom_config: Dict = None,
-        **kwargs: Dict,
-    ) -> Response:
-        """Make basic HTTP DELETE request for you but with some added flavors.
-
-        :param url: Target url
-        :param follow_redirects: As the name says -- if enabled (default), redirects will be followed.
-        :param timeout: The time to wait for the request to finish in seconds. The default is 10 seconds.
-        :param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers and
-            create a referer header as if this request came from Google's search of this URL's domain.
-        :param proxy: A string of a proxy to use for http and https requests, the format accepted is `http://username:password@localhost:8030`
-        :param retries: The number of retries to do through httpx if the request failed for any reason. The default is 3 retries.
-        :param cookies: Set cookies for the next request.
-        :param custom_config: A dictionary of custom parser arguments to use with this request. Any argument passed will override any class parameters values.
-        :param kwargs: Any additional keyword arguments are passed directly to `httpx.delete()` function so check httpx documentation for details.
-        :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
-        """
-        if not custom_config:
-            custom_config = {}
-        elif not isinstance(custom_config, dict):
-            ValueError(
-                f"The custom parser config must be of type dictionary, got {cls.__class__}"
-            )
-
-        adaptor_arguments = tuple(
-            {**cls._generate_parser_arguments(), **custom_config}.items()
-        )
-
-        if not cookies:
-            cookies = {}
-        elif not isinstance(cookies, dict):
-            ValueError(f"The cookies must be of type dictionary, got {cls.__class__}")
-
-        response_object = StaticEngine(
-            url,
-            proxy,
-            stealthy_headers,
-            follow_redirects,
-            timeout,
-            retries,
-            tuple(cookies.items()),
-            adaptor_arguments=adaptor_arguments,
-        ).delete(**kwargs)
-        return response_object
 
 
-class AsyncFetcher(Fetcher):
+class AsyncFetcher(BaseFetcher):
-    @classmethod
+    """A basic `Fetcher` class type that can only do basic GET, POST, PUT, and DELETE HTTP requests based on `curl_cffi`."""
-    async def get(
-        cls,
-        url: str,
-        follow_redirects: bool = True,
-        timeout: Optional[Union[int, float]] = 10,
-        stealthy_headers: bool = True,
-        proxy: Optional[str] = None,
-        retries: Optional[int] = 3,
-        cookies: Optional[Dict] = None,
-        custom_config: Dict = None,
-        **kwargs: Dict,
-    ) -> Response:
-        """Make basic HTTP GET request for you but with some added flavors.
 
-        :param url: Target url.
+    get = _AsyncFetcherClient.get
-        :param follow_redirects: As the name says -- if enabled (default), redirects will be followed.
+    post = _AsyncFetcherClient.post
-        :param timeout: The time to wait for the request to finish in seconds. The default is 10 seconds.
+    put = _AsyncFetcherClient.put
-        :param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers and
+    delete = _AsyncFetcherClient.delete
-            create a referer header as if this request had came from Google's search of this URL's domain.
-        :param proxy: A string of a proxy to use for http and https requests, the format accepted is `http://username:password@localhost:8030`
-        :param retries: The number of retries to do through httpx if the request failed for any reason. The default is 3 retries.
-        :param cookies: Set cookies for the next request.
-        :param custom_config: A dictionary of custom parser arguments to use with this request. Any argument passed will override any class parameters values.
-        :param kwargs: Any additional keyword arguments are passed directly to `httpx.get()` function so check httpx documentation for details.
-        :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
-        """
-        if not custom_config:
-            custom_config = {}
-        elif not isinstance(custom_config, dict):
-            ValueError(
-                f"The custom parser config must be of type dictionary, got {cls.__class__}"
-            )
-
-        adaptor_arguments = tuple(
-            {**cls._generate_parser_arguments(), **custom_config}.items()
-        )
-
-        if not cookies:
-            cookies = {}
-        elif not isinstance(cookies, dict):
-            ValueError(f"The cookies must be of type dictionary, got {cls.__class__}")
-
-        response_object = await StaticEngine(
-            url,
-            proxy,
-            stealthy_headers,
-            follow_redirects,
-            timeout,
-            retries=retries,
-            cookies=tuple(cookies.items()),
-            adaptor_arguments=adaptor_arguments,
-        ).async_get(**kwargs)
-        return response_object
-
-    @classmethod
-    async def post(
-        cls,
-        url: str,
-        follow_redirects: bool = True,
-        timeout: Optional[Union[int, float]] = 10,
-        stealthy_headers: bool = True,
-        proxy: Optional[str] = None,
-        retries: Optional[int] = 3,
-        cookies: Optional[Dict] = None,
-        custom_config: Dict = None,
-        **kwargs: Dict,
-    ) -> Response:
-        """Make basic HTTP POST request for you but with some added flavors.
-
-        :param url: Target url.
-        :param follow_redirects: As the name says -- if enabled (default), redirects will be followed.
-        :param timeout: The time to wait for the request to finish in seconds. The default is 10 seconds.
-        :param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers and
-            create a referer header as if this request came from Google's search of this URL's domain.
-        :param proxy: A string of a proxy to use for http and https requests, the format accepted is `http://username:password@localhost:8030`
-        :param retries: The number of retries to do through httpx if the request failed for any reason. The default is 3 retries.
-        :param cookies: Set cookies for the next request.
-        :param custom_config: A dictionary of custom parser arguments to use with this request. Any argument passed will override any class parameters values.
-        :param kwargs: Any additional keyword arguments are passed directly to `httpx.post()` function so check httpx documentation for details.
-        :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
-        """
-        if not custom_config:
-            custom_config = {}
-        elif not isinstance(custom_config, dict):
-            ValueError(
-                f"The custom parser config must be of type dictionary, got {cls.__class__}"
-            )
-
-        adaptor_arguments = tuple(
-            {**cls._generate_parser_arguments(), **custom_config}.items()
-        )
-
-        if not cookies:
-            cookies = {}
-        elif not isinstance(cookies, dict):
-            ValueError(f"The cookies must be of type dictionary, got {cls.__class__}")
-
-        response_object = await StaticEngine(
-            url,
-            proxy,
-            stealthy_headers,
-            follow_redirects,
-            timeout,
-            retries=retries,
-            cookies=tuple(cookies.items()),
-            adaptor_arguments=adaptor_arguments,
-        ).async_post(**kwargs)
-        return response_object
-
-    @classmethod
-    async def put(
-        cls,
-        url: str,
-        follow_redirects: bool = True,
-        timeout: Optional[Union[int, float]] = 10,
-        stealthy_headers: bool = True,
-        proxy: Optional[str] = None,
-        retries: Optional[int] = 3,
-        cookies: Optional[Dict] = None,
-        custom_config: Dict = None,
-        **kwargs: Dict,
-    ) -> Response:
-        """Make basic HTTP PUT request for you but with some added flavors.
-
-        :param url: Target url
-        :param follow_redirects: As the name says -- if enabled (default), redirects will be followed.
-        :param timeout: The time to wait for the request to finish in seconds. The default is 10 seconds.
-        :param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers and
-            create a referer header as if this request came from Google's search of this URL's domain.
-        :param proxy: A string of a proxy to use for http and https requests, the format accepted is `http://username:password@localhost:8030`
-        :param retries: The number of retries to do through httpx if the request failed for any reason. The default is 3 retries.
-        :param cookies: Set cookies for the next request.
-        :param custom_config: A dictionary of custom parser arguments to use with this request. Any argument passed will override any class parameters values.
-        :param kwargs: Any additional keyword arguments are passed directly to `httpx.put()` function so check httpx documentation for details.
-        :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
-        """
-        if not custom_config:
-            custom_config = {}
-        elif not isinstance(custom_config, dict):
-            ValueError(
-                f"The custom parser config must be of type dictionary, got {cls.__class__}"
-            )
-
-        adaptor_arguments = tuple(
-            {**cls._generate_parser_arguments(), **custom_config}.items()
-        )
-
-        if not cookies:
-            cookies = {}
-        elif not isinstance(cookies, dict):
-            ValueError(f"The cookies must be of type dictionary, got {cls.__class__}")
-
-        response_object = await StaticEngine(
-            url,
-            proxy,
-            stealthy_headers,
-            follow_redirects,
-            timeout,
-            retries=retries,
-            cookies=tuple(cookies.items()),
-            adaptor_arguments=adaptor_arguments,
-        ).async_put(**kwargs)
-        return response_object
-
-    @classmethod
-    async def delete(
-        cls,
-        url: str,
-        follow_redirects: bool = True,
-        timeout: Optional[Union[int, float]] = 10,
-        stealthy_headers: bool = True,
-        proxy: Optional[str] = None,
-        retries: Optional[int] = 3,
-        cookies: Optional[Dict] = None,
-        custom_config: Dict = None,
-        **kwargs: Dict,
-    ) -> Response:
-        """Make basic HTTP DELETE request for you but with some added flavors.
-
-        :param url: Target url
-        :param follow_redirects: As the name says -- if enabled (default), redirects will be followed.
-        :param timeout: The time to wait for the request to finish in seconds. The default is 10 seconds.
-        :param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers and
-            create a referer header as if this request came from Google's search of this URL's domain.
-        :param proxy: A string of a proxy to use for http and https requests, the format accepted is `http://username:password@localhost:8030`
-        :param retries: The number of retries to do through httpx if the request failed for any reason. The default is 3 retries.
-        :param cookies: Set cookies for the next request.
-        :param custom_config: A dictionary of custom parser arguments to use with this request. Any argument passed will override any class parameters values.
-        :param kwargs: Any additional keyword arguments are passed directly to `httpx.delete()` function so check httpx documentation for details.
-        :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
-        """
-        if not custom_config:
-            custom_config = {}
-        elif not isinstance(custom_config, dict):
-            ValueError(
-                f"The custom parser config must be of type dictionary, got {cls.__class__}"
-            )
-
-        adaptor_arguments = tuple(
-            {**cls._generate_parser_arguments(), **custom_config}.items()
-        )
-
-        if not cookies:
-            cookies = {}
-        elif not isinstance(cookies, dict):
-            ValueError(f"The cookies must be of type dictionary, got {cls.__class__}")
-
-        response_object = await StaticEngine(
-            url,
-            proxy,
-            stealthy_headers,
-            follow_redirects,
-            timeout,
-            retries=retries,
-            cookies=tuple(cookies.items()),
-            adaptor_arguments=adaptor_arguments,
-        ).async_delete(**kwargs)
-        return response_object
 
 
 class StealthyFetcher(BaseFetcher):

setup.py

@@ -48,7 +48,6 @@ setup(
         "Programming Language :: Python :: Implementation :: CPython",
         "Typing :: Typed",
     ],
-    # Instead of using requirements file to dodge possible errors from tox?
     install_requires=[
         "lxml>=5.0",
         "cssselect>=1.2",
@@ -56,7 +55,7 @@ setup(
         "click",
         "orjson>=3",
         "tldextract",
-        "httpx[brotli,zstd, socks]",
+        "curl_cffi>=0.11.1",
         "playwright>=1.49.1",
         "rebrowser-playwright>=1.49.1",
         "camoufox[geoip]>=0.4.11",