fix(Fetcher): Fix the issue of passing cookies

2025-04-29 03:01:19 +03:00
parent 98ad50d5aa
commit 1240ee3eb0
2 changed files with 81 additions and 2 deletions
@@ -17,6 +17,7 @@ class StaticEngine:
        follow_redirects: bool = True,
        timeout: Optional[Union[int, float]] = None,
        retries: Optional[int] = 3,
+        cookies: Optional[Dict] = None,
        adaptor_arguments: Tuple = None,
    ):
        """An engine that utilizes httpx library, check the `Fetcher` class for more documentation.
@@ -26,6 +27,7 @@ class StaticEngine:
            create a referer header as if this request had came from Google's search of this URL's domain.
        :param proxy: A string of a proxy to use for http and https requests, the format accepted is `http://username:password@localhost:8030`
        :param follow_redirects: As the name says -- if enabled (default), redirects will be followed.
+        :param cookies: Set cookies for the next request.
        :param timeout: The time to wait for the request to finish in seconds. The default is 10 seconds.
        :param adaptor_arguments: The arguments that will be passed in the end while creating the final Adaptor's class.
        """
@@ -35,6 +37,7 @@ class StaticEngine:
        self.timeout = timeout
        self.follow_redirects = bool(follow_redirects)
        self.retries = retries
+        self.cookies = dict(cookies) if cookies else {}
        self._extra_headers = generate_headers(browser_mode=False)
        # Because we are using `lru_cache` for a slight optimization but both dict/dict_items are not hashable so they can't be cached
        # So my solution here was to convert it to tuple then convert it back to dictionary again here as tuples are hashable, ofc `tuple().__hash__()`
@@ -98,7 +101,9 @@ class StaticEngine:
    def _make_request(self, method: str, **kwargs) -> Response:
        headers = self._headers_job(kwargs.pop("headers", {}))
        with httpx.Client(
-            proxy=self.proxy, transport=httpx.HTTPTransport(retries=self.retries)
+            proxy=self.proxy,
+            transport=httpx.HTTPTransport(retries=self.retries),
+            cookies=self.cookies,
        ) as client:
            request = getattr(client, method)(
                url=self.url,
@@ -112,7 +117,9 @@ class StaticEngine:
    async def _async_make_request(self, method: str, **kwargs) -> Response:
        headers = self._headers_job(kwargs.pop("headers", {}))
        async with httpx.AsyncClient(
-            proxy=self.proxy, transport=httpx.AsyncHTTPTransport(retries=self.retries)
+            proxy=self.proxy,
+            transport=httpx.AsyncHTTPTransport(retries=self.retries),
+            cookies=self.cookies,
        ) as client:
            request = await getattr(client, method)(
                url=self.url,
@@ -31,6 +31,7 @@ class Fetcher(BaseFetcher):
        stealthy_headers: bool = True,
        proxy: Optional[str] = None,
        retries: Optional[int] = 3,
+        cookies: Optional[Dict] = None,
        custom_config: Dict = None,
        **kwargs: Dict,
    ) -> Response:
@@ -43,6 +44,7 @@ class Fetcher(BaseFetcher):
            create a referer header as if this request had came from Google's search of this URL's domain.
        :param proxy: A string of a proxy to use for http and https requests, the format accepted is `http://username:password@localhost:8030`
        :param retries: The number of retries to do through httpx if the request failed for any reason. The default is 3 retries.
+        :param cookies: Set cookies for the next request.
        :param custom_config: A dictionary of custom parser arguments to use with this request. Any argument passed will override any class parameters values.
        :param kwargs: Any additional keyword arguments are passed directly to `httpx.get()` function so check httpx documentation for details.
        :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
@@ -57,6 +59,12 @@ class Fetcher(BaseFetcher):
        adaptor_arguments = tuple(
            {**cls._generate_parser_arguments(), **custom_config}.items()
        )
+
+        if not cookies:
+            cookies = {}
+        elif not isinstance(cookies, dict):
+            ValueError(f"The cookies must be of type dictionary, got {cls.__class__}")
+
        response_object = StaticEngine(
            url,
            proxy,
@@ -64,6 +72,7 @@ class Fetcher(BaseFetcher):
            follow_redirects,
            timeout,
            retries,
+            tuple(cookies.items()),
            adaptor_arguments=adaptor_arguments,
        ).get(**kwargs)
        return response_object
@@ -77,6 +86,7 @@ class Fetcher(BaseFetcher):
        stealthy_headers: bool = True,
        proxy: Optional[str] = None,
        retries: Optional[int] = 3,
+        cookies: Optional[Dict] = None,
        custom_config: Dict = None,
        **kwargs: Dict,
    ) -> Response:
@@ -89,6 +99,7 @@ class Fetcher(BaseFetcher):
            create a referer header as if this request came from Google's search of this URL's domain.
        :param proxy: A string of a proxy to use for http and https requests, the format accepted is `http://username:password@localhost:8030`
        :param retries: The number of retries to do through httpx if the request failed for any reason. The default is 3 retries.
+        :param cookies: Set cookies for the next request.
        :param custom_config: A dictionary of custom parser arguments to use with this request. Any argument passed will override any class parameters values.
        :param kwargs: Any additional keyword arguments are passed directly to `httpx.post()` function so check httpx documentation for details.
        :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
@@ -103,6 +114,12 @@ class Fetcher(BaseFetcher):
        adaptor_arguments = tuple(
            {**cls._generate_parser_arguments(), **custom_config}.items()
        )
+
+        if not cookies:
+            cookies = {}
+        elif not isinstance(cookies, dict):
+            ValueError(f"The cookies must be of type dictionary, got {cls.__class__}")
+
        response_object = StaticEngine(
            url,
            proxy,
@@ -110,6 +127,7 @@ class Fetcher(BaseFetcher):
            follow_redirects,
            timeout,
            retries,
+            tuple(cookies.items()),
            adaptor_arguments=adaptor_arguments,
        ).post(**kwargs)
        return response_object
@@ -123,6 +141,7 @@ class Fetcher(BaseFetcher):
        stealthy_headers: bool = True,
        proxy: Optional[str] = None,
        retries: Optional[int] = 3,
+        cookies: Optional[Dict] = None,
        custom_config: Dict = None,
        **kwargs: Dict,
    ) -> Response:
@@ -135,6 +154,7 @@ class Fetcher(BaseFetcher):
            create a referer header as if this request came from Google's search of this URL's domain.
        :param proxy: A string of a proxy to use for http and https requests, the format accepted is `http://username:password@localhost:8030`
        :param retries: The number of retries to do through httpx if the request failed for any reason. The default is 3 retries.
+        :param cookies: Set cookies for the next request.
        :param custom_config: A dictionary of custom parser arguments to use with this request. Any argument passed will override any class parameters values.
        :param kwargs: Any additional keyword arguments are passed directly to `httpx.put()` function so check httpx documentation for details.

@@ -150,6 +170,12 @@ class Fetcher(BaseFetcher):
        adaptor_arguments = tuple(
            {**cls._generate_parser_arguments(), **custom_config}.items()
        )
+
+        if not cookies:
+            cookies = {}
+        elif not isinstance(cookies, dict):
+            ValueError(f"The cookies must be of type dictionary, got {cls.__class__}")
+
        response_object = StaticEngine(
            url,
            proxy,
@@ -157,6 +183,7 @@ class Fetcher(BaseFetcher):
            follow_redirects,
            timeout,
            retries,
+            tuple(cookies.items()),
            adaptor_arguments=adaptor_arguments,
        ).put(**kwargs)
        return response_object
@@ -170,6 +197,7 @@ class Fetcher(BaseFetcher):
        stealthy_headers: bool = True,
        proxy: Optional[str] = None,
        retries: Optional[int] = 3,
+        cookies: Optional[Dict] = None,
        custom_config: Dict = None,
        **kwargs: Dict,
    ) -> Response:
@@ -182,6 +210,7 @@ class Fetcher(BaseFetcher):
            create a referer header as if this request came from Google's search of this URL's domain.
        :param proxy: A string of a proxy to use for http and https requests, the format accepted is `http://username:password@localhost:8030`
        :param retries: The number of retries to do through httpx if the request failed for any reason. The default is 3 retries.
+        :param cookies: Set cookies for the next request.
        :param custom_config: A dictionary of custom parser arguments to use with this request. Any argument passed will override any class parameters values.
        :param kwargs: Any additional keyword arguments are passed directly to `httpx.delete()` function so check httpx documentation for details.
        :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
@@ -196,6 +225,12 @@ class Fetcher(BaseFetcher):
        adaptor_arguments = tuple(
            {**cls._generate_parser_arguments(), **custom_config}.items()
        )
+
+        if not cookies:
+            cookies = {}
+        elif not isinstance(cookies, dict):
+            ValueError(f"The cookies must be of type dictionary, got {cls.__class__}")
+
        response_object = StaticEngine(
            url,
            proxy,
@@ -203,6 +238,7 @@ class Fetcher(BaseFetcher):
            follow_redirects,
            timeout,
            retries,
+            tuple(cookies.items()),
            adaptor_arguments=adaptor_arguments,
        ).delete(**kwargs)
        return response_object
@@ -218,6 +254,7 @@ class AsyncFetcher(Fetcher):
        stealthy_headers: bool = True,
        proxy: Optional[str] = None,
        retries: Optional[int] = 3,
+        cookies: Optional[Dict] = None,
        custom_config: Dict = None,
        **kwargs: Dict,
    ) -> Response:
@@ -230,6 +267,7 @@ class AsyncFetcher(Fetcher):
            create a referer header as if this request had came from Google's search of this URL's domain.
        :param proxy: A string of a proxy to use for http and https requests, the format accepted is `http://username:password@localhost:8030`
        :param retries: The number of retries to do through httpx if the request failed for any reason. The default is 3 retries.
+        :param cookies: Set cookies for the next request.
        :param custom_config: A dictionary of custom parser arguments to use with this request. Any argument passed will override any class parameters values.
        :param kwargs: Any additional keyword arguments are passed directly to `httpx.get()` function so check httpx documentation for details.
        :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
@@ -244,6 +282,12 @@ class AsyncFetcher(Fetcher):
        adaptor_arguments = tuple(
            {**cls._generate_parser_arguments(), **custom_config}.items()
        )
+
+        if not cookies:
+            cookies = {}
+        elif not isinstance(cookies, dict):
+            ValueError(f"The cookies must be of type dictionary, got {cls.__class__}")
+
        response_object = await StaticEngine(
            url,
            proxy,
@@ -251,6 +295,7 @@ class AsyncFetcher(Fetcher):
            follow_redirects,
            timeout,
            retries=retries,
+            cookies=tuple(cookies.items()),
            adaptor_arguments=adaptor_arguments,
        ).async_get(**kwargs)
        return response_object
@@ -264,6 +309,7 @@ class AsyncFetcher(Fetcher):
        stealthy_headers: bool = True,
        proxy: Optional[str] = None,
        retries: Optional[int] = 3,
+        cookies: Optional[Dict] = None,
        custom_config: Dict = None,
        **kwargs: Dict,
    ) -> Response:
@@ -276,6 +322,7 @@ class AsyncFetcher(Fetcher):
            create a referer header as if this request came from Google's search of this URL's domain.
        :param proxy: A string of a proxy to use for http and https requests, the format accepted is `http://username:password@localhost:8030`
        :param retries: The number of retries to do through httpx if the request failed for any reason. The default is 3 retries.
+        :param cookies: Set cookies for the next request.
        :param custom_config: A dictionary of custom parser arguments to use with this request. Any argument passed will override any class parameters values.
        :param kwargs: Any additional keyword arguments are passed directly to `httpx.post()` function so check httpx documentation for details.
        :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
@@ -290,6 +337,12 @@ class AsyncFetcher(Fetcher):
        adaptor_arguments = tuple(
            {**cls._generate_parser_arguments(), **custom_config}.items()
        )
+
+        if not cookies:
+            cookies = {}
+        elif not isinstance(cookies, dict):
+            ValueError(f"The cookies must be of type dictionary, got {cls.__class__}")
+
        response_object = await StaticEngine(
            url,
            proxy,
@@ -297,6 +350,7 @@ class AsyncFetcher(Fetcher):
            follow_redirects,
            timeout,
            retries=retries,
+            cookies=tuple(cookies.items()),
            adaptor_arguments=adaptor_arguments,
        ).async_post(**kwargs)
        return response_object
@@ -310,6 +364,7 @@ class AsyncFetcher(Fetcher):
        stealthy_headers: bool = True,
        proxy: Optional[str] = None,
        retries: Optional[int] = 3,
+        cookies: Optional[Dict] = None,
        custom_config: Dict = None,
        **kwargs: Dict,
    ) -> Response:
@@ -322,6 +377,7 @@ class AsyncFetcher(Fetcher):
            create a referer header as if this request came from Google's search of this URL's domain.
        :param proxy: A string of a proxy to use for http and https requests, the format accepted is `http://username:password@localhost:8030`
        :param retries: The number of retries to do through httpx if the request failed for any reason. The default is 3 retries.
+        :param cookies: Set cookies for the next request.
        :param custom_config: A dictionary of custom parser arguments to use with this request. Any argument passed will override any class parameters values.
        :param kwargs: Any additional keyword arguments are passed directly to `httpx.put()` function so check httpx documentation for details.
        :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
@@ -336,6 +392,12 @@ class AsyncFetcher(Fetcher):
        adaptor_arguments = tuple(
            {**cls._generate_parser_arguments(), **custom_config}.items()
        )
+
+        if not cookies:
+            cookies = {}
+        elif not isinstance(cookies, dict):
+            ValueError(f"The cookies must be of type dictionary, got {cls.__class__}")
+
        response_object = await StaticEngine(
            url,
            proxy,
@@ -343,6 +405,7 @@ class AsyncFetcher(Fetcher):
            follow_redirects,
            timeout,
            retries=retries,
+            cookies=tuple(cookies.items()),
            adaptor_arguments=adaptor_arguments,
        ).async_put(**kwargs)
        return response_object
@@ -356,6 +419,7 @@ class AsyncFetcher(Fetcher):
        stealthy_headers: bool = True,
        proxy: Optional[str] = None,
        retries: Optional[int] = 3,
+        cookies: Optional[Dict] = None,
        custom_config: Dict = None,
        **kwargs: Dict,
    ) -> Response:
@@ -368,6 +432,7 @@ class AsyncFetcher(Fetcher):
            create a referer header as if this request came from Google's search of this URL's domain.
        :param proxy: A string of a proxy to use for http and https requests, the format accepted is `http://username:password@localhost:8030`
        :param retries: The number of retries to do through httpx if the request failed for any reason. The default is 3 retries.
+        :param cookies: Set cookies for the next request.
        :param custom_config: A dictionary of custom parser arguments to use with this request. Any argument passed will override any class parameters values.
        :param kwargs: Any additional keyword arguments are passed directly to `httpx.delete()` function so check httpx documentation for details.
        :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
@@ -382,6 +447,12 @@ class AsyncFetcher(Fetcher):
        adaptor_arguments = tuple(
            {**cls._generate_parser_arguments(), **custom_config}.items()
        )
+
+        if not cookies:
+            cookies = {}
+        elif not isinstance(cookies, dict):
+            ValueError(f"The cookies must be of type dictionary, got {cls.__class__}")
+
        response_object = await StaticEngine(
            url,
            proxy,
@@ -389,6 +460,7 @@ class AsyncFetcher(Fetcher):
            follow_redirects,
            timeout,
            retries=retries,
+            cookies=tuple(cookies.items()),
            adaptor_arguments=adaptor_arguments,
        ).async_delete(**kwargs)
        return response_object