fix(Fetcher): Fix the issue of passing cookies

scrapling/engines/static.py

@@ -17,6 +17,7 @@ class StaticEngine:
         follow_redirects: bool = True,
         timeout: Optional[Union[int, float]] = None,
         retries: Optional[int] = 3,
+        cookies: Optional[Dict] = None,
         adaptor_arguments: Tuple = None,
     ):
         """An engine that utilizes httpx library, check the `Fetcher` class for more documentation.
@@ -26,6 +27,7 @@ class StaticEngine:
             create a referer header as if this request had came from Google's search of this URL's domain.
         :param proxy: A string of a proxy to use for http and https requests, the format accepted is `http://username:password@localhost:8030`
         :param follow_redirects: As the name says -- if enabled (default), redirects will be followed.
+        :param cookies: Set cookies for the next request.
         :param timeout: The time to wait for the request to finish in seconds. The default is 10 seconds.
         :param adaptor_arguments: The arguments that will be passed in the end while creating the final Adaptor's class.
         """
@@ -35,6 +37,7 @@ class StaticEngine:
         self.timeout = timeout
         self.follow_redirects = bool(follow_redirects)
         self.retries = retries
+        self.cookies = dict(cookies) if cookies else {}
         self._extra_headers = generate_headers(browser_mode=False)
         # Because we are using `lru_cache` for a slight optimization but both dict/dict_items are not hashable so they can't be cached
         # So my solution here was to convert it to tuple then convert it back to dictionary again here as tuples are hashable, ofc `tuple().__hash__()`
@@ -98,7 +101,9 @@ class StaticEngine:
     def _make_request(self, method: str, **kwargs) -> Response:
         headers = self._headers_job(kwargs.pop("headers", {}))
         with httpx.Client(
-            proxy=self.proxy, transport=httpx.HTTPTransport(retries=self.retries)
+            proxy=self.proxy,
+            transport=httpx.HTTPTransport(retries=self.retries),
+            cookies=self.cookies,
         ) as client:
             request = getattr(client, method)(
                 url=self.url,
@@ -112,7 +117,9 @@ class StaticEngine:
     async def _async_make_request(self, method: str, **kwargs) -> Response:
         headers = self._headers_job(kwargs.pop("headers", {}))
         async with httpx.AsyncClient(
-            proxy=self.proxy, transport=httpx.AsyncHTTPTransport(retries=self.retries)
+            proxy=self.proxy,
+            transport=httpx.AsyncHTTPTransport(retries=self.retries),
+            cookies=self.cookies,
         ) as client:
             request = await getattr(client, method)(
                 url=self.url,

scrapling/fetchers.py

@@ -31,6 +31,7 @@ class Fetcher(BaseFetcher):
         stealthy_headers: bool = True,
         proxy: Optional[str] = None,
         retries: Optional[int] = 3,
+        cookies: Optional[Dict] = None,
         custom_config: Dict = None,
         **kwargs: Dict,
     ) -> Response:
@@ -43,6 +44,7 @@ class Fetcher(BaseFetcher):
             create a referer header as if this request had came from Google's search of this URL's domain.
         :param proxy: A string of a proxy to use for http and https requests, the format accepted is `http://username:password@localhost:8030`
         :param retries: The number of retries to do through httpx if the request failed for any reason. The default is 3 retries.
+        :param cookies: Set cookies for the next request.
         :param custom_config: A dictionary of custom parser arguments to use with this request. Any argument passed will override any class parameters values.
         :param kwargs: Any additional keyword arguments are passed directly to `httpx.get()` function so check httpx documentation for details.
         :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
@@ -57,6 +59,12 @@ class Fetcher(BaseFetcher):
         adaptor_arguments = tuple(
             {**cls._generate_parser_arguments(), **custom_config}.items()
         )
+
+        if not cookies:
+            cookies = {}
+        elif not isinstance(cookies, dict):
+            ValueError(f"The cookies must be of type dictionary, got {cls.__class__}")
+
         response_object = StaticEngine(
             url,
             proxy,
@@ -64,6 +72,7 @@ class Fetcher(BaseFetcher):
             follow_redirects,
             timeout,
             retries,
+            tuple(cookies.items()),
             adaptor_arguments=adaptor_arguments,
         ).get(**kwargs)
         return response_object
@@ -77,6 +86,7 @@ class Fetcher(BaseFetcher):
         stealthy_headers: bool = True,
         proxy: Optional[str] = None,
         retries: Optional[int] = 3,
+        cookies: Optional[Dict] = None,
         custom_config: Dict = None,
         **kwargs: Dict,
     ) -> Response:
@@ -89,6 +99,7 @@ class Fetcher(BaseFetcher):
             create a referer header as if this request came from Google's search of this URL's domain.
         :param proxy: A string of a proxy to use for http and https requests, the format accepted is `http://username:password@localhost:8030`
         :param retries: The number of retries to do through httpx if the request failed for any reason. The default is 3 retries.
+        :param cookies: Set cookies for the next request.
         :param custom_config: A dictionary of custom parser arguments to use with this request. Any argument passed will override any class parameters values.
         :param kwargs: Any additional keyword arguments are passed directly to `httpx.post()` function so check httpx documentation for details.
         :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
@@ -103,6 +114,12 @@ class Fetcher(BaseFetcher):
         adaptor_arguments = tuple(
             {**cls._generate_parser_arguments(), **custom_config}.items()
         )
+
+        if not cookies:
+            cookies = {}
+        elif not isinstance(cookies, dict):
+            ValueError(f"The cookies must be of type dictionary, got {cls.__class__}")
+
         response_object = StaticEngine(
             url,
             proxy,
@@ -110,6 +127,7 @@ class Fetcher(BaseFetcher):
             follow_redirects,
             timeout,
             retries,
+            tuple(cookies.items()),
             adaptor_arguments=adaptor_arguments,
         ).post(**kwargs)
         return response_object
@@ -123,6 +141,7 @@ class Fetcher(BaseFetcher):
         stealthy_headers: bool = True,
         proxy: Optional[str] = None,
         retries: Optional[int] = 3,
+        cookies: Optional[Dict] = None,
         custom_config: Dict = None,
         **kwargs: Dict,
     ) -> Response:
@@ -135,6 +154,7 @@ class Fetcher(BaseFetcher):
             create a referer header as if this request came from Google's search of this URL's domain.
         :param proxy: A string of a proxy to use for http and https requests, the format accepted is `http://username:password@localhost:8030`
         :param retries: The number of retries to do through httpx if the request failed for any reason. The default is 3 retries.
+        :param cookies: Set cookies for the next request.
         :param custom_config: A dictionary of custom parser arguments to use with this request. Any argument passed will override any class parameters values.
         :param kwargs: Any additional keyword arguments are passed directly to `httpx.put()` function so check httpx documentation for details.
 
@@ -150,6 +170,12 @@ class Fetcher(BaseFetcher):
         adaptor_arguments = tuple(
             {**cls._generate_parser_arguments(), **custom_config}.items()
         )
+
+        if not cookies:
+            cookies = {}
+        elif not isinstance(cookies, dict):
+            ValueError(f"The cookies must be of type dictionary, got {cls.__class__}")
+
         response_object = StaticEngine(
             url,
             proxy,
@@ -157,6 +183,7 @@ class Fetcher(BaseFetcher):
             follow_redirects,
             timeout,
             retries,
+            tuple(cookies.items()),
             adaptor_arguments=adaptor_arguments,
         ).put(**kwargs)
         return response_object
@@ -170,6 +197,7 @@ class Fetcher(BaseFetcher):
         stealthy_headers: bool = True,
         proxy: Optional[str] = None,
         retries: Optional[int] = 3,
+        cookies: Optional[Dict] = None,
         custom_config: Dict = None,
         **kwargs: Dict,
     ) -> Response:
@@ -182,6 +210,7 @@ class Fetcher(BaseFetcher):
             create a referer header as if this request came from Google's search of this URL's domain.
         :param proxy: A string of a proxy to use for http and https requests, the format accepted is `http://username:password@localhost:8030`
         :param retries: The number of retries to do through httpx if the request failed for any reason. The default is 3 retries.
+        :param cookies: Set cookies for the next request.
         :param custom_config: A dictionary of custom parser arguments to use with this request. Any argument passed will override any class parameters values.
         :param kwargs: Any additional keyword arguments are passed directly to `httpx.delete()` function so check httpx documentation for details.
         :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
@@ -196,6 +225,12 @@ class Fetcher(BaseFetcher):
         adaptor_arguments = tuple(
             {**cls._generate_parser_arguments(), **custom_config}.items()
         )
+
+        if not cookies:
+            cookies = {}
+        elif not isinstance(cookies, dict):
+            ValueError(f"The cookies must be of type dictionary, got {cls.__class__}")
+
         response_object = StaticEngine(
             url,
             proxy,
@@ -203,6 +238,7 @@ class Fetcher(BaseFetcher):
             follow_redirects,
             timeout,
             retries,
+            tuple(cookies.items()),
             adaptor_arguments=adaptor_arguments,
         ).delete(**kwargs)
         return response_object
@@ -218,6 +254,7 @@ class AsyncFetcher(Fetcher):
         stealthy_headers: bool = True,
         proxy: Optional[str] = None,
         retries: Optional[int] = 3,
+        cookies: Optional[Dict] = None,
         custom_config: Dict = None,
         **kwargs: Dict,
     ) -> Response:
@@ -230,6 +267,7 @@ class AsyncFetcher(Fetcher):
             create a referer header as if this request had came from Google's search of this URL's domain.
         :param proxy: A string of a proxy to use for http and https requests, the format accepted is `http://username:password@localhost:8030`
         :param retries: The number of retries to do through httpx if the request failed for any reason. The default is 3 retries.
+        :param cookies: Set cookies for the next request.
         :param custom_config: A dictionary of custom parser arguments to use with this request. Any argument passed will override any class parameters values.
         :param kwargs: Any additional keyword arguments are passed directly to `httpx.get()` function so check httpx documentation for details.
         :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
@@ -244,6 +282,12 @@ class AsyncFetcher(Fetcher):
         adaptor_arguments = tuple(
             {**cls._generate_parser_arguments(), **custom_config}.items()
         )
+
+        if not cookies:
+            cookies = {}
+        elif not isinstance(cookies, dict):
+            ValueError(f"The cookies must be of type dictionary, got {cls.__class__}")
+
         response_object = await StaticEngine(
             url,
             proxy,
@@ -251,6 +295,7 @@ class AsyncFetcher(Fetcher):
             follow_redirects,
             timeout,
             retries=retries,
+            cookies=tuple(cookies.items()),
             adaptor_arguments=adaptor_arguments,
         ).async_get(**kwargs)
         return response_object
@@ -264,6 +309,7 @@ class AsyncFetcher(Fetcher):
         stealthy_headers: bool = True,
         proxy: Optional[str] = None,
         retries: Optional[int] = 3,
+        cookies: Optional[Dict] = None,
         custom_config: Dict = None,
         **kwargs: Dict,
     ) -> Response:
@@ -276,6 +322,7 @@ class AsyncFetcher(Fetcher):
             create a referer header as if this request came from Google's search of this URL's domain.
         :param proxy: A string of a proxy to use for http and https requests, the format accepted is `http://username:password@localhost:8030`
         :param retries: The number of retries to do through httpx if the request failed for any reason. The default is 3 retries.
+        :param cookies: Set cookies for the next request.
         :param custom_config: A dictionary of custom parser arguments to use with this request. Any argument passed will override any class parameters values.
         :param kwargs: Any additional keyword arguments are passed directly to `httpx.post()` function so check httpx documentation for details.
         :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
@@ -290,6 +337,12 @@ class AsyncFetcher(Fetcher):
         adaptor_arguments = tuple(
             {**cls._generate_parser_arguments(), **custom_config}.items()
         )
+
+        if not cookies:
+            cookies = {}
+        elif not isinstance(cookies, dict):
+            ValueError(f"The cookies must be of type dictionary, got {cls.__class__}")
+
         response_object = await StaticEngine(
             url,
             proxy,
@@ -297,6 +350,7 @@ class AsyncFetcher(Fetcher):
             follow_redirects,
             timeout,
             retries=retries,
+            cookies=tuple(cookies.items()),
             adaptor_arguments=adaptor_arguments,
         ).async_post(**kwargs)
         return response_object
@@ -310,6 +364,7 @@ class AsyncFetcher(Fetcher):
         stealthy_headers: bool = True,
         proxy: Optional[str] = None,
         retries: Optional[int] = 3,
+        cookies: Optional[Dict] = None,
         custom_config: Dict = None,
         **kwargs: Dict,
     ) -> Response:
@@ -322,6 +377,7 @@ class AsyncFetcher(Fetcher):
             create a referer header as if this request came from Google's search of this URL's domain.
         :param proxy: A string of a proxy to use for http and https requests, the format accepted is `http://username:password@localhost:8030`
         :param retries: The number of retries to do through httpx if the request failed for any reason. The default is 3 retries.
+        :param cookies: Set cookies for the next request.
         :param custom_config: A dictionary of custom parser arguments to use with this request. Any argument passed will override any class parameters values.
         :param kwargs: Any additional keyword arguments are passed directly to `httpx.put()` function so check httpx documentation for details.
         :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
@@ -336,6 +392,12 @@ class AsyncFetcher(Fetcher):
         adaptor_arguments = tuple(
             {**cls._generate_parser_arguments(), **custom_config}.items()
         )
+
+        if not cookies:
+            cookies = {}
+        elif not isinstance(cookies, dict):
+            ValueError(f"The cookies must be of type dictionary, got {cls.__class__}")
+
         response_object = await StaticEngine(
             url,
             proxy,
@@ -343,6 +405,7 @@ class AsyncFetcher(Fetcher):
             follow_redirects,
             timeout,
             retries=retries,
+            cookies=tuple(cookies.items()),
             adaptor_arguments=adaptor_arguments,
         ).async_put(**kwargs)
         return response_object
@@ -356,6 +419,7 @@ class AsyncFetcher(Fetcher):
         stealthy_headers: bool = True,
         proxy: Optional[str] = None,
         retries: Optional[int] = 3,
+        cookies: Optional[Dict] = None,
         custom_config: Dict = None,
         **kwargs: Dict,
     ) -> Response:
@@ -368,6 +432,7 @@ class AsyncFetcher(Fetcher):
             create a referer header as if this request came from Google's search of this URL's domain.
         :param proxy: A string of a proxy to use for http and https requests, the format accepted is `http://username:password@localhost:8030`
         :param retries: The number of retries to do through httpx if the request failed for any reason. The default is 3 retries.
+        :param cookies: Set cookies for the next request.
         :param custom_config: A dictionary of custom parser arguments to use with this request. Any argument passed will override any class parameters values.
         :param kwargs: Any additional keyword arguments are passed directly to `httpx.delete()` function so check httpx documentation for details.
         :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
@@ -382,6 +447,12 @@ class AsyncFetcher(Fetcher):
         adaptor_arguments = tuple(
             {**cls._generate_parser_arguments(), **custom_config}.items()
         )
+
+        if not cookies:
+            cookies = {}
+        elif not isinstance(cookies, dict):
+            ValueError(f"The cookies must be of type dictionary, got {cls.__class__}")
+
         response_object = await StaticEngine(
             url,
             proxy,
@@ -389,6 +460,7 @@ class AsyncFetcher(Fetcher):
             follow_redirects,
             timeout,
             retries=retries,
+            cookies=tuple(cookies.items()),
             adaptor_arguments=adaptor_arguments,
         ).async_delete(**kwargs)
         return response_object