From 1240ee3eb02e436f37a7f9988192a3495f6aba4a Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Tue, 29 Apr 2025 03:01:19 +0300 Subject: [PATCH] fix(Fetcher): Fix the issue of passing cookies --- scrapling/engines/static.py | 11 ++++-- scrapling/fetchers.py | 72 +++++++++++++++++++++++++++++++++++++ 2 files changed, 81 insertions(+), 2 deletions(-) diff --git a/scrapling/engines/static.py b/scrapling/engines/static.py index 6ef3810..06ee1ee 100644 --- a/scrapling/engines/static.py +++ b/scrapling/engines/static.py @@ -17,6 +17,7 @@ class StaticEngine: follow_redirects: bool = True, timeout: Optional[Union[int, float]] = None, retries: Optional[int] = 3, + cookies: Optional[Dict] = None, adaptor_arguments: Tuple = None, ): """An engine that utilizes httpx library, check the `Fetcher` class for more documentation. @@ -26,6 +27,7 @@ class StaticEngine: create a referer header as if this request had came from Google's search of this URL's domain. :param proxy: A string of a proxy to use for http and https requests, the format accepted is `http://username:password@localhost:8030` :param follow_redirects: As the name says -- if enabled (default), redirects will be followed. + :param cookies: Set cookies for the next request. :param timeout: The time to wait for the request to finish in seconds. The default is 10 seconds. :param adaptor_arguments: The arguments that will be passed in the end while creating the final Adaptor's class. """ @@ -35,6 +37,7 @@ class StaticEngine: self.timeout = timeout self.follow_redirects = bool(follow_redirects) self.retries = retries + self.cookies = dict(cookies) if cookies else {} self._extra_headers = generate_headers(browser_mode=False) # Because we are using `lru_cache` for a slight optimization but both dict/dict_items are not hashable so they can't be cached # So my solution here was to convert it to tuple then convert it back to dictionary again here as tuples are hashable, ofc `tuple().__hash__()` @@ -98,7 +101,9 @@ class StaticEngine: def _make_request(self, method: str, **kwargs) -> Response: headers = self._headers_job(kwargs.pop("headers", {})) with httpx.Client( - proxy=self.proxy, transport=httpx.HTTPTransport(retries=self.retries) + proxy=self.proxy, + transport=httpx.HTTPTransport(retries=self.retries), + cookies=self.cookies, ) as client: request = getattr(client, method)( url=self.url, @@ -112,7 +117,9 @@ class StaticEngine: async def _async_make_request(self, method: str, **kwargs) -> Response: headers = self._headers_job(kwargs.pop("headers", {})) async with httpx.AsyncClient( - proxy=self.proxy, transport=httpx.AsyncHTTPTransport(retries=self.retries) + proxy=self.proxy, + transport=httpx.AsyncHTTPTransport(retries=self.retries), + cookies=self.cookies, ) as client: request = await getattr(client, method)( url=self.url, diff --git a/scrapling/fetchers.py b/scrapling/fetchers.py index 6bb9a6e..1575e20 100644 --- a/scrapling/fetchers.py +++ b/scrapling/fetchers.py @@ -31,6 +31,7 @@ class Fetcher(BaseFetcher): stealthy_headers: bool = True, proxy: Optional[str] = None, retries: Optional[int] = 3, + cookies: Optional[Dict] = None, custom_config: Dict = None, **kwargs: Dict, ) -> Response: @@ -43,6 +44,7 @@ class Fetcher(BaseFetcher): create a referer header as if this request had came from Google's search of this URL's domain. :param proxy: A string of a proxy to use for http and https requests, the format accepted is `http://username:password@localhost:8030` :param retries: The number of retries to do through httpx if the request failed for any reason. The default is 3 retries. + :param cookies: Set cookies for the next request. :param custom_config: A dictionary of custom parser arguments to use with this request. Any argument passed will override any class parameters values. :param kwargs: Any additional keyword arguments are passed directly to `httpx.get()` function so check httpx documentation for details. :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers` @@ -57,6 +59,12 @@ class Fetcher(BaseFetcher): adaptor_arguments = tuple( {**cls._generate_parser_arguments(), **custom_config}.items() ) + + if not cookies: + cookies = {} + elif not isinstance(cookies, dict): + ValueError(f"The cookies must be of type dictionary, got {cls.__class__}") + response_object = StaticEngine( url, proxy, @@ -64,6 +72,7 @@ class Fetcher(BaseFetcher): follow_redirects, timeout, retries, + tuple(cookies.items()), adaptor_arguments=adaptor_arguments, ).get(**kwargs) return response_object @@ -77,6 +86,7 @@ class Fetcher(BaseFetcher): stealthy_headers: bool = True, proxy: Optional[str] = None, retries: Optional[int] = 3, + cookies: Optional[Dict] = None, custom_config: Dict = None, **kwargs: Dict, ) -> Response: @@ -89,6 +99,7 @@ class Fetcher(BaseFetcher): create a referer header as if this request came from Google's search of this URL's domain. :param proxy: A string of a proxy to use for http and https requests, the format accepted is `http://username:password@localhost:8030` :param retries: The number of retries to do through httpx if the request failed for any reason. The default is 3 retries. + :param cookies: Set cookies for the next request. :param custom_config: A dictionary of custom parser arguments to use with this request. Any argument passed will override any class parameters values. :param kwargs: Any additional keyword arguments are passed directly to `httpx.post()` function so check httpx documentation for details. :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers` @@ -103,6 +114,12 @@ class Fetcher(BaseFetcher): adaptor_arguments = tuple( {**cls._generate_parser_arguments(), **custom_config}.items() ) + + if not cookies: + cookies = {} + elif not isinstance(cookies, dict): + ValueError(f"The cookies must be of type dictionary, got {cls.__class__}") + response_object = StaticEngine( url, proxy, @@ -110,6 +127,7 @@ class Fetcher(BaseFetcher): follow_redirects, timeout, retries, + tuple(cookies.items()), adaptor_arguments=adaptor_arguments, ).post(**kwargs) return response_object @@ -123,6 +141,7 @@ class Fetcher(BaseFetcher): stealthy_headers: bool = True, proxy: Optional[str] = None, retries: Optional[int] = 3, + cookies: Optional[Dict] = None, custom_config: Dict = None, **kwargs: Dict, ) -> Response: @@ -135,6 +154,7 @@ class Fetcher(BaseFetcher): create a referer header as if this request came from Google's search of this URL's domain. :param proxy: A string of a proxy to use for http and https requests, the format accepted is `http://username:password@localhost:8030` :param retries: The number of retries to do through httpx if the request failed for any reason. The default is 3 retries. + :param cookies: Set cookies for the next request. :param custom_config: A dictionary of custom parser arguments to use with this request. Any argument passed will override any class parameters values. :param kwargs: Any additional keyword arguments are passed directly to `httpx.put()` function so check httpx documentation for details. @@ -150,6 +170,12 @@ class Fetcher(BaseFetcher): adaptor_arguments = tuple( {**cls._generate_parser_arguments(), **custom_config}.items() ) + + if not cookies: + cookies = {} + elif not isinstance(cookies, dict): + ValueError(f"The cookies must be of type dictionary, got {cls.__class__}") + response_object = StaticEngine( url, proxy, @@ -157,6 +183,7 @@ class Fetcher(BaseFetcher): follow_redirects, timeout, retries, + tuple(cookies.items()), adaptor_arguments=adaptor_arguments, ).put(**kwargs) return response_object @@ -170,6 +197,7 @@ class Fetcher(BaseFetcher): stealthy_headers: bool = True, proxy: Optional[str] = None, retries: Optional[int] = 3, + cookies: Optional[Dict] = None, custom_config: Dict = None, **kwargs: Dict, ) -> Response: @@ -182,6 +210,7 @@ class Fetcher(BaseFetcher): create a referer header as if this request came from Google's search of this URL's domain. :param proxy: A string of a proxy to use for http and https requests, the format accepted is `http://username:password@localhost:8030` :param retries: The number of retries to do through httpx if the request failed for any reason. The default is 3 retries. + :param cookies: Set cookies for the next request. :param custom_config: A dictionary of custom parser arguments to use with this request. Any argument passed will override any class parameters values. :param kwargs: Any additional keyword arguments are passed directly to `httpx.delete()` function so check httpx documentation for details. :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers` @@ -196,6 +225,12 @@ class Fetcher(BaseFetcher): adaptor_arguments = tuple( {**cls._generate_parser_arguments(), **custom_config}.items() ) + + if not cookies: + cookies = {} + elif not isinstance(cookies, dict): + ValueError(f"The cookies must be of type dictionary, got {cls.__class__}") + response_object = StaticEngine( url, proxy, @@ -203,6 +238,7 @@ class Fetcher(BaseFetcher): follow_redirects, timeout, retries, + tuple(cookies.items()), adaptor_arguments=adaptor_arguments, ).delete(**kwargs) return response_object @@ -218,6 +254,7 @@ class AsyncFetcher(Fetcher): stealthy_headers: bool = True, proxy: Optional[str] = None, retries: Optional[int] = 3, + cookies: Optional[Dict] = None, custom_config: Dict = None, **kwargs: Dict, ) -> Response: @@ -230,6 +267,7 @@ class AsyncFetcher(Fetcher): create a referer header as if this request had came from Google's search of this URL's domain. :param proxy: A string of a proxy to use for http and https requests, the format accepted is `http://username:password@localhost:8030` :param retries: The number of retries to do through httpx if the request failed for any reason. The default is 3 retries. + :param cookies: Set cookies for the next request. :param custom_config: A dictionary of custom parser arguments to use with this request. Any argument passed will override any class parameters values. :param kwargs: Any additional keyword arguments are passed directly to `httpx.get()` function so check httpx documentation for details. :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers` @@ -244,6 +282,12 @@ class AsyncFetcher(Fetcher): adaptor_arguments = tuple( {**cls._generate_parser_arguments(), **custom_config}.items() ) + + if not cookies: + cookies = {} + elif not isinstance(cookies, dict): + ValueError(f"The cookies must be of type dictionary, got {cls.__class__}") + response_object = await StaticEngine( url, proxy, @@ -251,6 +295,7 @@ class AsyncFetcher(Fetcher): follow_redirects, timeout, retries=retries, + cookies=tuple(cookies.items()), adaptor_arguments=adaptor_arguments, ).async_get(**kwargs) return response_object @@ -264,6 +309,7 @@ class AsyncFetcher(Fetcher): stealthy_headers: bool = True, proxy: Optional[str] = None, retries: Optional[int] = 3, + cookies: Optional[Dict] = None, custom_config: Dict = None, **kwargs: Dict, ) -> Response: @@ -276,6 +322,7 @@ class AsyncFetcher(Fetcher): create a referer header as if this request came from Google's search of this URL's domain. :param proxy: A string of a proxy to use for http and https requests, the format accepted is `http://username:password@localhost:8030` :param retries: The number of retries to do through httpx if the request failed for any reason. The default is 3 retries. + :param cookies: Set cookies for the next request. :param custom_config: A dictionary of custom parser arguments to use with this request. Any argument passed will override any class parameters values. :param kwargs: Any additional keyword arguments are passed directly to `httpx.post()` function so check httpx documentation for details. :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers` @@ -290,6 +337,12 @@ class AsyncFetcher(Fetcher): adaptor_arguments = tuple( {**cls._generate_parser_arguments(), **custom_config}.items() ) + + if not cookies: + cookies = {} + elif not isinstance(cookies, dict): + ValueError(f"The cookies must be of type dictionary, got {cls.__class__}") + response_object = await StaticEngine( url, proxy, @@ -297,6 +350,7 @@ class AsyncFetcher(Fetcher): follow_redirects, timeout, retries=retries, + cookies=tuple(cookies.items()), adaptor_arguments=adaptor_arguments, ).async_post(**kwargs) return response_object @@ -310,6 +364,7 @@ class AsyncFetcher(Fetcher): stealthy_headers: bool = True, proxy: Optional[str] = None, retries: Optional[int] = 3, + cookies: Optional[Dict] = None, custom_config: Dict = None, **kwargs: Dict, ) -> Response: @@ -322,6 +377,7 @@ class AsyncFetcher(Fetcher): create a referer header as if this request came from Google's search of this URL's domain. :param proxy: A string of a proxy to use for http and https requests, the format accepted is `http://username:password@localhost:8030` :param retries: The number of retries to do through httpx if the request failed for any reason. The default is 3 retries. + :param cookies: Set cookies for the next request. :param custom_config: A dictionary of custom parser arguments to use with this request. Any argument passed will override any class parameters values. :param kwargs: Any additional keyword arguments are passed directly to `httpx.put()` function so check httpx documentation for details. :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers` @@ -336,6 +392,12 @@ class AsyncFetcher(Fetcher): adaptor_arguments = tuple( {**cls._generate_parser_arguments(), **custom_config}.items() ) + + if not cookies: + cookies = {} + elif not isinstance(cookies, dict): + ValueError(f"The cookies must be of type dictionary, got {cls.__class__}") + response_object = await StaticEngine( url, proxy, @@ -343,6 +405,7 @@ class AsyncFetcher(Fetcher): follow_redirects, timeout, retries=retries, + cookies=tuple(cookies.items()), adaptor_arguments=adaptor_arguments, ).async_put(**kwargs) return response_object @@ -356,6 +419,7 @@ class AsyncFetcher(Fetcher): stealthy_headers: bool = True, proxy: Optional[str] = None, retries: Optional[int] = 3, + cookies: Optional[Dict] = None, custom_config: Dict = None, **kwargs: Dict, ) -> Response: @@ -368,6 +432,7 @@ class AsyncFetcher(Fetcher): create a referer header as if this request came from Google's search of this URL's domain. :param proxy: A string of a proxy to use for http and https requests, the format accepted is `http://username:password@localhost:8030` :param retries: The number of retries to do through httpx if the request failed for any reason. The default is 3 retries. + :param cookies: Set cookies for the next request. :param custom_config: A dictionary of custom parser arguments to use with this request. Any argument passed will override any class parameters values. :param kwargs: Any additional keyword arguments are passed directly to `httpx.delete()` function so check httpx documentation for details. :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers` @@ -382,6 +447,12 @@ class AsyncFetcher(Fetcher): adaptor_arguments = tuple( {**cls._generate_parser_arguments(), **custom_config}.items() ) + + if not cookies: + cookies = {} + elif not isinstance(cookies, dict): + ValueError(f"The cookies must be of type dictionary, got {cls.__class__}") + response_object = await StaticEngine( url, proxy, @@ -389,6 +460,7 @@ class AsyncFetcher(Fetcher): follow_redirects, timeout, retries=retries, + cookies=tuple(cookies.items()), adaptor_arguments=adaptor_arguments, ).async_delete(**kwargs) return response_object