From c84143129e4365a6b89592bb01d456811dfa15db Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Sat, 10 May 2025 19:16:34 +0300 Subject: [PATCH] fix(fetchers): Adjusting all cookies returned from each fetcher The cookies returned now can be passed again to each fetcher's engine without further validation by us. --- scrapling/engines/camo.py | 13 ++++--------- scrapling/engines/pw.py | 13 ++++--------- scrapling/engines/static.py | 2 +- scrapling/engines/toolbelt/custom.py | 4 ++-- 4 files changed, 11 insertions(+), 21 deletions(-) diff --git a/scrapling/engines/camo.py b/scrapling/engines/camo.py index 98e0747..f3ba36b 100644 --- a/scrapling/engines/camo.py +++ b/scrapling/engines/camo.py @@ -164,7 +164,7 @@ class CamoufoxEngine: else StatusText.get(301), encoding=current_response.headers.get("content-type", "") or "utf-8", - cookies={}, + cookies=tuple(), headers=current_response.all_headers() if current_response else {}, @@ -207,7 +207,7 @@ class CamoufoxEngine: else StatusText.get(301), encoding=current_response.headers.get("content-type", "") or "utf-8", - cookies={}, + cookies=tuple(), headers=await current_response.all_headers() if current_response else {}, @@ -450,9 +450,7 @@ class CamoufoxEngine: status=final_response.status, reason=status_text, encoding=encoding, - cookies={ - cookie["name"]: cookie["value"] for cookie in page.context.cookies() - }, + cookies=tuple(dict(cookie) for cookie in page.context.cookies()), headers=first_response.all_headers(), request_headers=first_response.request.all_headers(), history=history, @@ -554,10 +552,7 @@ class CamoufoxEngine: status=final_response.status, reason=status_text, encoding=encoding, - cookies={ - cookie["name"]: cookie["value"] - for cookie in await page.context.cookies() - }, + cookies=tuple(dict(cookie) for cookie in await page.context.cookies()), headers=await first_response.all_headers(), request_headers=await first_response.request.all_headers(), history=history, diff --git a/scrapling/engines/pw.py b/scrapling/engines/pw.py index 521f13f..80b033b 100644 --- a/scrapling/engines/pw.py +++ b/scrapling/engines/pw.py @@ -242,7 +242,7 @@ class PlaywrightEngine: else StatusText.get(301), encoding=current_response.headers.get("content-type", "") or "utf-8", - cookies={}, + cookies=tuple(), headers=current_response.all_headers() if current_response else {}, @@ -285,7 +285,7 @@ class PlaywrightEngine: else StatusText.get(301), encoding=current_response.headers.get("content-type", "") or "utf-8", - cookies={}, + cookies=tuple(), headers=await current_response.all_headers() if current_response else {}, @@ -405,9 +405,7 @@ class PlaywrightEngine: status=final_response.status, reason=status_text, encoding=encoding, - cookies={ - cookie["name"]: cookie["value"] for cookie in page.context.cookies() - }, + cookies=tuple(dict(cookie) for cookie in page.context.cookies()), headers=first_response.all_headers(), request_headers=first_response.request.all_headers(), history=history, @@ -519,10 +517,7 @@ class PlaywrightEngine: status=final_response.status, reason=status_text, encoding=encoding, - cookies={ - cookie["name"]: cookie["value"] - for cookie in await page.context.cookies() - }, + cookies=tuple(dict(cookie) for cookie in await page.context.cookies()), headers=await first_response.all_headers(), request_headers=await first_response.request.all_headers(), history=history, diff --git a/scrapling/engines/static.py b/scrapling/engines/static.py index 06ee1ee..187d36e 100644 --- a/scrapling/engines/static.py +++ b/scrapling/engines/static.py @@ -17,7 +17,7 @@ class StaticEngine: follow_redirects: bool = True, timeout: Optional[Union[int, float]] = None, retries: Optional[int] = 3, - cookies: Optional[Dict] = None, + cookies: Optional[Tuple] = None, adaptor_arguments: Tuple = None, ): """An engine that utilizes httpx library, check the `Fetcher` class for more documentation. diff --git a/scrapling/engines/toolbelt/custom.py b/scrapling/engines/toolbelt/custom.py index c0e7814..63ea1fc 100644 --- a/scrapling/engines/toolbelt/custom.py +++ b/scrapling/engines/toolbelt/custom.py @@ -109,7 +109,7 @@ class Response(Adaptor): body: bytes, status: int, reason: str, - cookies: Dict, + cookies: Union[Tuple[Dict[str, str], ...], Dict[str, str]], headers: Dict, request_headers: Dict, encoding: str = "utf-8", @@ -132,7 +132,7 @@ class Response(Adaptor): encoding=encoding, **adaptor_arguments, ) - # For back-ward compatibility + # For backward compatibility self.adaptor = self # For easier debugging while working from a Python shell log.info(