freedak/Scrapling
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(fetchers): Adjusting all cookies returned from each fetcher

Browse Source 
The cookies returned now can be passed again to each fetcher's engine without further validation by us.
This commit is contained in:
Karim shoair
2025-05-10 19:16:34 +03:00
parent bd47c070aa
commit c84143129e
4 changed files with 11 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scrapling/engines/camo.py
+4 -9
View File
@@ -164,7 +164,7 @@ class CamoufoxEngine:
else StatusText.get(301), else StatusText.get(301),
encoding=current_response.headers.get("content-type", "") encoding=current_response.headers.get("content-type", "")
or "utf-8", or "utf-8",
cookies={}, cookies=tuple(),
headers=current_response.all_headers() headers=current_response.all_headers()
if current_response if current_response
else {}, else {},
@@ -207,7 +207,7 @@ class CamoufoxEngine:
else StatusText.get(301), else StatusText.get(301),
encoding=current_response.headers.get("content-type", "") encoding=current_response.headers.get("content-type", "")
or "utf-8", or "utf-8",
cookies={}, cookies=tuple(),
headers=await current_response.all_headers() headers=await current_response.all_headers()
if current_response if current_response
else {}, else {},
@@ -450,9 +450,7 @@ class CamoufoxEngine:
status=final_response.status, status=final_response.status,
reason=status_text, reason=status_text,
encoding=encoding, encoding=encoding,
cookies={ cookies=tuple(dict(cookie) for cookie in page.context.cookies()),
cookie["name"]: cookie["value"] for cookie in page.context.cookies()
},
headers=first_response.all_headers(), headers=first_response.all_headers(),
request_headers=first_response.request.all_headers(), request_headers=first_response.request.all_headers(),
history=history, history=history,
@@ -554,10 +552,7 @@ class CamoufoxEngine:
status=final_response.status, status=final_response.status,
reason=status_text, reason=status_text,
encoding=encoding, encoding=encoding,
cookies={ cookies=tuple(dict(cookie) for cookie in await page.context.cookies()),
cookie["name"]: cookie["value"]
for cookie in await page.context.cookies()
},
headers=await first_response.all_headers(), headers=await first_response.all_headers(),
request_headers=await first_response.request.all_headers(), request_headers=await first_response.request.all_headers(),
history=history, history=history,
scrapling/engines/pw.py
+4 -9
View File
@@ -242,7 +242,7 @@ class PlaywrightEngine:
else StatusText.get(301), else StatusText.get(301),
encoding=current_response.headers.get("content-type", "") encoding=current_response.headers.get("content-type", "")
or "utf-8", or "utf-8",
cookies={}, cookies=tuple(),
headers=current_response.all_headers() headers=current_response.all_headers()
if current_response if current_response
else {}, else {},
@@ -285,7 +285,7 @@ class PlaywrightEngine:
else StatusText.get(301), else StatusText.get(301),
encoding=current_response.headers.get("content-type", "") encoding=current_response.headers.get("content-type", "")
or "utf-8", or "utf-8",
cookies={}, cookies=tuple(),
headers=await current_response.all_headers() headers=await current_response.all_headers()
if current_response if current_response
else {}, else {},
@@ -405,9 +405,7 @@ class PlaywrightEngine:
status=final_response.status, status=final_response.status,
reason=status_text, reason=status_text,
encoding=encoding, encoding=encoding,
cookies={ cookies=tuple(dict(cookie) for cookie in page.context.cookies()),
cookie["name"]: cookie["value"] for cookie in page.context.cookies()
},
headers=first_response.all_headers(), headers=first_response.all_headers(),
request_headers=first_response.request.all_headers(), request_headers=first_response.request.all_headers(),
history=history, history=history,
@@ -519,10 +517,7 @@ class PlaywrightEngine:
status=final_response.status, status=final_response.status,
reason=status_text, reason=status_text,
encoding=encoding, encoding=encoding,
cookies={ cookies=tuple(dict(cookie) for cookie in await page.context.cookies()),
cookie["name"]: cookie["value"]
for cookie in await page.context.cookies()
},
headers=await first_response.all_headers(), headers=await first_response.all_headers(),
request_headers=await first_response.request.all_headers(), request_headers=await first_response.request.all_headers(),
history=history, history=history,
scrapling/engines/static.py
+1 -1
View File
@@ -17,7 +17,7 @@ class StaticEngine:
follow_redirects: bool = True, follow_redirects: bool = True,
timeout: Optional[Union[int, float]] = None, timeout: Optional[Union[int, float]] = None,
retries: Optional[int] = 3, retries: Optional[int] = 3,
cookies: Optional[Dict] = None, cookies: Optional[Tuple] = None,
adaptor_arguments: Tuple = None, adaptor_arguments: Tuple = None,
): ):
"""An engine that utilizes httpx library, check the `Fetcher` class for more documentation. """An engine that utilizes httpx library, check the `Fetcher` class for more documentation.
scrapling/engines/toolbelt/custom.py
+2 -2
View File
@@ -109,7 +109,7 @@ class Response(Adaptor):
body: bytes, body: bytes,
status: int, status: int,
reason: str, reason: str,
cookies: Dict, cookies: Union[Tuple[Dict[str, str], ...], Dict[str, str]],
headers: Dict, headers: Dict,
request_headers: Dict, request_headers: Dict,
encoding: str = "utf-8", encoding: str = "utf-8",
@@ -132,7 +132,7 @@ class Response(Adaptor):
encoding=encoding, encoding=encoding,
**adaptor_arguments, **adaptor_arguments,
) )
# For back-ward compatibility # For backward compatibility
self.adaptor = self self.adaptor = self
# For easier debugging while working from a Python shell # For easier debugging while working from a Python shell
log.info( log.info(