freedak/Scrapling
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(PlayWrightFetcher): Add the redirections history

Browse Source
This commit is contained in:
Karim shoair
2025-02-22 17:47:27 +02:00
parent 3d2418b67a
commit 8049ac0f28
1 changed files with 42 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scrapling/engines/pw.py
+42
View File
@@ -259,6 +259,26 @@ class PlaywrightEngine:
# PlayWright API sometimes give empty status text for some reason! # PlayWright API sometimes give empty status text for some reason!
status_text = final_response.status_text or StatusText.get(final_response.status) status_text = final_response.status_text or StatusText.get(final_response.status)
history = []
current_request = first_response.request.redirected_from
while current_request:
current_response = current_request.response()
history.insert(0, Response(
url=current_request.url,
# using current_response.text() will trigger "Error: Response.text: Response body is unavailable for redirect responses"
text='',
body=b'',
status=current_response.status if current_response else 301,
reason=(current_response.status_text or StatusText.get(current_response.status)) if current_response else StatusText.get(301),
encoding=current_response.headers.get('content-type', '') or 'utf-8',
cookies={},
headers=current_response.all_headers() if current_response else {},
request_headers=current_request.all_headers(),
**self.adaptor_arguments
))
current_request = current_request.redirected_from
response = Response( response = Response(
url=page.url, url=page.url,
text=page.content(), text=page.content(),
@@ -269,6 +289,7 @@ class PlaywrightEngine:
cookies={cookie['name']: cookie['value'] for cookie in page.context.cookies()}, cookies={cookie['name']: cookie['value'] for cookie in page.context.cookies()},
headers=first_response.all_headers(), headers=first_response.all_headers(),
request_headers=first_response.request.all_headers(), request_headers=first_response.request.all_headers(),
history=history,
**self.adaptor_arguments **self.adaptor_arguments
) )
page.close() page.close()
@@ -345,6 +366,26 @@ class PlaywrightEngine:
# PlayWright API sometimes give empty status text for some reason! # PlayWright API sometimes give empty status text for some reason!
status_text = final_response.status_text or StatusText.get(final_response.status) status_text = final_response.status_text or StatusText.get(final_response.status)
history = []
current_request = first_response.request.redirected_from
while current_request:
current_response = await current_request.response()
history.insert(0, Response(
url=current_request.url,
# using current_response.text() will trigger "Error: Response.text: Response body is unavailable for redirect responses"
text='',
body=b'',
status=current_response.status if current_response else 301,
reason=(current_response.status_text or StatusText.get(current_response.status)) if current_response else StatusText.get(301),
encoding=current_response.headers.get('content-type', '') or 'utf-8',
cookies={},
headers=await current_response.all_headers() if current_response else {},
request_headers=await current_request.all_headers(),
**self.adaptor_arguments
))
current_request = current_request.redirected_from
response = Response( response = Response(
url=page.url, url=page.url,
text=await page.content(), text=await page.content(),
@@ -355,6 +396,7 @@ class PlaywrightEngine:
cookies={cookie['name']: cookie['value'] for cookie in await page.context.cookies()}, cookies={cookie['name']: cookie['value'] for cookie in await page.context.cookies()},
headers=await first_response.all_headers(), headers=await first_response.all_headers(),
request_headers=await first_response.request.all_headers(), request_headers=await first_response.request.all_headers(),
history=history,
**self.adaptor_arguments **self.adaptor_arguments
) )
await page.close() await page.close()