diff --git a/scrapling/engines/pw.py b/scrapling/engines/pw.py index c0ca34c..6b8003c 100644 --- a/scrapling/engines/pw.py +++ b/scrapling/engines/pw.py @@ -259,6 +259,26 @@ class PlaywrightEngine: # PlayWright API sometimes give empty status text for some reason! status_text = final_response.status_text or StatusText.get(final_response.status) + history = [] + current_request = first_response.request.redirected_from + while current_request: + current_response = current_request.response() + + history.insert(0, Response( + url=current_request.url, + # using current_response.text() will trigger "Error: Response.text: Response body is unavailable for redirect responses" + text='', + body=b'', + status=current_response.status if current_response else 301, + reason=(current_response.status_text or StatusText.get(current_response.status)) if current_response else StatusText.get(301), + encoding=current_response.headers.get('content-type', '') or 'utf-8', + cookies={}, + headers=current_response.all_headers() if current_response else {}, + request_headers=current_request.all_headers(), + **self.adaptor_arguments + )) + current_request = current_request.redirected_from + response = Response( url=page.url, text=page.content(), @@ -269,6 +289,7 @@ class PlaywrightEngine: cookies={cookie['name']: cookie['value'] for cookie in page.context.cookies()}, headers=first_response.all_headers(), request_headers=first_response.request.all_headers(), + history=history, **self.adaptor_arguments ) page.close() @@ -345,6 +366,26 @@ class PlaywrightEngine: # PlayWright API sometimes give empty status text for some reason! status_text = final_response.status_text or StatusText.get(final_response.status) + history = [] + current_request = first_response.request.redirected_from + while current_request: + current_response = await current_request.response() + + history.insert(0, Response( + url=current_request.url, + # using current_response.text() will trigger "Error: Response.text: Response body is unavailable for redirect responses" + text='', + body=b'', + status=current_response.status if current_response else 301, + reason=(current_response.status_text or StatusText.get(current_response.status)) if current_response else StatusText.get(301), + encoding=current_response.headers.get('content-type', '') or 'utf-8', + cookies={}, + headers=await current_response.all_headers() if current_response else {}, + request_headers=await current_request.all_headers(), + **self.adaptor_arguments + )) + current_request = current_request.redirected_from + response = Response( url=page.url, text=await page.content(), @@ -355,6 +396,7 @@ class PlaywrightEngine: cookies={cookie['name']: cookie['value'] for cookie in await page.context.cookies()}, headers=await first_response.all_headers(), request_headers=await first_response.request.all_headers(), + history=history, **self.adaptor_arguments ) await page.close()