diff --git a/scrapling/engines/_browsers/_camoufox.py b/scrapling/engines/_browsers/_camoufox.py index 4ac0558..8d70dbd 100644 --- a/scrapling/engines/_browsers/_camoufox.py +++ b/scrapling/engines/_browsers/_camoufox.py @@ -343,7 +343,7 @@ class StealthySession(StealthySessionMixin, SyncSession): page_info.page.wait_for_timeout(params.wait) response = ResponseFactory.from_playwright_response( - page_info.page, first_response, final_response[0], params.selector_config, bool(params.page_action) + page_info.page, first_response, final_response[0], params.selector_config ) # Close the page to free up resources @@ -636,7 +636,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession): # Create response object response = await ResponseFactory.from_async_playwright_response( - page_info.page, first_response, final_response[0], params.selector_config, bool(params.page_action) + page_info.page, first_response, final_response[0], params.selector_config ) # Close the page to free up resources diff --git a/scrapling/engines/_browsers/_controllers.py b/scrapling/engines/_browsers/_controllers.py index bcfd0ac..9fc3490 100644 --- a/scrapling/engines/_browsers/_controllers.py +++ b/scrapling/engines/_browsers/_controllers.py @@ -268,7 +268,7 @@ class DynamicSession(DynamicSessionMixin, SyncSession): # Create response object response = ResponseFactory.from_playwright_response( - page_info.page, first_response, final_response[0], params.selector_config, bool(params.page_action) + page_info.page, first_response, final_response[0], params.selector_config ) # Close the page to free up resources @@ -492,7 +492,7 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession): # Create response object response = await ResponseFactory.from_async_playwright_response( - page_info.page, first_response, final_response[0], params.selector_config, bool(params.page_action) + page_info.page, first_response, final_response[0], params.selector_config ) # Close the page to free up resources diff --git a/scrapling/engines/toolbelt/convertor.py b/scrapling/engines/toolbelt/convertor.py index 5bb4a49..de705a9 100644 --- a/scrapling/engines/toolbelt/convertor.py +++ b/scrapling/engines/toolbelt/convertor.py @@ -85,7 +85,6 @@ class ResponseFactory: first_response: SyncResponse, final_response: Optional[SyncResponse], parser_arguments: Dict, - automated_page: bool = False, ) -> Response: """ Transforms a Playwright response into an internal `Response` object, encapsulating @@ -101,7 +100,6 @@ class ResponseFactory: :param first_response: An earlier or initial Playwright `Response` object that may serve as a fallback response in the absence of the final one. :param parser_arguments: A dictionary containing additional arguments needed for parsing or further customization of the returned `Response`. These arguments are dynamically unpacked into the `Response` object. - :param automated_page: If True, it means the `page_action` argument was being used, so the response retrieving method changes to use Playwright's page instead of the final response. :return: A fully populated `Response` object containing the page's URL, content, status, headers, cookies, and other derived metadata. :rtype: Response @@ -117,7 +115,7 @@ class ResponseFactory: history = cls._process_response_history(first_response, parser_arguments) try: - page_content = final_response.text() if not automated_page else cls._get_page_content(page) + page_content = cls._get_page_content(page) except Exception as e: # pragma: no cover log.error(f"Error getting page content: {e}") page_content = "" @@ -126,6 +124,7 @@ class ResponseFactory: **{ "url": page.url, "content": page_content, + "raw_response": final_response.text(), "status": final_response.status, "reason": status_text, "encoding": encoding, @@ -219,7 +218,6 @@ class ResponseFactory: first_response: AsyncResponse, final_response: Optional[AsyncResponse], parser_arguments: Dict, - automated_page: bool = False, ) -> Response: """ Transforms a Playwright response into an internal `Response` object, encapsulating @@ -235,7 +233,6 @@ class ResponseFactory: :param first_response: An earlier or initial Playwright `Response` object that may serve as a fallback response in the absence of the final one. :param parser_arguments: A dictionary containing additional arguments needed for parsing or further customization of the returned `Response`. These arguments are dynamically unpacked into the `Response` object. - :param automated_page: If True, it means the `page_action` argument was being used, so the response retrieving method changes to use Playwright's page instead of the final response. :return: A fully populated `Response` object containing the page's URL, content, status, headers, cookies, and other derived metadata. :rtype: Response @@ -251,7 +248,7 @@ class ResponseFactory: history = await cls._async_process_response_history(first_response, parser_arguments) try: - page_content = await (final_response.text() if not automated_page else cls._get_async_page_content(page)) + page_content = await cls._get_async_page_content(page) except Exception as e: # pragma: no cover log.error(f"Error getting page content in async: {e}") page_content = "" @@ -261,6 +258,7 @@ class ResponseFactory: "url": page.url, "content": page_content, "status": final_response.status, + "raw_response": await final_response.text(), "reason": status_text, "encoding": encoding, "cookies": tuple(dict(cookie) for cookie in await page.context.cookies()), diff --git a/scrapling/engines/toolbelt/custom.py b/scrapling/engines/toolbelt/custom.py index 1f20038..44e550f 100644 --- a/scrapling/engines/toolbelt/custom.py +++ b/scrapling/engines/toolbelt/custom.py @@ -31,6 +31,7 @@ class Response(Selector): request_headers: Dict, encoding: str = "utf-8", method: str = "GET", + raw_response: str | bytes = "", history: List | None = None, **selector_config: Any, ): @@ -39,6 +40,7 @@ class Response(Selector): self.reason = reason self.cookies = cookies self.headers = headers + self.raw_response = raw_response or content self.request_headers = request_headers self.history = history or [] super().__init__(