fix: added raw_response to separate collected response from rendered response

scrapling/engines/_browsers/_camoufox.py

@@ -343,7 +343,7 @@ class StealthySession(StealthySessionMixin, SyncSession):
 
             page_info.page.wait_for_timeout(params.wait)
             response = ResponseFactory.from_playwright_response(
-                page_info.page, first_response, final_response[0], params.selector_config, bool(params.page_action)
+                page_info.page, first_response, final_response[0], params.selector_config
             )
 
             # Close the page to free up resources
@@ -636,7 +636,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):
 
             # Create response object
             response = await ResponseFactory.from_async_playwright_response(
-                page_info.page, first_response, final_response[0], params.selector_config, bool(params.page_action)
+                page_info.page, first_response, final_response[0], params.selector_config
             )
 
             # Close the page to free up resources

scrapling/engines/_browsers/_controllers.py

@@ -268,7 +268,7 @@ class DynamicSession(DynamicSessionMixin, SyncSession):
 
             # Create response object
             response = ResponseFactory.from_playwright_response(
-                page_info.page, first_response, final_response[0], params.selector_config, bool(params.page_action)
+                page_info.page, first_response, final_response[0], params.selector_config
             )
 
             # Close the page to free up resources
@@ -492,7 +492,7 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession):
 
             # Create response object
             response = await ResponseFactory.from_async_playwright_response(
-                page_info.page, first_response, final_response[0], params.selector_config, bool(params.page_action)
+                page_info.page, first_response, final_response[0], params.selector_config
             )
 
             # Close the page to free up resources

scrapling/engines/toolbelt/convertor.py

@@ -85,7 +85,6 @@ class ResponseFactory:
         first_response: SyncResponse,
         final_response: Optional[SyncResponse],
         parser_arguments: Dict,
-        automated_page: bool = False,
     ) -> Response:
         """
         Transforms a Playwright response into an internal `Response` object, encapsulating
@@ -101,7 +100,6 @@ class ResponseFactory:
         :param first_response: An earlier or initial Playwright `Response` object that may serve as a fallback response in the absence of the final one.
         :param parser_arguments: A dictionary containing additional arguments needed for parsing or further customization of the returned `Response`. These arguments are dynamically unpacked into
             the `Response` object.
-        :param automated_page: If True, it means the `page_action` argument was being used, so the response retrieving method changes to use Playwright's page instead of the final response.
 
         :return: A fully populated `Response` object containing the page's URL, content, status, headers, cookies, and other derived metadata.
         :rtype: Response
@@ -117,7 +115,7 @@ class ResponseFactory:
 
         history = cls._process_response_history(first_response, parser_arguments)
         try:
-            page_content = final_response.text() if not automated_page else cls._get_page_content(page)
+            page_content = cls._get_page_content(page)
         except Exception as e:  # pragma: no cover
             log.error(f"Error getting page content: {e}")
             page_content = ""
@@ -126,6 +124,7 @@ class ResponseFactory:
             **{
                 "url": page.url,
                 "content": page_content,
+                "raw_response": final_response.text(),
                 "status": final_response.status,
                 "reason": status_text,
                 "encoding": encoding,
@@ -219,7 +218,6 @@ class ResponseFactory:
         first_response: AsyncResponse,
         final_response: Optional[AsyncResponse],
         parser_arguments: Dict,
-        automated_page: bool = False,
     ) -> Response:
         """
         Transforms a Playwright response into an internal `Response` object, encapsulating
@@ -235,7 +233,6 @@ class ResponseFactory:
         :param first_response: An earlier or initial Playwright `Response` object that may serve as a fallback response in the absence of the final one.
         :param parser_arguments: A dictionary containing additional arguments needed for parsing or further customization of the returned `Response`. These arguments are dynamically unpacked into
             the `Response` object.
-        :param automated_page: If True, it means the `page_action` argument was being used, so the response retrieving method changes to use Playwright's page instead of the final response.
 
         :return: A fully populated `Response` object containing the page's URL, content, status, headers, cookies, and other derived metadata.
         :rtype: Response
@@ -251,7 +248,7 @@ class ResponseFactory:
 
         history = await cls._async_process_response_history(first_response, parser_arguments)
         try:
-            page_content = await (final_response.text() if not automated_page else cls._get_async_page_content(page))
+            page_content = await cls._get_async_page_content(page)
         except Exception as e:  # pragma: no cover
             log.error(f"Error getting page content in async: {e}")
             page_content = ""
@@ -261,6 +258,7 @@ class ResponseFactory:
                 "url": page.url,
                 "content": page_content,
                 "status": final_response.status,
+                "raw_response": await final_response.text(),
                 "reason": status_text,
                 "encoding": encoding,
                 "cookies": tuple(dict(cookie) for cookie in await page.context.cookies()),

scrapling/engines/toolbelt/custom.py

@@ -31,6 +31,7 @@ class Response(Selector):
         request_headers: Dict,
         encoding: str = "utf-8",
         method: str = "GET",
+        raw_response: str | bytes = "",
         history: List | None = None,
         **selector_config: Any,
     ):
@@ -39,6 +40,7 @@ class Response(Selector):
         self.reason = reason
         self.cookies = cookies
         self.headers = headers
+        self.raw_response = raw_response or content
         self.request_headers = request_headers
         self.history = history or []
         super().__init__(