feat(browser sessions): Collect XHR requests done while loading the page

Solves #159

scrapling/engines/toolbelt/convertor.py

@@ -8,7 +8,7 @@ from playwright.async_api import Page as AsyncPage, Response as AsyncResponse
 
 from scrapling.core.utils import log
 from .custom import Response, StatusText
-from scrapling.core._types import Dict, Optional
+from scrapling.core._types import Dict, List, Optional
 
 __CHARSET_RE__ = re_compile(r"charset=([\w-]+)")
 
@@ -81,11 +81,13 @@ class ResponseFactory:
     @classmethod
     def from_playwright_response(
         cls,
-        page: SyncPage,
+        page: Optional[SyncPage],
         first_response: SyncResponse,
         final_response: Optional[SyncResponse],
         parser_arguments: Dict,
         meta: Optional[Dict] = None,
+        xhr_captured: Optional[List[SyncResponse]] = None,
+        collect_history: bool = True,
     ) -> Response:
         """
         Transforms a Playwright response into an internal `Response` object, encapsulating
@@ -102,7 +104,8 @@ class ResponseFactory:
         :param parser_arguments: A dictionary containing additional arguments needed for parsing or further customization of the returned `Response`. These arguments are dynamically unpacked into
             the `Response` object.
         :param meta: Additional meta data to be saved with the response.
-
+        :param xhr_captured: Optional list of captured Playwright XHR/fetch responses to convert and attach to the returned Response.
+        :param collect_history: Optional boolean indicating whether to collect redirections history or not.
         :return: A fully populated `Response` object containing the page's URL, content, status, headers, cookies, and other derived metadata.
         :rtype: Response
         """
@@ -115,9 +118,9 @@ class ResponseFactory:
         # PlayWright API sometimes give empty status text for some reason!
         status_text = final_response.status_text or StatusText.get(final_response.status)
 
-        history = cls._process_response_history(first_response, parser_arguments)
+        history = cls._process_response_history(first_response, parser_arguments) if collect_history else []
         try:
-            if "html" in final_response.all_headers().get("content-type", ""):
+            if page and "html" in final_response.all_headers().get("content-type", ""):
                 page_content = cls._get_page_content(page).encode("utf-8")
             else:
                 page_content = final_response.body()
@@ -125,14 +128,14 @@ class ResponseFactory:
             log.error(f"Error getting page content: {e}")
             page_content = b""
 
-        return Response(
+        response = Response(
             **{
-                "url": page.url,
+                "url": page.url if page else first_response.url,
                 "content": page_content,
                 "status": final_response.status,
                 "reason": status_text,
                 "encoding": encoding,
-                "cookies": tuple(dict(cookie) for cookie in page.context.cookies()),
+                "cookies": tuple(dict(cookie) for cookie in page.context.cookies()) if page else {},
                 "headers": first_response.all_headers(),
                 "request_headers": first_response.request.all_headers(),
                 "history": history,
@@ -140,6 +143,11 @@ class ResponseFactory:
                 **parser_arguments,
             }
         )
+        if xhr_captured:
+            response.captured_xhr = [
+                cls.from_playwright_response(None, p, None, {}, collect_history=False) for p in xhr_captured
+            ]
+        return response
 
     @classmethod
     async def _async_process_response_history(
@@ -219,11 +227,13 @@ class ResponseFactory:
     @classmethod
     async def from_async_playwright_response(
         cls,
-        page: AsyncPage,
+        page: Optional[AsyncPage],
         first_response: AsyncResponse,
         final_response: Optional[AsyncResponse],
         parser_arguments: Dict,
         meta: Optional[Dict] = None,
+        xhr_captured: Optional[List[AsyncResponse]] = None,
+        collect_history: bool = True,
     ) -> Response:
         """
         Transforms a Playwright response into an internal `Response` object, encapsulating
@@ -240,6 +250,8 @@ class ResponseFactory:
         :param parser_arguments: A dictionary containing additional arguments needed for parsing or further customization of the returned `Response`. These arguments are dynamically unpacked into
             the `Response` object.
         :param meta: Additional meta data to be saved with the response.
+        :param xhr_captured: Optional list of captured async Playwright XHR/fetch responses to convert and attach to the returned Response.
+        :param collect_history: Optional boolean indicating whether to collect redirections history or not.
 
         :return: A fully populated `Response` object containing the page's URL, content, status, headers, cookies, and other derived metadata.
         :rtype: Response
@@ -253,9 +265,9 @@ class ResponseFactory:
         # PlayWright API sometimes give empty status text for some reason!
         status_text = final_response.status_text or StatusText.get(final_response.status)
 
-        history = await cls._async_process_response_history(first_response, parser_arguments)
+        history = await cls._async_process_response_history(first_response, parser_arguments) if collect_history else []
         try:
-            if "html" in (await final_response.all_headers()).get("content-type", ""):
+            if page and "html" in (await final_response.all_headers()).get("content-type", ""):
                 page_content = (await cls._get_async_page_content(page)).encode("utf-8")
             else:
                 page_content = await final_response.body()
@@ -263,14 +275,14 @@ class ResponseFactory:
             log.error(f"Error getting page content in async: {e}")
             page_content = b""
 
-        return Response(
+        response = Response(
             **{
-                "url": page.url,
+                "url": page.url if page else first_response.url,
                 "content": page_content,
                 "status": final_response.status,
                 "reason": status_text,
                 "encoding": encoding,
-                "cookies": tuple(dict(cookie) for cookie in await page.context.cookies()),
+                "cookies": tuple(dict(cookie) for cookie in await page.context.cookies()) if page else {},
                 "headers": await first_response.all_headers(),
                 "request_headers": await first_response.request.all_headers(),
                 "history": history,
@@ -278,6 +290,11 @@ class ResponseFactory:
                 **parser_arguments,
             }
         )
+        if xhr_captured:
+            response.captured_xhr = [
+                await cls.from_async_playwright_response(None, p, None, {}, collect_history=False) for p in xhr_captured
+            ]
+        return response
 
     @staticmethod
     def from_http_request(response: CurlResponse, parser_arguments: Dict, meta: Optional[Dict] = None) -> Response:

scrapling/engines/toolbelt/custom.py

@@ -67,6 +67,7 @@ class Response(Selector):
 
         self.meta: Dict[str, Any] = meta or {}
         self.request: Optional["Request"] = None  # Will be set by crawler
+        self.captured_xhr: List["Response"] = []
 
     @property
     def body(self) -> bytes: