feat(browser sessions): Collect XHR requests done while loading the page

Solves #159

scrapling/engines/_browsers/_base.py

@@ -1,4 +1,5 @@
 from time import time
+from re import search as re_search
 from asyncio import sleep as asyncio_sleep, Lock
 from contextlib import contextmanager, asynccontextmanager
 
@@ -146,11 +147,18 @@ class SyncSession:
             self._wait_for_networkidle(page)
 
     @staticmethod
-    def _create_response_handler(page_info: PageInfo[Page], response_container: List) -> Callable:
-        """Create a response handler that captures the final navigation response.
+    def _create_response_handler(
+        page_info: PageInfo[Page],
+        response_container: List,
+        xhr_pattern: Optional[str] = None,
+        xhr_container: Optional[List] = None,
+    ) -> Callable:
+        """Create a response handler that captures the final navigation response and optionally XHR/fetch responses.
 
         :param page_info: The PageInfo object containing the page
         :param response_container: A list to store the final response (mutable container)
+        :param xhr_pattern: Optional regex pattern to match XHR/fetch response URLs
+        :param xhr_container: Optional list to store captured XHR/fetch responses
         :return: A callback function for page.on("response", ...)
         """
 
@@ -161,6 +169,13 @@ class SyncSession:
                 and finished_response.request.frame == page_info.page.main_frame
             ):
                 response_container[0] = finished_response
+            elif (
+                xhr_pattern
+                and xhr_container is not None
+                and finished_response.request.resource_type in ("xhr", "fetch")
+                and re_search(xhr_pattern, finished_response.url)
+            ):
+                xhr_container.append(finished_response)
 
         return handle_response
 
@@ -317,11 +332,18 @@ class AsyncSession:
             await self._wait_for_networkidle(page)
 
     @staticmethod
-    def _create_response_handler(page_info: PageInfo[AsyncPage], response_container: List) -> Callable:
-        """Create an async response handler that captures the final navigation response.
+    def _create_response_handler(
+        page_info: PageInfo[AsyncPage],
+        response_container: List,
+        xhr_pattern: Optional[str] = None,
+        xhr_container: Optional[List] = None,
+    ) -> Callable:
+        """Create an async response handler that captures the final navigation response and optionally XHR/fetch responses.
 
         :param page_info: The PageInfo object containing the page
         :param response_container: A list to store the final response (mutable container)
+        :param xhr_pattern: Optional regex pattern to match XHR/fetch response URLs
+        :param xhr_container: Optional list to store captured XHR/fetch responses
         :return: A callback function for page.on("response", ...)
         """
 
@@ -332,6 +354,13 @@ class AsyncSession:
                 and finished_response.request.frame == page_info.page.main_frame
             ):
                 response_container[0] = finished_response
+            elif (
+                xhr_pattern
+                and xhr_container is not None
+                and finished_response.request.resource_type in ("xhr", "fetch")
+                and re_search(xhr_pattern, finished_response.url)
+            ):
+                xhr_container.append(finished_response)
 
         return handle_response
 

scrapling/engines/_browsers/_controllers.py

@@ -139,9 +139,17 @@ class DynamicSession(SyncSession, DynamicSessionMixin):
             with self._page_generator(
                 params.timeout, params.extra_headers, params.disable_resources, proxy, params.blocked_domains
             ) as page_info:
-                final_response = [None]
+                final_response, xhr_captured = [None], []
                 page = page_info.page
-                page.on("response", self._create_response_handler(page_info, final_response))
+                page.on(
+                    "response",
+                    self._create_response_handler(
+                        page_info,
+                        final_response,
+                        xhr_pattern=self._config.capture_xhr,
+                        xhr_container=xhr_captured,
+                    ),
+                )
 
                 try:
                     first_response = page.goto(url, referer=referer)
@@ -167,7 +175,12 @@ class DynamicSession(SyncSession, DynamicSessionMixin):
                     page.wait_for_timeout(params.wait)
 
                     response = ResponseFactory.from_playwright_response(
-                        page, first_response, final_response[0], params.selector_config, meta={"proxy": proxy}
+                        page,
+                        first_response,
+                        final_response[0],
+                        params.selector_config,
+                        meta={"proxy": proxy},
+                        xhr_captured=xhr_captured,
                     )
                     return response
 
@@ -306,9 +319,17 @@ class AsyncDynamicSession(AsyncSession, DynamicSessionMixin):
             async with self._page_generator(
                 params.timeout, params.extra_headers, params.disable_resources, proxy, params.blocked_domains
             ) as page_info:
-                final_response = [None]
+                final_response, xhr_captured = [None], []
                 page = page_info.page
-                page.on("response", self._create_response_handler(page_info, final_response))
+                page.on(
+                    "response",
+                    self._create_response_handler(
+                        page_info,
+                        final_response,
+                        xhr_pattern=self._config.capture_xhr,
+                        xhr_container=xhr_captured,
+                    ),
+                )
 
                 try:
                     first_response = await page.goto(url, referer=referer)
@@ -334,7 +355,12 @@ class AsyncDynamicSession(AsyncSession, DynamicSessionMixin):
                     await page.wait_for_timeout(params.wait)
 
                     response = await ResponseFactory.from_async_playwright_response(
-                        page, first_response, final_response[0], params.selector_config, meta={"proxy": proxy}
+                        page,
+                        first_response,
+                        final_response[0],
+                        params.selector_config,
+                        meta={"proxy": proxy},
+                        xhr_captured=xhr_captured,
                     )
                     return response
 

scrapling/engines/_browsers/_stealth.py

@@ -3,12 +3,8 @@ from re import compile as re_compile
 from time import sleep as time_sleep
 from asyncio import sleep as asyncio_sleep
 
-from playwright.sync_api import Locator, Page, BrowserContext
-from playwright.async_api import (
-    Page as async_Page,
-    Locator as AsyncLocator,
-    BrowserContext as AsyncBrowserContext,
-)
+from playwright.sync_api import Locator, Page
+from playwright.async_api import Page as async_Page, Locator as AsyncLocator
 from patchright.sync_api import sync_playwright
 from patchright.async_api import async_playwright
 
@@ -226,9 +222,17 @@ class StealthySession(SyncSession, StealthySessionMixin):
             with self._page_generator(
                 params.timeout, params.extra_headers, params.disable_resources, proxy, params.blocked_domains
             ) as page_info:
-                final_response = [None]
+                final_response, xhr_captured = [None], []
                 page = page_info.page
-                page.on("response", self._create_response_handler(page_info, final_response))
+                page.on(
+                    "response",
+                    self._create_response_handler(
+                        page_info,
+                        final_response,
+                        xhr_pattern=self._config.capture_xhr,
+                        xhr_container=xhr_captured,
+                    ),
+                )
 
                 try:
                     first_response = page.goto(url, referer=referer)
@@ -259,7 +263,12 @@ class StealthySession(SyncSession, StealthySessionMixin):
                     page.wait_for_timeout(params.wait)
 
                     response = ResponseFactory.from_playwright_response(
-                        page, first_response, final_response[0], params.selector_config, meta={"proxy": proxy}
+                        page,
+                        first_response,
+                        final_response[0],
+                        params.selector_config,
+                        meta={"proxy": proxy},
+                        xhr_captured=xhr_captured,
                     )
                     return response
 
@@ -480,9 +489,17 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin):
             async with self._page_generator(
                 params.timeout, params.extra_headers, params.disable_resources, proxy, params.blocked_domains
             ) as page_info:
-                final_response = [None]
+                final_response, xhr_captured = [None], []
                 page = page_info.page
-                page.on("response", self._create_response_handler(page_info, final_response))
+                page.on(
+                    "response",
+                    self._create_response_handler(
+                        page_info,
+                        final_response,
+                        xhr_pattern=self._config.capture_xhr,
+                        xhr_container=xhr_captured,
+                    ),
+                )
 
                 try:
                     first_response = await page.goto(url, referer=referer)
@@ -513,7 +530,12 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin):
                     await page.wait_for_timeout(params.wait)
 
                     response = await ResponseFactory.from_async_playwright_response(
-                        page, first_response, final_response[0], params.selector_config, meta={"proxy": proxy}
+                        page,
+                        first_response,
+                        final_response[0],
+                        params.selector_config,
+                        meta={"proxy": proxy},
+                        xhr_captured=xhr_captured,
                     )
                     return response
 

scrapling/engines/_browsers/_types.py

@@ -89,6 +89,7 @@ class PlaywrightSession(TypedDict, total=False):
     blocked_domains: Optional[Set[str]]
     retries: int
     retry_delay: int | float
+    capture_xhr: str | None
 
 
 class PlaywrightFetchParams(TypedDict, total=False):

scrapling/engines/_browsers/_validators.py

@@ -87,6 +87,7 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False, weakref=True):
     blocked_domains: Optional[Set[str]] = None
     retries: RetriesCount = 3
     retry_delay: Seconds = 1
+    capture_xhr: str | None = None
 
     def __post_init__(self):  # pragma: no cover
         """Custom validation after msgspec validation"""
@@ -112,6 +113,8 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False, weakref=True):
             self.selector_config = {}
         if not self.additional_args:
             self.additional_args = {}
+        if not self.capture_xhr:
+            self.capture_xhr = None
 
         if self.init_script is not None:
             validation_msg = _is_invalid_file_path(self.init_script)