From 68f7c5c36f043855db16c48a7679e7a9ed33e6ef Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Sun, 29 Mar 2026 22:53:54 +0200 Subject: [PATCH] feat(browser sessions): Collect XHR requests done while loading the page Solves #159 --- scrapling/engines/_browsers/_base.py | 37 +++++++++++++++-- scrapling/engines/_browsers/_controllers.py | 38 ++++++++++++++--- scrapling/engines/_browsers/_stealth.py | 46 +++++++++++++++------ scrapling/engines/_browsers/_types.py | 1 + scrapling/engines/_browsers/_validators.py | 3 ++ scrapling/engines/toolbelt/convertor.py | 45 +++++++++++++------- scrapling/engines/toolbelt/custom.py | 1 + 7 files changed, 135 insertions(+), 36 deletions(-) diff --git a/scrapling/engines/_browsers/_base.py b/scrapling/engines/_browsers/_base.py index 501db76..15371f7 100644 --- a/scrapling/engines/_browsers/_base.py +++ b/scrapling/engines/_browsers/_base.py @@ -1,4 +1,5 @@ from time import time +from re import search as re_search from asyncio import sleep as asyncio_sleep, Lock from contextlib import contextmanager, asynccontextmanager @@ -146,11 +147,18 @@ class SyncSession: self._wait_for_networkidle(page) @staticmethod - def _create_response_handler(page_info: PageInfo[Page], response_container: List) -> Callable: - """Create a response handler that captures the final navigation response. + def _create_response_handler( + page_info: PageInfo[Page], + response_container: List, + xhr_pattern: Optional[str] = None, + xhr_container: Optional[List] = None, + ) -> Callable: + """Create a response handler that captures the final navigation response and optionally XHR/fetch responses. :param page_info: The PageInfo object containing the page :param response_container: A list to store the final response (mutable container) + :param xhr_pattern: Optional regex pattern to match XHR/fetch response URLs + :param xhr_container: Optional list to store captured XHR/fetch responses :return: A callback function for page.on("response", ...) """ @@ -161,6 +169,13 @@ class SyncSession: and finished_response.request.frame == page_info.page.main_frame ): response_container[0] = finished_response + elif ( + xhr_pattern + and xhr_container is not None + and finished_response.request.resource_type in ("xhr", "fetch") + and re_search(xhr_pattern, finished_response.url) + ): + xhr_container.append(finished_response) return handle_response @@ -317,11 +332,18 @@ class AsyncSession: await self._wait_for_networkidle(page) @staticmethod - def _create_response_handler(page_info: PageInfo[AsyncPage], response_container: List) -> Callable: - """Create an async response handler that captures the final navigation response. + def _create_response_handler( + page_info: PageInfo[AsyncPage], + response_container: List, + xhr_pattern: Optional[str] = None, + xhr_container: Optional[List] = None, + ) -> Callable: + """Create an async response handler that captures the final navigation response and optionally XHR/fetch responses. :param page_info: The PageInfo object containing the page :param response_container: A list to store the final response (mutable container) + :param xhr_pattern: Optional regex pattern to match XHR/fetch response URLs + :param xhr_container: Optional list to store captured XHR/fetch responses :return: A callback function for page.on("response", ...) """ @@ -332,6 +354,13 @@ class AsyncSession: and finished_response.request.frame == page_info.page.main_frame ): response_container[0] = finished_response + elif ( + xhr_pattern + and xhr_container is not None + and finished_response.request.resource_type in ("xhr", "fetch") + and re_search(xhr_pattern, finished_response.url) + ): + xhr_container.append(finished_response) return handle_response diff --git a/scrapling/engines/_browsers/_controllers.py b/scrapling/engines/_browsers/_controllers.py index fa6e8b6..f788819 100644 --- a/scrapling/engines/_browsers/_controllers.py +++ b/scrapling/engines/_browsers/_controllers.py @@ -139,9 +139,17 @@ class DynamicSession(SyncSession, DynamicSessionMixin): with self._page_generator( params.timeout, params.extra_headers, params.disable_resources, proxy, params.blocked_domains ) as page_info: - final_response = [None] + final_response, xhr_captured = [None], [] page = page_info.page - page.on("response", self._create_response_handler(page_info, final_response)) + page.on( + "response", + self._create_response_handler( + page_info, + final_response, + xhr_pattern=self._config.capture_xhr, + xhr_container=xhr_captured, + ), + ) try: first_response = page.goto(url, referer=referer) @@ -167,7 +175,12 @@ class DynamicSession(SyncSession, DynamicSessionMixin): page.wait_for_timeout(params.wait) response = ResponseFactory.from_playwright_response( - page, first_response, final_response[0], params.selector_config, meta={"proxy": proxy} + page, + first_response, + final_response[0], + params.selector_config, + meta={"proxy": proxy}, + xhr_captured=xhr_captured, ) return response @@ -306,9 +319,17 @@ class AsyncDynamicSession(AsyncSession, DynamicSessionMixin): async with self._page_generator( params.timeout, params.extra_headers, params.disable_resources, proxy, params.blocked_domains ) as page_info: - final_response = [None] + final_response, xhr_captured = [None], [] page = page_info.page - page.on("response", self._create_response_handler(page_info, final_response)) + page.on( + "response", + self._create_response_handler( + page_info, + final_response, + xhr_pattern=self._config.capture_xhr, + xhr_container=xhr_captured, + ), + ) try: first_response = await page.goto(url, referer=referer) @@ -334,7 +355,12 @@ class AsyncDynamicSession(AsyncSession, DynamicSessionMixin): await page.wait_for_timeout(params.wait) response = await ResponseFactory.from_async_playwright_response( - page, first_response, final_response[0], params.selector_config, meta={"proxy": proxy} + page, + first_response, + final_response[0], + params.selector_config, + meta={"proxy": proxy}, + xhr_captured=xhr_captured, ) return response diff --git a/scrapling/engines/_browsers/_stealth.py b/scrapling/engines/_browsers/_stealth.py index 8fb248f..ef3a445 100644 --- a/scrapling/engines/_browsers/_stealth.py +++ b/scrapling/engines/_browsers/_stealth.py @@ -3,12 +3,8 @@ from re import compile as re_compile from time import sleep as time_sleep from asyncio import sleep as asyncio_sleep -from playwright.sync_api import Locator, Page, BrowserContext -from playwright.async_api import ( - Page as async_Page, - Locator as AsyncLocator, - BrowserContext as AsyncBrowserContext, -) +from playwright.sync_api import Locator, Page +from playwright.async_api import Page as async_Page, Locator as AsyncLocator from patchright.sync_api import sync_playwright from patchright.async_api import async_playwright @@ -226,9 +222,17 @@ class StealthySession(SyncSession, StealthySessionMixin): with self._page_generator( params.timeout, params.extra_headers, params.disable_resources, proxy, params.blocked_domains ) as page_info: - final_response = [None] + final_response, xhr_captured = [None], [] page = page_info.page - page.on("response", self._create_response_handler(page_info, final_response)) + page.on( + "response", + self._create_response_handler( + page_info, + final_response, + xhr_pattern=self._config.capture_xhr, + xhr_container=xhr_captured, + ), + ) try: first_response = page.goto(url, referer=referer) @@ -259,7 +263,12 @@ class StealthySession(SyncSession, StealthySessionMixin): page.wait_for_timeout(params.wait) response = ResponseFactory.from_playwright_response( - page, first_response, final_response[0], params.selector_config, meta={"proxy": proxy} + page, + first_response, + final_response[0], + params.selector_config, + meta={"proxy": proxy}, + xhr_captured=xhr_captured, ) return response @@ -480,9 +489,17 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin): async with self._page_generator( params.timeout, params.extra_headers, params.disable_resources, proxy, params.blocked_domains ) as page_info: - final_response = [None] + final_response, xhr_captured = [None], [] page = page_info.page - page.on("response", self._create_response_handler(page_info, final_response)) + page.on( + "response", + self._create_response_handler( + page_info, + final_response, + xhr_pattern=self._config.capture_xhr, + xhr_container=xhr_captured, + ), + ) try: first_response = await page.goto(url, referer=referer) @@ -513,7 +530,12 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin): await page.wait_for_timeout(params.wait) response = await ResponseFactory.from_async_playwright_response( - page, first_response, final_response[0], params.selector_config, meta={"proxy": proxy} + page, + first_response, + final_response[0], + params.selector_config, + meta={"proxy": proxy}, + xhr_captured=xhr_captured, ) return response diff --git a/scrapling/engines/_browsers/_types.py b/scrapling/engines/_browsers/_types.py index 30d6af1..f4f31cd 100644 --- a/scrapling/engines/_browsers/_types.py +++ b/scrapling/engines/_browsers/_types.py @@ -89,6 +89,7 @@ class PlaywrightSession(TypedDict, total=False): blocked_domains: Optional[Set[str]] retries: int retry_delay: int | float + capture_xhr: str | None class PlaywrightFetchParams(TypedDict, total=False): diff --git a/scrapling/engines/_browsers/_validators.py b/scrapling/engines/_browsers/_validators.py index e2424f4..dfe7310 100644 --- a/scrapling/engines/_browsers/_validators.py +++ b/scrapling/engines/_browsers/_validators.py @@ -87,6 +87,7 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False, weakref=True): blocked_domains: Optional[Set[str]] = None retries: RetriesCount = 3 retry_delay: Seconds = 1 + capture_xhr: str | None = None def __post_init__(self): # pragma: no cover """Custom validation after msgspec validation""" @@ -112,6 +113,8 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False, weakref=True): self.selector_config = {} if not self.additional_args: self.additional_args = {} + if not self.capture_xhr: + self.capture_xhr = None if self.init_script is not None: validation_msg = _is_invalid_file_path(self.init_script) diff --git a/scrapling/engines/toolbelt/convertor.py b/scrapling/engines/toolbelt/convertor.py index 8606003..8d624e0 100644 --- a/scrapling/engines/toolbelt/convertor.py +++ b/scrapling/engines/toolbelt/convertor.py @@ -8,7 +8,7 @@ from playwright.async_api import Page as AsyncPage, Response as AsyncResponse from scrapling.core.utils import log from .custom import Response, StatusText -from scrapling.core._types import Dict, Optional +from scrapling.core._types import Dict, List, Optional __CHARSET_RE__ = re_compile(r"charset=([\w-]+)") @@ -81,11 +81,13 @@ class ResponseFactory: @classmethod def from_playwright_response( cls, - page: SyncPage, + page: Optional[SyncPage], first_response: SyncResponse, final_response: Optional[SyncResponse], parser_arguments: Dict, meta: Optional[Dict] = None, + xhr_captured: Optional[List[SyncResponse]] = None, + collect_history: bool = True, ) -> Response: """ Transforms a Playwright response into an internal `Response` object, encapsulating @@ -102,7 +104,8 @@ class ResponseFactory: :param parser_arguments: A dictionary containing additional arguments needed for parsing or further customization of the returned `Response`. These arguments are dynamically unpacked into the `Response` object. :param meta: Additional meta data to be saved with the response. - + :param xhr_captured: Optional list of captured Playwright XHR/fetch responses to convert and attach to the returned Response. + :param collect_history: Optional boolean indicating whether to collect redirections history or not. :return: A fully populated `Response` object containing the page's URL, content, status, headers, cookies, and other derived metadata. :rtype: Response """ @@ -115,9 +118,9 @@ class ResponseFactory: # PlayWright API sometimes give empty status text for some reason! status_text = final_response.status_text or StatusText.get(final_response.status) - history = cls._process_response_history(first_response, parser_arguments) + history = cls._process_response_history(first_response, parser_arguments) if collect_history else [] try: - if "html" in final_response.all_headers().get("content-type", ""): + if page and "html" in final_response.all_headers().get("content-type", ""): page_content = cls._get_page_content(page).encode("utf-8") else: page_content = final_response.body() @@ -125,14 +128,14 @@ class ResponseFactory: log.error(f"Error getting page content: {e}") page_content = b"" - return Response( + response = Response( **{ - "url": page.url, + "url": page.url if page else first_response.url, "content": page_content, "status": final_response.status, "reason": status_text, "encoding": encoding, - "cookies": tuple(dict(cookie) for cookie in page.context.cookies()), + "cookies": tuple(dict(cookie) for cookie in page.context.cookies()) if page else {}, "headers": first_response.all_headers(), "request_headers": first_response.request.all_headers(), "history": history, @@ -140,6 +143,11 @@ class ResponseFactory: **parser_arguments, } ) + if xhr_captured: + response.captured_xhr = [ + cls.from_playwright_response(None, p, None, {}, collect_history=False) for p in xhr_captured + ] + return response @classmethod async def _async_process_response_history( @@ -219,11 +227,13 @@ class ResponseFactory: @classmethod async def from_async_playwright_response( cls, - page: AsyncPage, + page: Optional[AsyncPage], first_response: AsyncResponse, final_response: Optional[AsyncResponse], parser_arguments: Dict, meta: Optional[Dict] = None, + xhr_captured: Optional[List[AsyncResponse]] = None, + collect_history: bool = True, ) -> Response: """ Transforms a Playwright response into an internal `Response` object, encapsulating @@ -240,6 +250,8 @@ class ResponseFactory: :param parser_arguments: A dictionary containing additional arguments needed for parsing or further customization of the returned `Response`. These arguments are dynamically unpacked into the `Response` object. :param meta: Additional meta data to be saved with the response. + :param xhr_captured: Optional list of captured async Playwright XHR/fetch responses to convert and attach to the returned Response. + :param collect_history: Optional boolean indicating whether to collect redirections history or not. :return: A fully populated `Response` object containing the page's URL, content, status, headers, cookies, and other derived metadata. :rtype: Response @@ -253,9 +265,9 @@ class ResponseFactory: # PlayWright API sometimes give empty status text for some reason! status_text = final_response.status_text or StatusText.get(final_response.status) - history = await cls._async_process_response_history(first_response, parser_arguments) + history = await cls._async_process_response_history(first_response, parser_arguments) if collect_history else [] try: - if "html" in (await final_response.all_headers()).get("content-type", ""): + if page and "html" in (await final_response.all_headers()).get("content-type", ""): page_content = (await cls._get_async_page_content(page)).encode("utf-8") else: page_content = await final_response.body() @@ -263,14 +275,14 @@ class ResponseFactory: log.error(f"Error getting page content in async: {e}") page_content = b"" - return Response( + response = Response( **{ - "url": page.url, + "url": page.url if page else first_response.url, "content": page_content, "status": final_response.status, "reason": status_text, "encoding": encoding, - "cookies": tuple(dict(cookie) for cookie in await page.context.cookies()), + "cookies": tuple(dict(cookie) for cookie in await page.context.cookies()) if page else {}, "headers": await first_response.all_headers(), "request_headers": await first_response.request.all_headers(), "history": history, @@ -278,6 +290,11 @@ class ResponseFactory: **parser_arguments, } ) + if xhr_captured: + response.captured_xhr = [ + await cls.from_async_playwright_response(None, p, None, {}, collect_history=False) for p in xhr_captured + ] + return response @staticmethod def from_http_request(response: CurlResponse, parser_arguments: Dict, meta: Optional[Dict] = None) -> Response: diff --git a/scrapling/engines/toolbelt/custom.py b/scrapling/engines/toolbelt/custom.py index 58a7485..483a32b 100644 --- a/scrapling/engines/toolbelt/custom.py +++ b/scrapling/engines/toolbelt/custom.py @@ -67,6 +67,7 @@ class Response(Selector): self.meta: Dict[str, Any] = meta or {} self.request: Optional["Request"] = None # Will be set by crawler + self.captured_xhr: List["Response"] = [] @property def body(self) -> bytes: