freedak/Scrapling
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(browser sessions): Collect XHR requests done while loading the page

Browse Source 
Solves #159
This commit is contained in:
Karim shoair
2026-03-29 22:53:54 +02:00
parent 0f6dcccf5f
commit 68f7c5c36f
7 changed files with 135 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scrapling/engines/_browsers/_controllers.py
+32 -6
View File
@@ -139,9 +139,17 @@ class DynamicSession(SyncSession, DynamicSessionMixin):
with self._page_generator(
params.timeout, params.extra_headers, params.disable_resources, proxy, params.blocked_domains
) as page_info:
final_response = [None]
final_response, xhr_captured = [None], []
page = page_info.page
page.on("response", self._create_response_handler(page_info, final_response))
page.on(
"response",
self._create_response_handler(
page_info,
final_response,
xhr_pattern=self._config.capture_xhr,
xhr_container=xhr_captured,
),
)
try:
first_response = page.goto(url, referer=referer)
@@ -167,7 +175,12 @@ class DynamicSession(SyncSession, DynamicSessionMixin):
page.wait_for_timeout(params.wait)
response = ResponseFactory.from_playwright_response(
page, first_response, final_response[0], params.selector_config, meta={"proxy": proxy}
page,
first_response,
final_response[0],
params.selector_config,
meta={"proxy": proxy},
xhr_captured=xhr_captured,
)
return response
@@ -306,9 +319,17 @@ class AsyncDynamicSession(AsyncSession, DynamicSessionMixin):
async with self._page_generator(
params.timeout, params.extra_headers, params.disable_resources, proxy, params.blocked_domains
) as page_info:
final_response = [None]
final_response, xhr_captured = [None], []
page = page_info.page
page.on("response", self._create_response_handler(page_info, final_response))
page.on(
"response",
self._create_response_handler(
page_info,
final_response,
xhr_pattern=self._config.capture_xhr,
xhr_container=xhr_captured,
),
)
try:
first_response = await page.goto(url, referer=referer)
@@ -334,7 +355,12 @@ class AsyncDynamicSession(AsyncSession, DynamicSessionMixin):
await page.wait_for_timeout(params.wait)
response = await ResponseFactory.from_async_playwright_response(
page, first_response, final_response[0], params.selector_config, meta={"proxy": proxy}
page,
first_response,
final_response[0],
params.selector_config,
meta={"proxy": proxy},
xhr_captured=xhr_captured,
)
return response