freedak/Scrapling
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(proxy control): Force a proxy at request level at any given point

Browse Source 
And merge request's meta with response's meta
This commit is contained in:
Karim shoair
2026-02-02 14:26:52 +02:00
parent 6e0277f23b
commit f88502718f
9 changed files with 54 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scrapling/engines/_browsers/_stealth.py
+16 -4
View File
@@ -204,8 +204,11 @@ class StealthySession(SyncSession, StealthySessionMixin):
:param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute.
:param solve_cloudflare: Solves all types of the Cloudflare's Turnstile/Interstitial challenges before returning the response to you.
:param selector_config: The arguments that will be passed in the end while creating the final Selector's class.
:param proxy: Static proxy to override rotator and session proxy. A new browser context will be created and used with it.
:return: A `Response` object.
"""
static_proxy = kwargs.pop("proxy", None)
params = _validate(kwargs, self, StealthConfig)
if not self._is_alive: # pragma: no cover
raise RuntimeError("Context manager has been closed")
@@ -218,7 +221,10 @@ class StealthySession(SyncSession, StealthySessionMixin):
)
for attempt in range(self._config.retries):
proxy = self._config.proxy_rotator.get_proxy() if self._config.proxy_rotator else None
if self._config.proxy_rotator and static_proxy is None:
proxy = self._config.proxy_rotator.get_proxy()
else:
proxy = static_proxy
with self._page_generator(
params.timeout, params.extra_headers, params.disable_resources, proxy
@@ -256,7 +262,7 @@ class StealthySession(SyncSession, StealthySessionMixin):
page.wait_for_timeout(params.wait)
response = ResponseFactory.from_playwright_response(
page, first_response, final_response[0], params.selector_config
page, first_response, final_response[0], params.selector_config, meta={"proxy": proxy}
)
return response
@@ -454,8 +460,11 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin):
:param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute.
:param solve_cloudflare: Solves all types of the Cloudflare's Turnstile/Interstitial challenges before returning the response to you.
:param selector_config: The arguments that will be passed in the end while creating the final Selector's class.
:param proxy: Static proxy to override rotator and session proxy. A new browser context will be created and used with it.
:return: A `Response` object.
"""
static_proxy = kwargs.pop("proxy", None)
params = _validate(kwargs, self, StealthConfig)
if not self._is_alive: # pragma: no cover
@@ -469,7 +478,10 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin):
)
for attempt in range(self._config.retries):
proxy = self._config.proxy_rotator.get_proxy() if self._config.proxy_rotator else None
if self._config.proxy_rotator and static_proxy is None:
proxy = self._config.proxy_rotator.get_proxy()
else:
proxy = static_proxy
async with self._page_generator(
params.timeout, params.extra_headers, params.disable_resources, proxy
@@ -507,7 +519,7 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin):
await page.wait_for_timeout(params.wait)
response = await ResponseFactory.from_async_playwright_response(
page, first_response, final_response[0], params.selector_config
page, first_response, final_response[0], params.selector_config, meta={"proxy": proxy}
)
return response