freedak/Scrapling
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(proxy control): Force a proxy at request level at any given point

Browse Source 
And merge request's meta with response's meta
This commit is contained in:
Karim shoair
2026-02-02 14:26:52 +02:00
parent 6e0277f23b
commit f88502718f
9 changed files with 54 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scrapling/engines/_browsers/_base.py
+2 -2
View File
@@ -175,7 +175,7 @@ class SyncSession:
proxy: Optional[ProxyType] = None,
) -> Generator["PageInfo[Page]", None, None]:
"""Acquire a page - either from persistent context or fresh context with proxy."""
if self._config.proxy_rotator:
if proxy:
# Rotation mode: create fresh context with the provided proxy
if not self.browser: # pragma: no cover
raise RuntimeError("Browser not initialized for proxy rotation mode")
@@ -344,7 +344,7 @@ class AsyncSession:
proxy: Optional[ProxyType] = None,
) -> AsyncGenerator["PageInfo[AsyncPage]", None]:
"""Acquire a page - either from persistent context or fresh context with proxy."""
if self._config.proxy_rotator:
if proxy:
# Rotation mode: create fresh context with the provided proxy
if not self.browser: # pragma: no cover
raise RuntimeError("Browser not initialized for proxy rotation mode")
scrapling/engines/_browsers/_controllers.py
+16 -4
View File
@@ -115,8 +115,11 @@ class DynamicSession(SyncSession, DynamicSessionMixin):
:param network_idle: Wait for the page until there are no network connections for at least 500 ms.
:param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute.
:param selector_config: The arguments that will be passed in the end while creating the final Selector's class.
:param proxy: Static proxy to override rotator and session proxy. A new browser context will be created and used with it.
:return: A `Response` object.
"""
static_proxy = kwargs.pop("proxy", None)
params = _validate(kwargs, self, PlaywrightConfig)
if not self._is_alive: # pragma: no cover
raise RuntimeError("Context manager has been closed")
@@ -129,7 +132,10 @@ class DynamicSession(SyncSession, DynamicSessionMixin):
)
for attempt in range(self._config.retries):
proxy = self._config.proxy_rotator.get_proxy() if self._config.proxy_rotator else None
if self._config.proxy_rotator and static_proxy is None:
proxy = self._config.proxy_rotator.get_proxy()
else:
proxy = static_proxy
with self._page_generator(
params.timeout, params.extra_headers, params.disable_resources, proxy
@@ -162,7 +168,7 @@ class DynamicSession(SyncSession, DynamicSessionMixin):
page.wait_for_timeout(params.wait)
response = ResponseFactory.from_playwright_response(
page, first_response, final_response[0], params.selector_config
page, first_response, final_response[0], params.selector_config, meta={"proxy": proxy}
)
return response
@@ -276,8 +282,11 @@ class AsyncDynamicSession(AsyncSession, DynamicSessionMixin):
:param network_idle: Wait for the page until there are no network connections for at least 500 ms.
:param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute.
:param selector_config: The arguments that will be passed in the end while creating the final Selector's class.
:param proxy: Static proxy to override rotator and session proxy. A new browser context will be created and used with it.
:return: A `Response` object.
"""
static_proxy = kwargs.pop("proxy", None)
params = _validate(kwargs, self, PlaywrightConfig)
if not self._is_alive: # pragma: no cover
@@ -291,7 +300,10 @@ class AsyncDynamicSession(AsyncSession, DynamicSessionMixin):
)
for attempt in range(self._config.retries):
proxy = self._config.proxy_rotator.get_proxy() if self._config.proxy_rotator else None
if self._config.proxy_rotator and static_proxy is None:
proxy = self._config.proxy_rotator.get_proxy()
else:
proxy = static_proxy
async with self._page_generator(
params.timeout, params.extra_headers, params.disable_resources, proxy
@@ -324,7 +336,7 @@ class AsyncDynamicSession(AsyncSession, DynamicSessionMixin):
await page.wait_for_timeout(params.wait)
response = await ResponseFactory.from_async_playwright_response(
page, first_response, final_response[0], params.selector_config
page, first_response, final_response[0], params.selector_config, meta={"proxy": proxy}
)
return response
scrapling/engines/_browsers/_stealth.py
+16 -4
View File
@@ -204,8 +204,11 @@ class StealthySession(SyncSession, StealthySessionMixin):
:param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute.
:param solve_cloudflare: Solves all types of the Cloudflare's Turnstile/Interstitial challenges before returning the response to you.
:param selector_config: The arguments that will be passed in the end while creating the final Selector's class.
:param proxy: Static proxy to override rotator and session proxy. A new browser context will be created and used with it.
:return: A `Response` object.
"""
static_proxy = kwargs.pop("proxy", None)
params = _validate(kwargs, self, StealthConfig)
if not self._is_alive: # pragma: no cover
raise RuntimeError("Context manager has been closed")
@@ -218,7 +221,10 @@ class StealthySession(SyncSession, StealthySessionMixin):
)
for attempt in range(self._config.retries):
proxy = self._config.proxy_rotator.get_proxy() if self._config.proxy_rotator else None
if self._config.proxy_rotator and static_proxy is None:
proxy = self._config.proxy_rotator.get_proxy()
else:
proxy = static_proxy
with self._page_generator(
params.timeout, params.extra_headers, params.disable_resources, proxy
@@ -256,7 +262,7 @@ class StealthySession(SyncSession, StealthySessionMixin):
page.wait_for_timeout(params.wait)
response = ResponseFactory.from_playwright_response(
page, first_response, final_response[0], params.selector_config
page, first_response, final_response[0], params.selector_config, meta={"proxy": proxy}
)
return response
@@ -454,8 +460,11 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin):
:param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute.
:param solve_cloudflare: Solves all types of the Cloudflare's Turnstile/Interstitial challenges before returning the response to you.
:param selector_config: The arguments that will be passed in the end while creating the final Selector's class.
:param proxy: Static proxy to override rotator and session proxy. A new browser context will be created and used with it.
:return: A `Response` object.
"""
static_proxy = kwargs.pop("proxy", None)
params = _validate(kwargs, self, StealthConfig)
if not self._is_alive: # pragma: no cover
@@ -469,7 +478,10 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin):
)
for attempt in range(self._config.retries):
proxy = self._config.proxy_rotator.get_proxy() if self._config.proxy_rotator else None
if self._config.proxy_rotator and static_proxy is None:
proxy = self._config.proxy_rotator.get_proxy()
else:
proxy = static_proxy
async with self._page_generator(
params.timeout, params.extra_headers, params.disable_resources, proxy
@@ -507,7 +519,7 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin):
await page.wait_for_timeout(params.wait)
response = await ResponseFactory.from_async_playwright_response(
page, first_response, final_response[0], params.selector_config
page, first_response, final_response[0], params.selector_config, meta={"proxy": proxy}
)
return response
scrapling/engines/_browsers/_types.py
+1
View File
@@ -99,6 +99,7 @@ if TYPE_CHECKING: # pragma: no cover
selector_config: Optional[Dict]
extra_headers: Optional[Dict[str, str]]
wait_selector_state: SelectorWaitStates
proxy: Optional[str | Dict[str, str]]
class StealthSession(PlaywrightSession, total=False):
allow_webgl: bool