From 74c5848060cae99e7076add01397609d5232e043 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Sun, 7 Jun 2026 18:35:18 +0300 Subject: [PATCH] fix: apply the session-level proxy when no per-request proxy is given The per-request proxy resolution never fell back to the session default, so FetcherSession(proxy=...) was silently ignored, and requests went direct. Same fix in the sync and async paths, with regression tests asserting on the proxy that reaches curl_cffi. Closes #295 --- scrapling/engines/static.py | 7 ++- tests/fetchers/async/test_requests_session.py | 51 ++++++++++++++++++- tests/fetchers/sync/test_requests_session.py | 51 ++++++++++++++++--- 3 files changed, 99 insertions(+), 10 deletions(-) diff --git a/scrapling/engines/static.py b/scrapling/engines/static.py index 9b3e951..6f18b2a 100644 --- a/scrapling/engines/static.py +++ b/scrapling/engines/static.py @@ -19,6 +19,7 @@ from scrapling.core._types import ( Unpack, Optional, Awaitable, + ProxyType, SUPPORTED_HTTP_METHODS, FollowRedirects, ) @@ -244,10 +245,11 @@ class _SyncSessionLogic(_ConfigurationLogic): try: for attempt in range(max_retries): + proxy: Optional[ProxyType] if self._proxy_rotator and static_proxy is None: proxy = self._proxy_rotator.get_proxy() else: - proxy = static_proxy + proxy = static_proxy or self._default_proxy request_args = self._merge_request_args(stealth=stealth, proxy=proxy, **kwargs) try: @@ -461,10 +463,11 @@ class _ASyncSessionLogic(_ConfigurationLogic): try: # Determine if we should use proxy rotation for attempt in range(max_retries): + proxy: Optional[ProxyType] if self._proxy_rotator and static_proxy is None: proxy = self._proxy_rotator.get_proxy() else: - proxy = static_proxy + proxy = static_proxy or self._default_proxy request_args = self._merge_request_args(stealth=stealth, proxy=proxy, **kwargs) try: diff --git a/tests/fetchers/async/test_requests_session.py b/tests/fetchers/async/test_requests_session.py index 3846e69..442a11f 100644 --- a/tests/fetchers/async/test_requests_session.py +++ b/tests/fetchers/async/test_requests_session.py @@ -1,6 +1,9 @@ +import pytest +from unittest.mock import patch, MagicMock, AsyncMock +from curl_cffi.curl import CurlError - -from scrapling.engines.static import AsyncFetcherClient +from scrapling.engines.static import _ASyncSessionLogic as AsyncFetcherSession, AsyncFetcherClient +from scrapling.engines.toolbelt import ProxyRotator class TestFetcherSession: @@ -13,3 +16,47 @@ class TestFetcherSession: # Should not have context manager methods assert client.__aenter__ is None assert client.__aexit__ is None + + @pytest.mark.asyncio + async def test_session_level_proxy_is_applied(self): + """Session-level proxy must reach the request, not be silently dropped (#295)""" + proxy = "http://10.255.255.1:9999" + + async with AsyncFetcherSession(proxy=proxy) as session: + with ( + patch.object(session._async_curl_session, "request", new=AsyncMock()) as mocked_request, + patch("scrapling.engines.static.ResponseFactory.from_http_request", return_value=MagicMock()), + ): + await session.get("http://example.com") + + assert mocked_request.call_args.kwargs["proxy"] == proxy + + @pytest.mark.asyncio + async def test_per_request_proxy_overrides_session_proxy(self): + """A per-request proxy must take precedence over the session-level proxy""" + request_proxy = "http://10.255.255.2:9999" + + async with AsyncFetcherSession(proxy="http://10.255.255.1:9999") as session: + with ( + patch.object(session._async_curl_session, "request", new=AsyncMock()) as mocked_request, + patch("scrapling.engines.static.ResponseFactory.from_http_request", return_value=MagicMock()), + ): + await session.get("http://example.com", proxy=request_proxy) + + assert mocked_request.call_args.kwargs["proxy"] == request_proxy + + @pytest.mark.asyncio + async def test_proxy_rotates_per_retry_attempt(self): + """With a rotator, every retry attempt must pull a fresh proxy""" + rotator = ProxyRotator(["http://p1:8080", "http://p2:8080"]) + + async with AsyncFetcherSession(proxy_rotator=rotator, retries=2, retry_delay=0) as session: + with ( + patch.object(session._async_curl_session, "request", new=AsyncMock()) as mocked_request, + patch("scrapling.engines.static.ResponseFactory.from_http_request", return_value=MagicMock()), + ): + mocked_request.side_effect = [CurlError("transient"), MagicMock()] + await session.get("http://example.com") + + proxies_used = [call.kwargs["proxy"] for call in mocked_request.call_args_list] + assert proxies_used == ["http://p1:8080", "http://p2:8080"] diff --git a/tests/fetchers/sync/test_requests_session.py b/tests/fetchers/sync/test_requests_session.py index 152fbc4..2620e37 100644 --- a/tests/fetchers/sync/test_requests_session.py +++ b/tests/fetchers/sync/test_requests_session.py @@ -1,7 +1,9 @@ import pytest - +from unittest.mock import patch, MagicMock +from curl_cffi.curl import CurlError from scrapling.engines.static import _SyncSessionLogic as FetcherSession, FetcherClient +from scrapling.engines.toolbelt import ProxyRotator class TestFetcherSession: @@ -9,11 +11,7 @@ class TestFetcherSession: def test_fetcher_session_creation(self): """Test FetcherSession creation""" - session = FetcherSession( - timeout=30, - retries=3, - stealthy_headers=True - ) + session = FetcherSession(timeout=30, retries=3, stealthy_headers=True) assert session._default_timeout == 30 assert session._default_retries == 3 @@ -43,3 +41,44 @@ class TestFetcherSession: # Should not have context manager methods assert client.__enter__ is None assert client.__exit__ is None + + def test_session_level_proxy_is_applied(self): + """Session-level proxy must reach the request, not be silently dropped (#295)""" + proxy = "http://10.255.255.1:9999" + + with FetcherSession(proxy=proxy) as session: + with ( + patch.object(session._curl_session, "request") as mocked_request, + patch("scrapling.engines.static.ResponseFactory.from_http_request", return_value=MagicMock()), + ): + session.get("http://example.com") + + assert mocked_request.call_args.kwargs["proxy"] == proxy + + def test_per_request_proxy_overrides_session_proxy(self): + """A per-request proxy must take precedence over the session-level proxy""" + request_proxy = "http://10.255.255.2:9999" + + with FetcherSession(proxy="http://10.255.255.1:9999") as session: + with ( + patch.object(session._curl_session, "request") as mocked_request, + patch("scrapling.engines.static.ResponseFactory.from_http_request", return_value=MagicMock()), + ): + session.get("http://example.com", proxy=request_proxy) + + assert mocked_request.call_args.kwargs["proxy"] == request_proxy + + def test_proxy_rotates_per_retry_attempt(self): + """With a rotator, every retry attempt must pull a fresh proxy""" + rotator = ProxyRotator(["http://p1:8080", "http://p2:8080"]) + + with FetcherSession(proxy_rotator=rotator, retries=2, retry_delay=0) as session: + with ( + patch.object(session._curl_session, "request") as mocked_request, + patch("scrapling.engines.static.ResponseFactory.from_http_request", return_value=MagicMock()), + ): + mocked_request.side_effect = [CurlError("transient"), MagicMock()] + session.get("http://example.com") + + proxies_used = [call.kwargs["proxy"] for call in mocked_request.call_args_list] + assert proxies_used == ["http://p1:8080", "http://p2:8080"]