fix: apply the session-level proxy when no per-request proxy is given
The per-request proxy resolution never fell back to the session default, so FetcherSession(proxy=...) was silently ignored, and requests went direct. Same fix in the sync and async paths, with regression tests asserting on the proxy that reaches curl_cffi. Closes #295
This commit is contained in:
@@ -19,6 +19,7 @@ from scrapling.core._types import (
|
|||||||
Unpack,
|
Unpack,
|
||||||
Optional,
|
Optional,
|
||||||
Awaitable,
|
Awaitable,
|
||||||
|
ProxyType,
|
||||||
SUPPORTED_HTTP_METHODS,
|
SUPPORTED_HTTP_METHODS,
|
||||||
FollowRedirects,
|
FollowRedirects,
|
||||||
)
|
)
|
||||||
@@ -244,10 +245,11 @@ class _SyncSessionLogic(_ConfigurationLogic):
|
|||||||
|
|
||||||
try:
|
try:
|
||||||
for attempt in range(max_retries):
|
for attempt in range(max_retries):
|
||||||
|
proxy: Optional[ProxyType]
|
||||||
if self._proxy_rotator and static_proxy is None:
|
if self._proxy_rotator and static_proxy is None:
|
||||||
proxy = self._proxy_rotator.get_proxy()
|
proxy = self._proxy_rotator.get_proxy()
|
||||||
else:
|
else:
|
||||||
proxy = static_proxy
|
proxy = static_proxy or self._default_proxy
|
||||||
|
|
||||||
request_args = self._merge_request_args(stealth=stealth, proxy=proxy, **kwargs)
|
request_args = self._merge_request_args(stealth=stealth, proxy=proxy, **kwargs)
|
||||||
try:
|
try:
|
||||||
@@ -461,10 +463,11 @@ class _ASyncSessionLogic(_ConfigurationLogic):
|
|||||||
try:
|
try:
|
||||||
# Determine if we should use proxy rotation
|
# Determine if we should use proxy rotation
|
||||||
for attempt in range(max_retries):
|
for attempt in range(max_retries):
|
||||||
|
proxy: Optional[ProxyType]
|
||||||
if self._proxy_rotator and static_proxy is None:
|
if self._proxy_rotator and static_proxy is None:
|
||||||
proxy = self._proxy_rotator.get_proxy()
|
proxy = self._proxy_rotator.get_proxy()
|
||||||
else:
|
else:
|
||||||
proxy = static_proxy
|
proxy = static_proxy or self._default_proxy
|
||||||
|
|
||||||
request_args = self._merge_request_args(stealth=stealth, proxy=proxy, **kwargs)
|
request_args = self._merge_request_args(stealth=stealth, proxy=proxy, **kwargs)
|
||||||
try:
|
try:
|
||||||
|
|||||||
@@ -1,6 +1,9 @@
|
|||||||
|
import pytest
|
||||||
|
from unittest.mock import patch, MagicMock, AsyncMock
|
||||||
|
from curl_cffi.curl import CurlError
|
||||||
|
|
||||||
|
from scrapling.engines.static import _ASyncSessionLogic as AsyncFetcherSession, AsyncFetcherClient
|
||||||
from scrapling.engines.static import AsyncFetcherClient
|
from scrapling.engines.toolbelt import ProxyRotator
|
||||||
|
|
||||||
|
|
||||||
class TestFetcherSession:
|
class TestFetcherSession:
|
||||||
@@ -13,3 +16,47 @@ class TestFetcherSession:
|
|||||||
# Should not have context manager methods
|
# Should not have context manager methods
|
||||||
assert client.__aenter__ is None
|
assert client.__aenter__ is None
|
||||||
assert client.__aexit__ is None
|
assert client.__aexit__ is None
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_session_level_proxy_is_applied(self):
|
||||||
|
"""Session-level proxy must reach the request, not be silently dropped (#295)"""
|
||||||
|
proxy = "http://10.255.255.1:9999"
|
||||||
|
|
||||||
|
async with AsyncFetcherSession(proxy=proxy) as session:
|
||||||
|
with (
|
||||||
|
patch.object(session._async_curl_session, "request", new=AsyncMock()) as mocked_request,
|
||||||
|
patch("scrapling.engines.static.ResponseFactory.from_http_request", return_value=MagicMock()),
|
||||||
|
):
|
||||||
|
await session.get("http://example.com")
|
||||||
|
|
||||||
|
assert mocked_request.call_args.kwargs["proxy"] == proxy
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_per_request_proxy_overrides_session_proxy(self):
|
||||||
|
"""A per-request proxy must take precedence over the session-level proxy"""
|
||||||
|
request_proxy = "http://10.255.255.2:9999"
|
||||||
|
|
||||||
|
async with AsyncFetcherSession(proxy="http://10.255.255.1:9999") as session:
|
||||||
|
with (
|
||||||
|
patch.object(session._async_curl_session, "request", new=AsyncMock()) as mocked_request,
|
||||||
|
patch("scrapling.engines.static.ResponseFactory.from_http_request", return_value=MagicMock()),
|
||||||
|
):
|
||||||
|
await session.get("http://example.com", proxy=request_proxy)
|
||||||
|
|
||||||
|
assert mocked_request.call_args.kwargs["proxy"] == request_proxy
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_proxy_rotates_per_retry_attempt(self):
|
||||||
|
"""With a rotator, every retry attempt must pull a fresh proxy"""
|
||||||
|
rotator = ProxyRotator(["http://p1:8080", "http://p2:8080"])
|
||||||
|
|
||||||
|
async with AsyncFetcherSession(proxy_rotator=rotator, retries=2, retry_delay=0) as session:
|
||||||
|
with (
|
||||||
|
patch.object(session._async_curl_session, "request", new=AsyncMock()) as mocked_request,
|
||||||
|
patch("scrapling.engines.static.ResponseFactory.from_http_request", return_value=MagicMock()),
|
||||||
|
):
|
||||||
|
mocked_request.side_effect = [CurlError("transient"), MagicMock()]
|
||||||
|
await session.get("http://example.com")
|
||||||
|
|
||||||
|
proxies_used = [call.kwargs["proxy"] for call in mocked_request.call_args_list]
|
||||||
|
assert proxies_used == ["http://p1:8080", "http://p2:8080"]
|
||||||
|
|||||||
@@ -1,7 +1,9 @@
|
|||||||
import pytest
|
import pytest
|
||||||
|
from unittest.mock import patch, MagicMock
|
||||||
|
from curl_cffi.curl import CurlError
|
||||||
|
|
||||||
from scrapling.engines.static import _SyncSessionLogic as FetcherSession, FetcherClient
|
from scrapling.engines.static import _SyncSessionLogic as FetcherSession, FetcherClient
|
||||||
|
from scrapling.engines.toolbelt import ProxyRotator
|
||||||
|
|
||||||
|
|
||||||
class TestFetcherSession:
|
class TestFetcherSession:
|
||||||
@@ -9,11 +11,7 @@ class TestFetcherSession:
|
|||||||
|
|
||||||
def test_fetcher_session_creation(self):
|
def test_fetcher_session_creation(self):
|
||||||
"""Test FetcherSession creation"""
|
"""Test FetcherSession creation"""
|
||||||
session = FetcherSession(
|
session = FetcherSession(timeout=30, retries=3, stealthy_headers=True)
|
||||||
timeout=30,
|
|
||||||
retries=3,
|
|
||||||
stealthy_headers=True
|
|
||||||
)
|
|
||||||
|
|
||||||
assert session._default_timeout == 30
|
assert session._default_timeout == 30
|
||||||
assert session._default_retries == 3
|
assert session._default_retries == 3
|
||||||
@@ -43,3 +41,44 @@ class TestFetcherSession:
|
|||||||
# Should not have context manager methods
|
# Should not have context manager methods
|
||||||
assert client.__enter__ is None
|
assert client.__enter__ is None
|
||||||
assert client.__exit__ is None
|
assert client.__exit__ is None
|
||||||
|
|
||||||
|
def test_session_level_proxy_is_applied(self):
|
||||||
|
"""Session-level proxy must reach the request, not be silently dropped (#295)"""
|
||||||
|
proxy = "http://10.255.255.1:9999"
|
||||||
|
|
||||||
|
with FetcherSession(proxy=proxy) as session:
|
||||||
|
with (
|
||||||
|
patch.object(session._curl_session, "request") as mocked_request,
|
||||||
|
patch("scrapling.engines.static.ResponseFactory.from_http_request", return_value=MagicMock()),
|
||||||
|
):
|
||||||
|
session.get("http://example.com")
|
||||||
|
|
||||||
|
assert mocked_request.call_args.kwargs["proxy"] == proxy
|
||||||
|
|
||||||
|
def test_per_request_proxy_overrides_session_proxy(self):
|
||||||
|
"""A per-request proxy must take precedence over the session-level proxy"""
|
||||||
|
request_proxy = "http://10.255.255.2:9999"
|
||||||
|
|
||||||
|
with FetcherSession(proxy="http://10.255.255.1:9999") as session:
|
||||||
|
with (
|
||||||
|
patch.object(session._curl_session, "request") as mocked_request,
|
||||||
|
patch("scrapling.engines.static.ResponseFactory.from_http_request", return_value=MagicMock()),
|
||||||
|
):
|
||||||
|
session.get("http://example.com", proxy=request_proxy)
|
||||||
|
|
||||||
|
assert mocked_request.call_args.kwargs["proxy"] == request_proxy
|
||||||
|
|
||||||
|
def test_proxy_rotates_per_retry_attempt(self):
|
||||||
|
"""With a rotator, every retry attempt must pull a fresh proxy"""
|
||||||
|
rotator = ProxyRotator(["http://p1:8080", "http://p2:8080"])
|
||||||
|
|
||||||
|
with FetcherSession(proxy_rotator=rotator, retries=2, retry_delay=0) as session:
|
||||||
|
with (
|
||||||
|
patch.object(session._curl_session, "request") as mocked_request,
|
||||||
|
patch("scrapling.engines.static.ResponseFactory.from_http_request", return_value=MagicMock()),
|
||||||
|
):
|
||||||
|
mocked_request.side_effect = [CurlError("transient"), MagicMock()]
|
||||||
|
session.get("http://example.com")
|
||||||
|
|
||||||
|
proxies_used = [call.kwargs["proxy"] for call in mocked_request.call_args_list]
|
||||||
|
assert proxies_used == ["http://p1:8080", "http://p2:8080"]
|
||||||
|
|||||||
Reference in New Issue
Block a user