diff --git a/scrapling/spiders/request.py b/scrapling/spiders/request.py index faca233..ef766b2 100644 --- a/scrapling/spiders/request.py +++ b/scrapling/spiders/request.py @@ -22,6 +22,13 @@ def _convert_to_bytes(value: str | bytes) -> bytes: return value.encode(encoding="utf-8", errors="ignore") +def _stable_value_repr(value: Any) -> str: + try: + return orjson.dumps(value, option=orjson.OPT_SORT_KEYS, default=repr).decode() + except TypeError: + return repr(value) + + class Request: def __init__( self, @@ -98,7 +105,7 @@ class Request: if include_kwargs: filtered_kwargs = { - key.lower(): str(value) + key.lower(): _stable_value_repr(value) for key, value in self._session_kwargs.items() if key.lower() not in ("data", "json") } diff --git a/tests/spiders/test_request.py b/tests/spiders/test_request.py index f54f2cd..00f6c49 100644 --- a/tests/spiders/test_request.py +++ b/tests/spiders/test_request.py @@ -106,6 +106,24 @@ class TestRequestProperties: assert r1.update_fingerprint(include_kwargs=True) != r2.update_fingerprint(include_kwargs=True) + def test_fingerprint_include_kwargs_handles_non_primitive_values(self): + class _Opaque: + def __repr__(self) -> str: + return "_Opaque(stable)" + + opaque = _Opaque() + r1 = Request("https://example.com", proxies={"http": "p1"}, custom=opaque) + r2 = Request("https://example.com", proxies={"http": "p1"}, custom=opaque) + r3 = Request("https://example.com", proxies={"http": "p2"}, custom=opaque) + + fp1 = r1.update_fingerprint(include_kwargs=True) + r2._fp = None + fp2 = r2.update_fingerprint(include_kwargs=True) + fp3 = r3.update_fingerprint(include_kwargs=True) + + assert fp1 == fp2 + assert fp1 != fp3 + def test_fingerprint_include_headers_preserves_header_value_case(self): """Test header values are fingerprinted without lowercasing.""" r1 = Request("https://example.com", headers={"X-Test": "A"})