From 68b9e128bca0a7ab532de791faf3b8b40b26cc9a Mon Sep 17 00:00:00 2001 From: Andrew Barnes Date: Sun, 8 Mar 2026 11:32:04 -0400 Subject: [PATCH] fix: return bare Google URL in referer instead of search query Real browsers send `https://www.google.com/` as the Referer header when clicking search results, not the full search URL with query parameters. The previous format was a fingerprinting signal that the referer was spoofed. Closes #172 --- scrapling/engines/toolbelt/fingerprints.py | 8 ++++---- tests/fetchers/test_utils.py | 3 +-- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/scrapling/engines/toolbelt/fingerprints.py b/scrapling/engines/toolbelt/fingerprints.py index 6774691..d84a682 100644 --- a/scrapling/engines/toolbelt/fingerprints.py +++ b/scrapling/engines/toolbelt/fingerprints.py @@ -20,13 +20,13 @@ chrome_version = 143 @lru_cache(10, typed=True) def generate_convincing_referer(url: str) -> str | None: - """Takes the domain from the URL without the subdomain/suffix and make it look like you were searching Google for this website + """Generate a convincing Google referer header >>> generate_convincing_referer('https://www.somewebsite.com/blah') - 'https://www.google.com/search?q=somewebsite' + 'https://www.google.com/' :param url: The URL you are about to fetch. - :return: Google's search URL of the domain name, or None for localhost/IP addresses + :return: Google's URL as referer, or None for localhost/IP addresses """ # Fixing the inaccurate return type hint in `get_tld` extracted: Result | None = cast(Result, get_tld(url, as_object=True, fail_silently=True)) @@ -43,7 +43,7 @@ def generate_convincing_referer(url: str) -> str | None: if all(part.isdigit() for part in website_name.split(".") if part): return None - return f"https://www.google.com/search?q={website_name}" + return "https://www.google.com/" @lru_cache(1, typed=True) diff --git a/tests/fetchers/test_utils.py b/tests/fetchers/test_utils.py index 0a4be9f..526f898 100644 --- a/tests/fetchers/test_utils.py +++ b/tests/fetchers/test_utils.py @@ -209,8 +209,7 @@ class TestFingerprintFunctions: url = "https://sub.example.com/page.html" result = generate_convincing_referer(url) - assert result.startswith("https://www.google.com/search?q=") - assert "example" in result + assert result == "https://www.google.com/" def test_generate_convincing_referer_caching(self): """Test referer generation caching"""