From 0c7e26ae5c9864d737d85320ec5c6febd29945cf Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Tue, 30 Dec 2025 21:58:09 +0200 Subject: [PATCH] fix(fingerprints): do not create google referer for IPs --- scrapling/engines/toolbelt/fingerprints.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/scrapling/engines/toolbelt/fingerprints.py b/scrapling/engines/toolbelt/fingerprints.py index 4e9263e..2ddbbe5 100644 --- a/scrapling/engines/toolbelt/fingerprints.py +++ b/scrapling/engines/toolbelt/fingerprints.py @@ -19,16 +19,26 @@ chrome_version = 143 @lru_cache(10, typed=True) -def generate_convincing_referer(url: str) -> str: +def generate_convincing_referer(url: str) -> str | None: """Takes the domain from the URL without the subdomain/suffix and make it look like you were searching Google for this website >>> generate_convincing_referer('https://www.somewebsite.com/blah') 'https://www.google.com/search?q=somewebsite' :param url: The URL you are about to fetch. - :return: Google's search URL of the domain name + :return: Google's search URL of the domain name, or None for localhost/IP addresses """ - website_name = extract(url).domain + extracted = extract(url) + website_name = extracted.domain + + # Skip generating referer for localhost, IP addresses, or when there's no valid domain + if not website_name or not extracted.suffix or website_name in ("localhost", "127.0.0.1", "::1"): + return None + + # Check if it's an IP address (simple check for IPv4) + if all(part.isdigit() for part in website_name.split(".") if part): + return None + return f"https://www.google.com/search?q={website_name}" @@ -64,7 +74,7 @@ def generate_headers(browser_mode: bool | str = False) -> Dict: os_name = ("windows", "macos", "linux") browsers.extend( [ - Browser(name="firefox", min_version=140), + Browser(name="firefox", min_version=142), Browser(name="edge", min_version=140), ] )