""" Functions related to generating headers and fingerprints generally """ from functools import lru_cache from platform import system as platform_system from tld import get_tld, Result from browserforge.headers import Browser, HeaderGenerator from browserforge.headers.generator import SUPPORTED_OPERATING_SYSTEMS from scrapling.core._types import Dict, Literal, Tuple, cast __OS_NAME__ = platform_system() OSName = Literal["linux", "macos", "windows"] # Current versions hardcoded for now (Playwright doesn't allow to know the version of a browser without launching it) chromium_version = 141 chrome_version = 143 @lru_cache(10, typed=True) def generate_convincing_referer(url: str) -> str | None: """Generate a convincing Google referer header >>> generate_convincing_referer('https://www.somewebsite.com/blah') 'https://www.google.com/' :param url: The URL you are about to fetch. :return: Google's URL as referer, or None for localhost/IP addresses """ # Fixing the inaccurate return type hint in `get_tld` extracted: Result | None = cast(Result, get_tld(url, as_object=True, fail_silently=True)) if not extracted: return None website_name = extracted.domain # Skip generating referer for localhost, IP addresses, or when there's no valid domain if not website_name or not extracted.tld or website_name in ("localhost", "127.0.0.1", "::1"): return None # Check if it's an IP address (simple check for IPv4) if all(part.isdigit() for part in website_name.split(".") if part): return None return "https://www.google.com/" @lru_cache(1, typed=True) def get_os_name() -> OSName | Tuple: """Get the current OS name in the same format needed for browserforge, if the OS is Unknown, return None so browserforge uses all. :return: Current OS name or `None` otherwise """ match __OS_NAME__: # pragma: no cover case "Linux": return "linux" case "Darwin": return "macos" case "Windows": return "windows" case _: return SUPPORTED_OPERATING_SYSTEMS def generate_headers(browser_mode: bool | str = False) -> Dict: """Generate real browser-like headers using browserforge's generator :param browser_mode: If enabled, the headers created are used for playwright, so it has to match everything :return: A dictionary of the generated headers """ # In the browser mode, we don't care about anything other than matching the OS and the browser type with the browser we are using, # So we don't raise any inconsistency red flags while websites fingerprinting us os_name = get_os_name() ver = chrome_version if browser_mode and browser_mode == "chrome" else chromium_version browsers = [Browser(name="chrome", min_version=ver, max_version=ver)] if not browser_mode: os_name = ("windows", "macos", "linux") browsers.extend( [ Browser(name="firefox", min_version=142), Browser(name="edge", min_version=140), ] ) return HeaderGenerator(browser=browsers, os=os_name, device="desktop").generate() __default_useragent__ = generate_headers(browser_mode=False).get("User-Agent")