Karim shoair
90c52c45c7
This might break the adaptive data users have for websites BUT: 1. tld uses ~3.7x less memory during extraction operations (1.5 MB vs 5.7 MB). 2. tld uses ~56% less memory on import (5.2 MB vs 11.9 MB). 3. Zero dependencies (vs 3 for tldextract). In return, it's 30ms slower for extracting 5000 URLs, which is negligible. Also, the type hints aren't always accurate, but it's fine; I corrected them.
89 lines
3.3 KiB
Python
89 lines
3.3 KiB
Python
"""
|
|
Functions related to generating headers and fingerprints generally
|
|
"""
|
|
|
|
from functools import lru_cache
|
|
from platform import system as platform_system
|
|
|
|
from tld import get_tld, Result
|
|
from browserforge.headers import Browser, HeaderGenerator
|
|
from browserforge.headers.generator import SUPPORTED_OPERATING_SYSTEMS
|
|
|
|
from scrapling.core._types import Dict, Literal, Tuple, cast
|
|
|
|
__OS_NAME__ = platform_system()
|
|
OSName = Literal["linux", "macos", "windows"]
|
|
# Current versions hardcoded for now (Playwright doesn't allow to know the version of a browser without launching it)
|
|
chromium_version = 141
|
|
chrome_version = 143
|
|
|
|
|
|
@lru_cache(10, typed=True)
|
|
def generate_convincing_referer(url: str) -> str | None:
|
|
"""Takes the domain from the URL without the subdomain/suffix and make it look like you were searching Google for this website
|
|
|
|
>>> generate_convincing_referer('https://www.somewebsite.com/blah')
|
|
'https://www.google.com/search?q=somewebsite'
|
|
|
|
:param url: The URL you are about to fetch.
|
|
:return: Google's search URL of the domain name, or None for localhost/IP addresses
|
|
"""
|
|
# Fixing the inaccurate return type hint in `get_tld`
|
|
extracted: Result | None = cast(Result, get_tld(url, as_object=True, fail_silently=True))
|
|
if not extracted:
|
|
return None
|
|
|
|
website_name = extracted.domain
|
|
|
|
# Skip generating referer for localhost, IP addresses, or when there's no valid domain
|
|
if not website_name or not extracted.tld or website_name in ("localhost", "127.0.0.1", "::1"):
|
|
return None
|
|
|
|
# Check if it's an IP address (simple check for IPv4)
|
|
if all(part.isdigit() for part in website_name.split(".") if part):
|
|
return None
|
|
|
|
return f"https://www.google.com/search?q={website_name}"
|
|
|
|
|
|
@lru_cache(1, typed=True)
|
|
def get_os_name() -> OSName | Tuple:
|
|
"""Get the current OS name in the same format needed for browserforge, if the OS is Unknown, return None so browserforge uses all.
|
|
|
|
:return: Current OS name or `None` otherwise
|
|
"""
|
|
match __OS_NAME__: # pragma: no cover
|
|
case "Linux":
|
|
return "linux"
|
|
case "Darwin":
|
|
return "macos"
|
|
case "Windows":
|
|
return "windows"
|
|
case _:
|
|
return SUPPORTED_OPERATING_SYSTEMS
|
|
|
|
|
|
def generate_headers(browser_mode: bool | str = False) -> Dict:
|
|
"""Generate real browser-like headers using browserforge's generator
|
|
|
|
:param browser_mode: If enabled, the headers created are used for playwright, so it has to match everything
|
|
:return: A dictionary of the generated headers
|
|
"""
|
|
# In the browser mode, we don't care about anything other than matching the OS and the browser type with the browser we are using,
|
|
# So we don't raise any inconsistency red flags while websites fingerprinting us
|
|
os_name = get_os_name()
|
|
ver = chrome_version if browser_mode and browser_mode == "chrome" else chromium_version
|
|
browsers = [Browser(name="chrome", min_version=ver, max_version=ver)]
|
|
if not browser_mode:
|
|
os_name = ("windows", "macos", "linux")
|
|
browsers.extend(
|
|
[
|
|
Browser(name="firefox", min_version=142),
|
|
Browser(name="edge", min_version=140),
|
|
]
|
|
)
|
|
return HeaderGenerator(browser=browsers, os=os_name, device="desktop").generate()
|
|
|
|
|
|
__default_useragent__ = generate_headers(browser_mode=False).get("User-Agent")
|