fix: update code and docstrings to remove the old google referer logic

This commit is contained in:
Karim shoair
2026-03-09 00:25:19 +02:00
parent b547e04207
commit a929de6ca4
7 changed files with 40 additions and 79 deletions
+4 -4
View File
@@ -26,7 +26,7 @@ from .toolbelt.custom import Response
from .toolbelt.convertor import ResponseFactory
from .toolbelt.proxy_rotation import ProxyRotator, is_proxy_error
from ._browsers._types import RequestsSession, GetRequestParams, DataRequestParams, ImpersonateType
from .toolbelt.fingerprints import generate_convincing_referer, generate_headers, __default_useragent__
from .toolbelt.fingerprints import generate_headers, __default_useragent__
_NO_SESSION: Any = object()
@@ -166,14 +166,14 @@ class _ConfigurationLogic(ABC):
"""
1. Adds a useragent to the headers if it doesn't have one
2. Generates real headers and append them to current headers
3. Generates a referer header that looks like as if this request came from a Google's search of the current URL's domain.
3. Sets a Google referer header.
"""
# Merge session headers with request headers, request takes precedence (if it was set)
final_headers = {**self._default_headers, **(headers if headers else {})}
headers_keys = {k.lower() for k in final_headers}
if stealth:
if "referer" not in headers_keys:
final_headers["referer"] = generate_convincing_referer(url)
final_headers["referer"] = "https://www.google.com/"
if not impersonate_enabled: # Curl will generate the suitable headers
extra_headers = generate_headers(browser_mode=False)
@@ -672,7 +672,7 @@ class FetcherSession:
"""
:param impersonate: Browser version to impersonate. Can be a single browser string or a list of browser strings for random selection. (Default: latest available Chrome version)
:param http3: Whether to use HTTP3. Defaults to False. It might be problematic if used it with `impersonate`.
:param stealthy_headers: If enabled (default), it creates and adds real browser headers. It also sets the referer header as if this request came from a Google search of URL's domain.
:param stealthy_headers: If enabled (default), it creates and adds real browser headers. It also sets a Google referer header.
:param proxies: Dict of proxies to use. Format: {"http": proxy_url, "https": proxy_url}.
:param proxy: Proxy URL to use. Format: "http://username:password@localhost:8030".
Cannot be used together with the `proxies` parameter.