freedak/Scrapling
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: update code and docstrings to remove the old google referer logic

Browse Source
This commit is contained in:
Karim shoair
2026-03-09 00:25:19 +02:00
parent b547e04207
commit a929de6ca4
7 changed files with 40 additions and 79 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scrapling/engines/_browsers/_controllers.py
+10 -15
View File
@@ -14,7 +14,6 @@ from scrapling.core.utils import log
from scrapling.core._types import Optional, ProxyType, Unpack
from scrapling.engines.toolbelt.proxy_rotation import is_proxy_error
from scrapling.engines.toolbelt.convertor import Response, ResponseFactory
from scrapling.engines.toolbelt.fingerprints import generate_convincing_referer
from scrapling.engines._browsers._types import PlaywrightSession, PlaywrightFetchParams
from scrapling.engines._browsers._base import SyncSession, AsyncSession, DynamicSessionMixin
from scrapling.engines._browsers._validators import validate_fetch as _validate, PlaywrightConfig
@@ -58,8 +57,8 @@ class DynamicSession(SyncSession, DynamicSessionMixin):
:param real_chrome: If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch an instance of your browser and use it.
:param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute.
:param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers through CDP.
:param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name.
:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
:param google_search: Enabled by default, Scrapling will set a Google referer header.
:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by `google_search` takes priority over the referer set here if used together._
:param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.
:param user_data_dir: Path to a User Data Directory, which stores browser session data like cookies and local storage. The default is to create a temporary directory.
:param extra_flags: A list of additional browser flags to pass to the browser on launch.
@@ -103,11 +102,11 @@ class DynamicSession(SyncSession, DynamicSessionMixin):
"""Opens up the browser and do your request based on your chosen options.
:param url: The Target url.
:param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name.
:param google_search: Enabled by default, Scrapling will set a Google referer header.
:param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
:param page_action: Added for automation. A function that takes the `page` object and does the automation you need.
:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by `google_search` takes priority over the referer set here if used together._
:param disable_resources: Drop requests for unnecessary resources for a speed boost.
Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
:param blocked_domains: A set of domain names to block requests to. Subdomains are also matched (e.g., ``"example.com"`` blocks ``"sub.example.com"`` too).
@@ -127,9 +126,7 @@ class DynamicSession(SyncSession, DynamicSessionMixin):
request_headers_keys = {h.lower() for h in params.extra_headers.keys()} if params.extra_headers else set()
referer = (
generate_convincing_referer(url)
if (params.google_search and "referer" not in request_headers_keys)
else None
"https://www.google.com/" if (params.google_search and "referer" not in request_headers_keys) else None
)
for attempt in range(self._config.retries):
@@ -226,8 +223,8 @@ class AsyncDynamicSession(AsyncSession, DynamicSessionMixin):
:param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.
:param real_chrome: If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch an instance of your browser and use it.
:param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers through CDP.
:param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name.
:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
:param google_search: Enabled by default, Scrapling will set a Google referer header.
:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by `google_search` takes priority over the referer set here if used together._
:param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.
:param max_pages: The maximum number of tabs to be opened at the same time. It will be used in rotation through a PagePool.
:param user_data_dir: Path to a User Data Directory, which stores browser session data like cookies and local storage. The default is to create a temporary directory.
@@ -271,11 +268,11 @@ class AsyncDynamicSession(AsyncSession, DynamicSessionMixin):
"""Opens up the browser and do your request based on your chosen options.
:param url: The Target url.
:param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name.
:param google_search: Enabled by default, Scrapling will set a Google referer header.
:param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
:param page_action: Added for automation. A function that takes the `page` object and does the automation you need.
:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by `google_search` takes priority over the referer set here if used together._
:param disable_resources: Drop requests for unnecessary resources for a speed boost.
Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
:param blocked_domains: A set of domain names to block requests to. Subdomains are also matched (e.g., ``"example.com"`` blocks ``"sub.example.com"`` too).
@@ -296,9 +293,7 @@ class AsyncDynamicSession(AsyncSession, DynamicSessionMixin):
request_headers_keys = {h.lower() for h in params.extra_headers.keys()} if params.extra_headers else set()
referer = (
generate_convincing_referer(url)
if (params.google_search and "referer" not in request_headers_keys)
else None
"https://www.google.com/" if (params.google_search and "referer" not in request_headers_keys) else None
)
for attempt in range(self._config.retries):
scrapling/engines/_browsers/_stealth.py
+10 -15
View File
@@ -16,7 +16,6 @@ from scrapling.core.utils import log
from scrapling.core._types import Any, Optional, ProxyType, Unpack
from scrapling.engines.toolbelt.proxy_rotation import is_proxy_error
from scrapling.engines.toolbelt.convertor import Response, ResponseFactory
from scrapling.engines.toolbelt.fingerprints import generate_convincing_referer
from scrapling.engines._browsers._types import StealthSession, StealthFetchParams
from scrapling.engines._browsers._base import SyncSession, AsyncSession, StealthySessionMixin
from scrapling.engines._browsers._validators import validate_fetch as _validate, StealthConfig
@@ -66,8 +65,8 @@ class StealthySession(SyncSession, StealthySessionMixin):
:param allow_webgl: Enabled by default. Disabling it disables WebGL and WebGL 2.0 support entirely. Disabling WebGL is not recommended as many WAFs now check if WebGL is enabled.
:param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute.
:param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers through CDP.
:param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name.
:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
:param google_search: Enabled by default, Scrapling will set a Google referer header.
:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by `google_search` takes priority over the referer set here if used together._
:param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.
:param user_data_dir: Path to a User Data Directory, which stores browser session data like cookies and local storage. The default is to create a temporary directory.
:param extra_flags: A list of additional browser flags to pass to the browser on launch.
@@ -189,11 +188,11 @@ class StealthySession(SyncSession, StealthySessionMixin):
"""Opens up the browser and do your request based on your chosen options.
:param url: The Target url.
:param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name.
:param google_search: Enabled by default, Scrapling will set a Google referer header.
:param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
:param page_action: Added for automation. A function that takes the `page` object and does the automation you need.
:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by `google_search` takes priority over the referer set here if used together._
:param disable_resources: Drop requests for unnecessary resources for a speed boost.
Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
:param blocked_domains: A set of domain names to block requests to. Subdomains are also matched (e.g., ``"example.com"`` blocks ``"sub.example.com"`` too).
@@ -214,9 +213,7 @@ class StealthySession(SyncSession, StealthySessionMixin):
request_headers_keys = {h.lower() for h in params.extra_headers.keys()} if params.extra_headers else set()
referer = (
generate_convincing_referer(url)
if (params.google_search and "referer" not in request_headers_keys)
else None
"https://www.google.com/" if (params.google_search and "referer" not in request_headers_keys) else None
)
for attempt in range(self._config.retries):
@@ -322,8 +319,8 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin):
:param allow_webgl: Enabled by default. Disabling it disables WebGL and WebGL 2.0 support entirely. Disabling WebGL is not recommended as many WAFs now check if WebGL is enabled.
:param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute.
:param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers through CDP.
:param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name.
:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
:param google_search: Enabled by default, Scrapling will set a Google referer header.
:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by `google_search` takes priority over the referer set here if used together._
:param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.
:param user_data_dir: Path to a User Data Directory, which stores browser session data like cookies and local storage. The default is to create a temporary directory.
:param extra_flags: A list of additional browser flags to pass to the browser on launch.
@@ -444,11 +441,11 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin):
"""Opens up the browser and do your request based on your chosen options.
:param url: The Target url.
:param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name.
:param google_search: Enabled by default, Scrapling will set a Google referer header.
:param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
:param page_action: Added for automation. A function that takes the `page` object and does the automation you need.
:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by `google_search` takes priority over the referer set here if used together._
:param disable_resources: Drop requests for unnecessary resources for a speed boost.
Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
:param blocked_domains: A set of domain names to block requests to. Subdomains are also matched (e.g., ``"example.com"`` blocks ``"sub.example.com"`` too).
@@ -470,9 +467,7 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin):
request_headers_keys = {h.lower() for h in params.extra_headers.keys()} if params.extra_headers else set()
referer = (
generate_convincing_referer(url)
if (params.google_search and "referer" not in request_headers_keys)
else None
"https://www.google.com/" if (params.google_search and "referer" not in request_headers_keys) else None
)
for attempt in range(self._config.retries):
scrapling/engines/static.py
+4 -4
View File
@@ -26,7 +26,7 @@ from .toolbelt.custom import Response
from .toolbelt.convertor import ResponseFactory
from .toolbelt.proxy_rotation import ProxyRotator, is_proxy_error
from ._browsers._types import RequestsSession, GetRequestParams, DataRequestParams, ImpersonateType
from .toolbelt.fingerprints import generate_convincing_referer, generate_headers, __default_useragent__
from .toolbelt.fingerprints import generate_headers, __default_useragent__
_NO_SESSION: Any = object()
@@ -166,14 +166,14 @@ class _ConfigurationLogic(ABC):
"""
1. Adds a useragent to the headers if it doesn't have one
2. Generates real headers and append them to current headers
3. Generates a referer header that looks like as if this request came from a Google's search of the current URL's domain.
3. Sets a Google referer header.
"""
# Merge session headers with request headers, request takes precedence (if it was set)
final_headers = {**self._default_headers, **(headers if headers else {})}
headers_keys = {k.lower() for k in final_headers}
if stealth:
if "referer" not in headers_keys:
final_headers["referer"] = generate_convincing_referer(url)
final_headers["referer"] = "https://www.google.com/"
if not impersonate_enabled: # Curl will generate the suitable headers
extra_headers = generate_headers(browser_mode=False)
@@ -672,7 +672,7 @@ class FetcherSession:
"""
:param impersonate: Browser version to impersonate. Can be a single browser string or a list of browser strings for random selection. (Default: latest available Chrome version)
:param http3: Whether to use HTTP3. Defaults to False. It might be problematic if used it with `impersonate`.
:param stealthy_headers: If enabled (default), it creates and adds real browser headers. It also sets the referer header as if this request came from a Google search of URL's domain.
:param stealthy_headers: If enabled (default), it creates and adds real browser headers. It also sets a Google referer header.
:param proxies: Dict of proxies to use. Format: {"http": proxy_url, "https": proxy_url}.
:param proxy: Proxy URL to use. Format: "http://username:password@localhost:8030".
Cannot be used together with the `proxies` parameter.
scrapling/engines/toolbelt/fingerprints.py
-29
View File
@@ -5,7 +5,6 @@ Functions related to generating headers and fingerprints generally
from functools import lru_cache
from platform import system as platform_system
from tld import get_tld, Result
from browserforge.headers import Browser, HeaderGenerator
from browserforge.headers.generator import SUPPORTED_OPERATING_SYSTEMS
@@ -18,34 +17,6 @@ chromium_version = 145
chrome_version = 145
@lru_cache(10, typed=True)
def generate_convincing_referer(url: str) -> str | None:
"""Generate a convincing Google referer header
>>> generate_convincing_referer('https://www.somewebsite.com/blah')
'https://www.google.com/'
:param url: The URL you are about to fetch.
:return: Google's URL as referer, or None for localhost/IP addresses
"""
# Fixing the inaccurate return type hint in `get_tld`
extracted: Result | None = cast(Result, get_tld(url, as_object=True, fail_silently=True))
if not extracted:
return None
website_name = extracted.domain
# Skip generating referer for localhost, IP addresses, or when there's no valid domain
if not website_name or not extracted.tld or website_name in ("localhost", "127.0.0.1", "::1"):
return None
# Check if it's an IP address (simple check for IPv4)
if all(part.isdigit() for part in website_name.split(".") if part):
return None
return "https://www.google.com/"
@lru_cache(1, typed=True)
def get_os_name() -> OSName | Tuple:
"""Get the current OS name in the same format needed for browserforge, if the OS is Unknown, return None so browserforge uses all.