From a929de6ca4f6f84f09b42226ee5670643cffccc8 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Mon, 9 Mar 2026 00:25:19 +0200 Subject: [PATCH] fix: update code and docstrings to remove the old google referer logic --- scrapling/core/ai.py | 20 +++++++------- scrapling/engines/_browsers/_controllers.py | 25 +++++++----------- scrapling/engines/_browsers/_stealth.py | 25 +++++++----------- scrapling/engines/static.py | 8 +++--- scrapling/engines/toolbelt/fingerprints.py | 29 --------------------- scrapling/fetchers/chrome.py | 4 +-- scrapling/fetchers/stealth_chrome.py | 8 +++--- 7 files changed, 40 insertions(+), 79 deletions(-) diff --git a/scrapling/core/ai.py b/scrapling/core/ai.py index f637437..3481970 100644 --- a/scrapling/core/ai.py +++ b/scrapling/core/ai.py @@ -105,7 +105,7 @@ class ScraplingMCPServer: :param auth: HTTP basic auth in dictionary format with `username` and `password` keys. :param verify: Whether to verify HTTPS certificates. :param http3: Whether to use HTTP3. Defaults to False. It might be problematic if used it with `impersonate`. - :param stealthy_headers: If enabled (default), it creates and adds real browser headers. It also sets the referer header as if this request came from a Google search of URL's domain. + :param stealthy_headers: If enabled (default), it creates and adds real browser headers. It also sets a Google referer header. """ normalized_proxy_auth = _normalize_credentials(proxy_auth) normalized_auth = _normalize_credentials(auth) @@ -186,7 +186,7 @@ class ScraplingMCPServer: :param auth: HTTP basic auth in dictionary format with `username` and `password` keys. :param verify: Whether to verify HTTPS certificates. :param http3: Whether to use HTTP3. Defaults to False. It might be problematic if used it with `impersonate`. - :param stealthy_headers: If enabled (default), it creates and adds real browser headers. It also sets the referer header as if this request came from a Google search of URL's domain. + :param stealthy_headers: If enabled (default), it creates and adds real browser headers. It also sets a Google referer header. """ normalized_proxy_auth = _normalize_credentials(proxy_auth) normalized_auth = _normalize_credentials(auth) @@ -276,8 +276,8 @@ class ScraplingMCPServer: :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`. :param real_chrome: If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch an instance of your browser and use it. :param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers through CDP. - :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name. - :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ + :param google_search: Enabled by default, Scrapling will set a Google referer header. + :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by `google_search` takes priority over the referer set here if used together._ :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only. """ page = await DynamicFetcher.async_fetch( @@ -358,8 +358,8 @@ class ScraplingMCPServer: :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`. :param real_chrome: If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch an instance of your browser and use it. :param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers through CDP. - :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name. - :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ + :param google_search: Enabled by default, Scrapling will set a Google referer header. + :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by `google_search` takes priority over the referer set here if used together._ :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only. """ async with AsyncDynamicSession( @@ -454,8 +454,8 @@ class ScraplingMCPServer: :param hide_canvas: Add random noise to canvas operations to prevent fingerprinting. :param block_webrtc: Forces WebRTC to respect proxy settings to prevent local IP address leak. :param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers through CDP. - :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name. - :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ + :param google_search: Enabled by default, Scrapling will set a Google referer header. + :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by `google_search` takes priority over the referer set here if used together._ :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only. :param additional_args: Additional arguments to be passed to Playwright's context as additional settings, and it takes higher priority than Scrapling's settings. """ @@ -551,8 +551,8 @@ class ScraplingMCPServer: :param hide_canvas: Add random noise to canvas operations to prevent fingerprinting. :param block_webrtc: Forces WebRTC to respect proxy settings to prevent local IP address leak. :param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers through CDP. - :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name. - :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ + :param google_search: Enabled by default, Scrapling will set a Google referer header. + :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by `google_search` takes priority over the referer set here if used together._ :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only. :param additional_args: Additional arguments to be passed to Playwright's context as additional settings, and it takes higher priority than Scrapling's settings. """ diff --git a/scrapling/engines/_browsers/_controllers.py b/scrapling/engines/_browsers/_controllers.py index 5d9e801..fa6e8b6 100644 --- a/scrapling/engines/_browsers/_controllers.py +++ b/scrapling/engines/_browsers/_controllers.py @@ -14,7 +14,6 @@ from scrapling.core.utils import log from scrapling.core._types import Optional, ProxyType, Unpack from scrapling.engines.toolbelt.proxy_rotation import is_proxy_error from scrapling.engines.toolbelt.convertor import Response, ResponseFactory -from scrapling.engines.toolbelt.fingerprints import generate_convincing_referer from scrapling.engines._browsers._types import PlaywrightSession, PlaywrightFetchParams from scrapling.engines._browsers._base import SyncSession, AsyncSession, DynamicSessionMixin from scrapling.engines._browsers._validators import validate_fetch as _validate, PlaywrightConfig @@ -58,8 +57,8 @@ class DynamicSession(SyncSession, DynamicSessionMixin): :param real_chrome: If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch an instance of your browser and use it. :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute. :param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers through CDP. - :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name. - :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ + :param google_search: Enabled by default, Scrapling will set a Google referer header. + :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by `google_search` takes priority over the referer set here if used together._ :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only. :param user_data_dir: Path to a User Data Directory, which stores browser session data like cookies and local storage. The default is to create a temporary directory. :param extra_flags: A list of additional browser flags to pass to the browser on launch. @@ -103,11 +102,11 @@ class DynamicSession(SyncSession, DynamicSessionMixin): """Opens up the browser and do your request based on your chosen options. :param url: The Target url. - :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name. + :param google_search: Enabled by default, Scrapling will set a Google referer header. :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000 :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object. :param page_action: Added for automation. A function that takes the `page` object and does the automation you need. - :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ + :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by `google_search` takes priority over the referer set here if used together._ :param disable_resources: Drop requests for unnecessary resources for a speed boost. Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`. :param blocked_domains: A set of domain names to block requests to. Subdomains are also matched (e.g., ``"example.com"`` blocks ``"sub.example.com"`` too). @@ -127,9 +126,7 @@ class DynamicSession(SyncSession, DynamicSessionMixin): request_headers_keys = {h.lower() for h in params.extra_headers.keys()} if params.extra_headers else set() referer = ( - generate_convincing_referer(url) - if (params.google_search and "referer" not in request_headers_keys) - else None + "https://www.google.com/" if (params.google_search and "referer" not in request_headers_keys) else None ) for attempt in range(self._config.retries): @@ -226,8 +223,8 @@ class AsyncDynamicSession(AsyncSession, DynamicSessionMixin): :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`. :param real_chrome: If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch an instance of your browser and use it. :param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers through CDP. - :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name. - :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ + :param google_search: Enabled by default, Scrapling will set a Google referer header. + :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by `google_search` takes priority over the referer set here if used together._ :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only. :param max_pages: The maximum number of tabs to be opened at the same time. It will be used in rotation through a PagePool. :param user_data_dir: Path to a User Data Directory, which stores browser session data like cookies and local storage. The default is to create a temporary directory. @@ -271,11 +268,11 @@ class AsyncDynamicSession(AsyncSession, DynamicSessionMixin): """Opens up the browser and do your request based on your chosen options. :param url: The Target url. - :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name. + :param google_search: Enabled by default, Scrapling will set a Google referer header. :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000 :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object. :param page_action: Added for automation. A function that takes the `page` object and does the automation you need. - :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ + :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by `google_search` takes priority over the referer set here if used together._ :param disable_resources: Drop requests for unnecessary resources for a speed boost. Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`. :param blocked_domains: A set of domain names to block requests to. Subdomains are also matched (e.g., ``"example.com"`` blocks ``"sub.example.com"`` too). @@ -296,9 +293,7 @@ class AsyncDynamicSession(AsyncSession, DynamicSessionMixin): request_headers_keys = {h.lower() for h in params.extra_headers.keys()} if params.extra_headers else set() referer = ( - generate_convincing_referer(url) - if (params.google_search and "referer" not in request_headers_keys) - else None + "https://www.google.com/" if (params.google_search and "referer" not in request_headers_keys) else None ) for attempt in range(self._config.retries): diff --git a/scrapling/engines/_browsers/_stealth.py b/scrapling/engines/_browsers/_stealth.py index 797a0c1..8fb248f 100644 --- a/scrapling/engines/_browsers/_stealth.py +++ b/scrapling/engines/_browsers/_stealth.py @@ -16,7 +16,6 @@ from scrapling.core.utils import log from scrapling.core._types import Any, Optional, ProxyType, Unpack from scrapling.engines.toolbelt.proxy_rotation import is_proxy_error from scrapling.engines.toolbelt.convertor import Response, ResponseFactory -from scrapling.engines.toolbelt.fingerprints import generate_convincing_referer from scrapling.engines._browsers._types import StealthSession, StealthFetchParams from scrapling.engines._browsers._base import SyncSession, AsyncSession, StealthySessionMixin from scrapling.engines._browsers._validators import validate_fetch as _validate, StealthConfig @@ -66,8 +65,8 @@ class StealthySession(SyncSession, StealthySessionMixin): :param allow_webgl: Enabled by default. Disabling it disables WebGL and WebGL 2.0 support entirely. Disabling WebGL is not recommended as many WAFs now check if WebGL is enabled. :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute. :param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers through CDP. - :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name. - :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ + :param google_search: Enabled by default, Scrapling will set a Google referer header. + :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by `google_search` takes priority over the referer set here if used together._ :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only. :param user_data_dir: Path to a User Data Directory, which stores browser session data like cookies and local storage. The default is to create a temporary directory. :param extra_flags: A list of additional browser flags to pass to the browser on launch. @@ -189,11 +188,11 @@ class StealthySession(SyncSession, StealthySessionMixin): """Opens up the browser and do your request based on your chosen options. :param url: The Target url. - :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name. + :param google_search: Enabled by default, Scrapling will set a Google referer header. :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000 :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object. :param page_action: Added for automation. A function that takes the `page` object and does the automation you need. - :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ + :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by `google_search` takes priority over the referer set here if used together._ :param disable_resources: Drop requests for unnecessary resources for a speed boost. Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`. :param blocked_domains: A set of domain names to block requests to. Subdomains are also matched (e.g., ``"example.com"`` blocks ``"sub.example.com"`` too). @@ -214,9 +213,7 @@ class StealthySession(SyncSession, StealthySessionMixin): request_headers_keys = {h.lower() for h in params.extra_headers.keys()} if params.extra_headers else set() referer = ( - generate_convincing_referer(url) - if (params.google_search and "referer" not in request_headers_keys) - else None + "https://www.google.com/" if (params.google_search and "referer" not in request_headers_keys) else None ) for attempt in range(self._config.retries): @@ -322,8 +319,8 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin): :param allow_webgl: Enabled by default. Disabling it disables WebGL and WebGL 2.0 support entirely. Disabling WebGL is not recommended as many WAFs now check if WebGL is enabled. :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute. :param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers through CDP. - :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name. - :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ + :param google_search: Enabled by default, Scrapling will set a Google referer header. + :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by `google_search` takes priority over the referer set here if used together._ :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only. :param user_data_dir: Path to a User Data Directory, which stores browser session data like cookies and local storage. The default is to create a temporary directory. :param extra_flags: A list of additional browser flags to pass to the browser on launch. @@ -444,11 +441,11 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin): """Opens up the browser and do your request based on your chosen options. :param url: The Target url. - :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name. + :param google_search: Enabled by default, Scrapling will set a Google referer header. :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000 :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object. :param page_action: Added for automation. A function that takes the `page` object and does the automation you need. - :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ + :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by `google_search` takes priority over the referer set here if used together._ :param disable_resources: Drop requests for unnecessary resources for a speed boost. Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`. :param blocked_domains: A set of domain names to block requests to. Subdomains are also matched (e.g., ``"example.com"`` blocks ``"sub.example.com"`` too). @@ -470,9 +467,7 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin): request_headers_keys = {h.lower() for h in params.extra_headers.keys()} if params.extra_headers else set() referer = ( - generate_convincing_referer(url) - if (params.google_search and "referer" not in request_headers_keys) - else None + "https://www.google.com/" if (params.google_search and "referer" not in request_headers_keys) else None ) for attempt in range(self._config.retries): diff --git a/scrapling/engines/static.py b/scrapling/engines/static.py index 204c105..1f4b09b 100644 --- a/scrapling/engines/static.py +++ b/scrapling/engines/static.py @@ -26,7 +26,7 @@ from .toolbelt.custom import Response from .toolbelt.convertor import ResponseFactory from .toolbelt.proxy_rotation import ProxyRotator, is_proxy_error from ._browsers._types import RequestsSession, GetRequestParams, DataRequestParams, ImpersonateType -from .toolbelt.fingerprints import generate_convincing_referer, generate_headers, __default_useragent__ +from .toolbelt.fingerprints import generate_headers, __default_useragent__ _NO_SESSION: Any = object() @@ -166,14 +166,14 @@ class _ConfigurationLogic(ABC): """ 1. Adds a useragent to the headers if it doesn't have one 2. Generates real headers and append them to current headers - 3. Generates a referer header that looks like as if this request came from a Google's search of the current URL's domain. + 3. Sets a Google referer header. """ # Merge session headers with request headers, request takes precedence (if it was set) final_headers = {**self._default_headers, **(headers if headers else {})} headers_keys = {k.lower() for k in final_headers} if stealth: if "referer" not in headers_keys: - final_headers["referer"] = generate_convincing_referer(url) + final_headers["referer"] = "https://www.google.com/" if not impersonate_enabled: # Curl will generate the suitable headers extra_headers = generate_headers(browser_mode=False) @@ -672,7 +672,7 @@ class FetcherSession: """ :param impersonate: Browser version to impersonate. Can be a single browser string or a list of browser strings for random selection. (Default: latest available Chrome version) :param http3: Whether to use HTTP3. Defaults to False. It might be problematic if used it with `impersonate`. - :param stealthy_headers: If enabled (default), it creates and adds real browser headers. It also sets the referer header as if this request came from a Google search of URL's domain. + :param stealthy_headers: If enabled (default), it creates and adds real browser headers. It also sets a Google referer header. :param proxies: Dict of proxies to use. Format: {"http": proxy_url, "https": proxy_url}. :param proxy: Proxy URL to use. Format: "http://username:password@localhost:8030". Cannot be used together with the `proxies` parameter. diff --git a/scrapling/engines/toolbelt/fingerprints.py b/scrapling/engines/toolbelt/fingerprints.py index f755fb4..9a84199 100644 --- a/scrapling/engines/toolbelt/fingerprints.py +++ b/scrapling/engines/toolbelt/fingerprints.py @@ -5,7 +5,6 @@ Functions related to generating headers and fingerprints generally from functools import lru_cache from platform import system as platform_system -from tld import get_tld, Result from browserforge.headers import Browser, HeaderGenerator from browserforge.headers.generator import SUPPORTED_OPERATING_SYSTEMS @@ -18,34 +17,6 @@ chromium_version = 145 chrome_version = 145 -@lru_cache(10, typed=True) -def generate_convincing_referer(url: str) -> str | None: - """Generate a convincing Google referer header - - >>> generate_convincing_referer('https://www.somewebsite.com/blah') - 'https://www.google.com/' - - :param url: The URL you are about to fetch. - :return: Google's URL as referer, or None for localhost/IP addresses - """ - # Fixing the inaccurate return type hint in `get_tld` - extracted: Result | None = cast(Result, get_tld(url, as_object=True, fail_silently=True)) - if not extracted: - return None - - website_name = extracted.domain - - # Skip generating referer for localhost, IP addresses, or when there's no valid domain - if not website_name or not extracted.tld or website_name in ("localhost", "127.0.0.1", "::1"): - return None - - # Check if it's an IP address (simple check for IPv4) - if all(part.isdigit() for part in website_name.split(".") if part): - return None - - return "https://www.google.com/" - - @lru_cache(1, typed=True) def get_os_name() -> OSName | Tuple: """Get the current OS name in the same format needed for browserforge, if the OS is Unknown, return None so browserforge uses all. diff --git a/scrapling/fetchers/chrome.py b/scrapling/fetchers/chrome.py index 3b746f4..594b178 100644 --- a/scrapling/fetchers/chrome.py +++ b/scrapling/fetchers/chrome.py @@ -28,7 +28,7 @@ class DynamicFetcher(BaseFetcher): :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`. :param real_chrome: If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch an instance of your browser and use it. :param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers through CDP. - :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name. + :param google_search: Enabled by default, Scrapling will set a Google referer header. :param extra_headers: A dictionary of extra headers to add to the request. :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only. :param extra_flags: A list of additional browser flags to pass to the browser on launch. @@ -68,7 +68,7 @@ class DynamicFetcher(BaseFetcher): :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`. :param real_chrome: If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch an instance of your browser and use it. :param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers through CDP. - :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name. + :param google_search: Enabled by default, Scrapling will set a Google referer header. :param extra_headers: A dictionary of extra headers to add to the request. :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only. :param extra_flags: A list of additional browser flags to pass to the browser on launch. diff --git a/scrapling/fetchers/stealth_chrome.py b/scrapling/fetchers/stealth_chrome.py index 6a70246..1cf89f3 100644 --- a/scrapling/fetchers/stealth_chrome.py +++ b/scrapling/fetchers/stealth_chrome.py @@ -39,8 +39,8 @@ class StealthyFetcher(BaseFetcher): :param allow_webgl: Enabled by default. Disabling it disables WebGL and WebGL 2.0 support entirely. Disabling WebGL is not recommended as many WAFs now check if WebGL is enabled. :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute. :param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers through CDP. - :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name. - :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ + :param google_search: Enabled by default, Scrapling will set a Google referer header. + :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by `google_search` takes priority over the referer set here if used together._ :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only. :param user_data_dir: Path to a User Data Directory, which stores browser session data like cookies and local storage. The default is to create a temporary directory. :param extra_flags: A list of additional browser flags to pass to the browser on launch. @@ -88,8 +88,8 @@ class StealthyFetcher(BaseFetcher): :param allow_webgl: Enabled by default. Disabling it disables WebGL and WebGL 2.0 support entirely. Disabling WebGL is not recommended as many WAFs now check if WebGL is enabled. :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute. :param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers through CDP. - :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name. - :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ + :param google_search: Enabled by default, Scrapling will set a Google referer header. + :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by `google_search` takes priority over the referer set here if used together._ :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only. :param user_data_dir: Path to a User Data Directory, which stores browser session data like cookies and local storage. The default is to create a temporary directory. :param extra_flags: A list of additional browser flags to pass to the browser on launch.