freedak/Scrapling
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'dev' into fix/full-path-selector-duplicate-id

Browse Source
This commit is contained in:
Karim shoair
2026-04-13 12:03:15 +02:00
committed by GitHub
parent 273c8c2fa0 c2e66ab753
commit 3c23436448
37 changed files with 3898 additions and 76 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scrapling/__init__.py
+1 -1
View File
@@ -1,5 +1,5 @@
__author__ = "Karim Shoair (karim.shoair@pm.me)"
__version__ = "0.4.5"
__version__ = "0.4.6"
__copyright__ = "Copyright (c) 2024 Karim Shoair"
from typing import Any, TYPE_CHECKING
scrapling/cli.py
+24
View File
@@ -53,6 +53,8 @@ def __Request_and_Save(
if not output_path.is_absolute():
output_path = Path.cwd() / output_file
if ai_targeted:
kwargs.setdefault("block_ads", True)
response = fetcher_func(url, **kwargs)
Convertor.write_content_to_file(response, str(output_path), css_selector, main_content_only=ai_targeted)
log.info(f"Content successfully saved to '{output_path}'")
@@ -309,6 +311,16 @@ def _common_browser_options(f):
default=True,
help="Run browser in headless mode (default: True)",
),
option(
"--dns-over-https/--no-dns-over-https",
default=False,
help="Route DNS through Cloudflare's DoH to prevent DNS leaks when using proxies (default: False)",
),
option(
"--block-ads/--no-block-ads",
default=False,
help="Block requests to known ad and tracker domains (default: False)",
),
]
for decorator in decorators:
f = decorator(f)
@@ -498,6 +510,8 @@ def __build_browser_kwargs(
real_chrome,
proxy,
parsed_headers,
dns_over_https,
block_ads,
) -> Dict[str, Any]:
"""Build shared kwargs dict for browser-based commands."""
kwargs: Dict[str, Any] = {
@@ -507,6 +521,8 @@ def __build_browser_kwargs(
"timeout": timeout,
"locale": locale,
"real_chrome": real_chrome,
"dns_over_https": dns_over_https,
"block_ads": block_ads,
}
if wait > 0:
kwargs["wait"] = wait
@@ -538,6 +554,8 @@ def fetch(
proxy,
extra_headers,
ai_targeted,
dns_over_https,
block_ads,
):
"""Opens up a browser and fetch content using DynamicFetcher."""
parsed_headers, _ = _ParseHeaders(extra_headers, False)
@@ -552,6 +570,8 @@ def fetch(
real_chrome,
proxy,
parsed_headers,
dns_over_https,
block_ads,
)
from scrapling.fetchers import DynamicFetcher
@@ -597,6 +617,8 @@ def stealthy_fetch(
allow_webgl,
hide_canvas,
ai_targeted,
dns_over_https,
block_ads,
):
"""Opens up a browser with advanced stealth features and fetch content using StealthyFetcher."""
parsed_headers, _ = _ParseHeaders(extra_headers, False)
@@ -611,6 +633,8 @@ def stealthy_fetch(
real_chrome,
proxy,
parsed_headers,
dns_over_https,
block_ads,
)
kwargs.update(
{
scrapling/core/_shell_signatures.py
+17
View File
@@ -2,6 +2,7 @@ from scrapling.core._types import (
Any,
Dict,
List,
Set,
Tuple,
Sequence,
Callable,
@@ -46,6 +47,7 @@ _FETCH_PARAMS = {
"wait": int | float,
"timezone_id": str | None,
"page_action": Optional[Callable],
"page_setup": Optional[Callable],
"proxy": Optional[str | Dict[str, str] | Tuple],
"extra_headers": Optional[Dict[str, str]],
"timeout": int | float,
@@ -58,6 +60,13 @@ _FETCH_PARAMS = {
"cdp_url": Optional[str],
"useragent": Optional[str],
"extra_flags": Optional[List[str]],
"blocked_domains": Optional[Set[str]],
"block_ads": bool,
"retries": int,
"retry_delay": int | float,
"capture_xhr": str | None,
"executable_path": Optional[str],
"dns_over_https": bool,
}
_STEALTHY_FETCH_PARAMS = {
@@ -72,6 +81,7 @@ _STEALTHY_FETCH_PARAMS = {
"wait": int | float,
"timezone_id": str | None,
"page_action": Optional[Callable],
"page_setup": Optional[Callable],
"proxy": Optional[str | Dict[str, str] | Tuple],
"extra_headers": Optional[Dict[str, str]],
"timeout": int | float,
@@ -84,6 +94,13 @@ _STEALTHY_FETCH_PARAMS = {
"cdp_url": Optional[str],
"useragent": Optional[str],
"extra_flags": Optional[List[str]],
"blocked_domains": Optional[Set[str]],
"block_ads": bool,
"retries": int,
"retry_delay": int | float,
"capture_xhr": str | None,
"executable_path": Optional[str],
"dns_over_https": bool,
"allow_webgl": bool,
"hide_canvas": bool,
"block_webrtc": bool,
scrapling/core/ai.py
+3
View File
@@ -183,6 +183,7 @@ class ScraplingMCPServer:
cookies=cookies,
cdp_url=cdp_url,
headless=headless,
block_ads=True,
max_pages=max_pages,
useragent=useragent,
timezone_id=timezone_id,
@@ -569,6 +570,7 @@ class ScraplingMCPServer:
cookies=cookies,
cdp_url=cdp_url,
headless=headless,
block_ads=True,
max_pages=len(urls),
useragent=useragent,
timezone_id=timezone_id,
@@ -777,6 +779,7 @@ class ScraplingMCPServer:
timeout=timeout,
cookies=cookies,
headless=headless,
block_ads=True,
useragent=useragent,
timezone_id=timezone_id,
real_chrome=real_chrome,
scrapling/engines/_browsers/_base.py
+7
View File
@@ -455,6 +455,13 @@ class BaseSessionMixin:
if config.extra_flags or extra_flags:
flags = list(set(tuple(flags) + tuple(config.extra_flags or extra_flags or ())))
if config.dns_over_https:
doh_flag = "--dns-over-https-templates=https://cloudflare-dns.com/dns-query"
if isinstance(flags, list):
flags.append(doh_flag)
else:
flags = list(flags) + [doh_flag]
self._browser_options.update(
{
"args": flags,
scrapling/engines/_browsers/_controllers.py
+20 -4
View File
@@ -47,7 +47,8 @@ class DynamicSession(SyncSession, DynamicSessionMixin):
:param network_idle: Wait for the page until there are no network connections for at least 500 ms.
:param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
:param page_action: Added for automation. A function that takes the `page` object and does the automation you need.
:param page_action: Added for automation. A function that takes the `page` object, runs after navigation, and does the automation you need.
:param page_setup: A function that takes the `page` object, runs before navigation. Use it to register event listeners or routes that must be set up before the page loads.
:param wait_selector: Wait for a specific CSS selector to be in a specific state.
:param init_script: An absolute path to a JavaScript file to be executed on page creation for all pages in this session.
:param locale: Specify user locale, for example, `en-GB`, `de-DE`, etc. Locale will affect navigator.language value, Accept-Language request header value as well as number and date formatting
@@ -105,7 +106,8 @@ class DynamicSession(SyncSession, DynamicSessionMixin):
:param google_search: Enabled by default, Scrapling will set a Google referer header.
:param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
:param page_action: Added for automation. A function that takes the `page` object and does the automation you need.
:param page_action: Added for automation. A function that takes the `page` object, runs after navigation, and does the automation you need.
:param page_setup: A function that takes the `page` object, runs before navigation. Use it to register event listeners or routes that must be set up before the page loads.
:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by `google_search` takes priority over the referer set here if used together._
:param disable_resources: Drop requests for unnecessary resources for a speed boost.
Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
@@ -152,6 +154,12 @@ class DynamicSession(SyncSession, DynamicSessionMixin):
),
)
if params.page_setup:
try:
params.page_setup(page)
except Exception as e: # pragma: no cover
log.error(f"Error executing page_setup: {e}")
try:
first_response = page.goto(url, referer=referer)
self._wait_for_page_stability(page, params.load_dom, params.network_idle)
@@ -228,7 +236,8 @@ class AsyncDynamicSession(AsyncSession, DynamicSessionMixin):
:param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute.
:param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
:param page_action: Added for automation. A function that takes the `page` object and does the automation you need.
:param page_action: Added for automation. A function that takes the `page` object, runs after navigation, and does the automation you need.
:param page_setup: A function that takes the `page` object, runs before navigation. Use it to register event listeners or routes that must be set up before the page loads.
:param wait_selector: Wait for a specific CSS selector to be in a specific state.
:param init_script: An absolute path to a JavaScript file to be executed on page creation for all pages in this session.
:param locale: Specify user locale, for example, `en-GB`, `de-DE`, etc. Locale will affect navigator.language value, Accept-Language request header value as well as number and date formatting
@@ -285,7 +294,8 @@ class AsyncDynamicSession(AsyncSession, DynamicSessionMixin):
:param google_search: Enabled by default, Scrapling will set a Google referer header.
:param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
:param page_action: Added for automation. A function that takes the `page` object and does the automation you need.
:param page_action: Added for automation. A function that takes the `page` object, runs after navigation, and does the automation you need.
:param page_setup: A function that takes the `page` object, runs before navigation. Use it to register event listeners or routes that must be set up before the page loads.
:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by `google_search` takes priority over the referer set here if used together._
:param disable_resources: Drop requests for unnecessary resources for a speed boost.
Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
@@ -333,6 +343,12 @@ class AsyncDynamicSession(AsyncSession, DynamicSessionMixin):
),
)
if params.page_setup:
try:
await params.page_setup(page)
except Exception as e: # pragma: no cover
log.error(f"Error executing page_setup: {e}")
try:
first_response = await page.goto(url, referer=referer)
await self._wait_for_page_stability(page, params.load_dom, params.network_idle)
scrapling/engines/_browsers/_stealth.py
+20 -4
View File
@@ -47,7 +47,8 @@ class StealthySession(SyncSession, StealthySessionMixin):
:param network_idle: Wait for the page until there are no network connections for at least 500 ms.
:param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
:param page_action: Added for automation. A function that takes the `page` object and does the automation you need.
:param page_action: Added for automation. A function that takes the `page` object, runs after navigation, and does the automation you need.
:param page_setup: A function that takes the `page` object, runs before navigation. Use it to register event listeners or routes that must be set up before the page loads.
:param wait_selector: Wait for a specific CSS selector to be in a specific state.
:param init_script: An absolute path to a JavaScript file to be executed on page creation for all pages in this session.
:param locale: Specify user locale, for example, `en-GB`, `de-DE`, etc. Locale will affect navigator.language value, Accept-Language request header value as well as number and date formatting
@@ -187,7 +188,8 @@ class StealthySession(SyncSession, StealthySessionMixin):
:param google_search: Enabled by default, Scrapling will set a Google referer header.
:param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
:param page_action: Added for automation. A function that takes the `page` object and does the automation you need.
:param page_action: Added for automation. A function that takes the `page` object, runs after navigation, and does the automation you need.
:param page_setup: A function that takes the `page` object, runs before navigation. Use it to register event listeners or routes that must be set up before the page loads.
:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by `google_search` takes priority over the referer set here if used together._
:param disable_resources: Drop requests for unnecessary resources for a speed boost.
Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
@@ -235,6 +237,12 @@ class StealthySession(SyncSession, StealthySessionMixin):
),
)
if params.page_setup:
try:
params.page_setup(page)
except Exception as e: # pragma: no cover
log.error(f"Error executing page_setup: {e}")
try:
first_response = page.goto(url, referer=referer)
self._wait_for_page_stability(page, params.load_dom, params.network_idle)
@@ -315,7 +323,8 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin):
:param network_idle: Wait for the page until there are no network connections for at least 500 ms.
:param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
:param page_action: Added for automation. A function that takes the `page` object and does the automation you need.
:param page_action: Added for automation. A function that takes the `page` object, runs after navigation, and does the automation you need.
:param page_setup: A function that takes the `page` object, runs before navigation. Use it to register event listeners or routes that must be set up before the page loads.
:param wait_selector: Wait for a specific CSS selector to be in a specific state.
:param init_script: An absolute path to a JavaScript file to be executed on page creation for all pages in this session.
:param locale: Specify user locale, for example, `en-GB`, `de-DE`, etc. Locale will affect navigator.language value, Accept-Language request header value as well as number and date formatting
@@ -454,7 +463,8 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin):
:param google_search: Enabled by default, Scrapling will set a Google referer header.
:param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
:param page_action: Added for automation. A function that takes the `page` object and does the automation you need.
:param page_action: Added for automation. A function that takes the `page` object, runs after navigation, and does the automation you need.
:param page_setup: A function that takes the `page` object, runs before navigation. Use it to register event listeners or routes that must be set up before the page loads.
:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by `google_search` takes priority over the referer set here if used together._
:param disable_resources: Drop requests for unnecessary resources for a speed boost.
Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
@@ -503,6 +513,12 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin):
),
)
if params.page_setup:
try:
await params.page_setup(page)
except Exception as e: # pragma: no cover
log.error(f"Error executing page_setup: {e}")
try:
first_response = await page.goto(url, referer=referer)
await self._wait_for_page_stability(page, params.load_dom, params.network_idle)
scrapling/engines/_browsers/_types.py
+4
View File
@@ -74,6 +74,7 @@ class PlaywrightSession(TypedDict, total=False):
wait: int | float
timezone_id: str | None
page_action: Optional[Callable]
page_setup: Optional[Callable]
proxy: Optional[str | Dict[str, str] | Tuple]
proxy_rotator: Optional[ProxyRotator]
extra_headers: Optional[Dict[str, str]]
@@ -88,10 +89,12 @@ class PlaywrightSession(TypedDict, total=False):
useragent: Optional[str]
extra_flags: Optional[List[str]]
blocked_domains: Optional[Set[str]]
block_ads: bool
retries: int
retry_delay: int | float
capture_xhr: str | None
executable_path: Optional[str]
dns_over_https: bool
class PlaywrightFetchParams(TypedDict, total=False):
@@ -103,6 +106,7 @@ class PlaywrightFetchParams(TypedDict, total=False):
disable_resources: bool
wait_selector: Optional[str]
page_action: Optional[Callable]
page_setup: Optional[Callable]
selector_config: Optional[Dict]
extra_headers: Optional[Dict[str, str]]
wait_selector_state: SelectorWaitStates
scrapling/engines/_browsers/_validators.py
+15 -1
View File
@@ -53,7 +53,7 @@ def _is_invalid_cdp_url(cdp_url: str) -> bool | str:
# Type aliases for cleaner annotations
PagesCount = Annotated[int, Meta(ge=1, le=50)]
RetriesCount = Annotated[int, Meta(ge=1, le=10)]
Seconds = Annotated[int, float, Meta(ge=0)]
Seconds = Annotated[float, Meta(ge=0)]
class PlaywrightConfig(Struct, kw_only=True, frozen=False, weakref=True):
@@ -71,6 +71,7 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False, weakref=True):
wait: Seconds = 0
timezone_id: str | None = ""
page_action: Optional[Callable] = None
page_setup: Optional[Callable] = None
proxy: Optional[str | Dict[str, str] | Tuple] = None # The default value for proxy in Playwright's source is `None`
proxy_rotator: Optional[ProxyRotator] = None
extra_headers: Optional[Dict[str, str]] = None
@@ -85,15 +86,19 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False, weakref=True):
useragent: Optional[str] = None
extra_flags: Optional[List[str]] = None
blocked_domains: Optional[Set[str]] = None
block_ads: bool = False
retries: RetriesCount = 3
retry_delay: Seconds = 1
capture_xhr: str | None = None
executable_path: Optional[str] = None
dns_over_https: bool = False
def __post_init__(self): # pragma: no cover
"""Custom validation after msgspec validation"""
if self.page_action and not callable(self.page_action):
raise TypeError(f"page_action must be callable, got {type(self.page_action).__name__}")
if self.page_setup and not callable(self.page_setup):
raise TypeError(f"page_setup must be callable, got {type(self.page_setup).__name__}")
if self.proxy and self.proxy_rotator:
raise ValueError(
"Cannot use 'proxy_rotator' together with 'proxy'. "
@@ -127,6 +132,14 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False, weakref=True):
if validation_msg:
raise ValueError(validation_msg)
if self.block_ads:
from scrapling.engines.toolbelt.ad_domains import AD_DOMAINS
if self.blocked_domains:
self.blocked_domains = self.blocked_domains | set(AD_DOMAINS)
else:
self.blocked_domains = set(AD_DOMAINS)
class StealthConfig(PlaywrightConfig, kw_only=True, frozen=False, weakref=True):
allow_webgl: bool = True
@@ -150,6 +163,7 @@ class _fetch_params:
timeout: Seconds
wait: Seconds
page_action: Optional[Callable]
page_setup: Optional[Callable]
extra_headers: Optional[Dict[str, str]]
disable_resources: bool
wait_selector: Optional[str]
scrapling/engines/toolbelt/ad_domains.py
+3537
View File
File diff suppressed because it is too large Load Diff
scrapling/engines/toolbelt/navigation.py
+25 -4
View File
@@ -19,6 +19,27 @@ class ProxyDict(Struct):
password: str = ""
def _is_domain_blocked(hostname: str, domains: frozenset) -> bool:
"""Check if a hostname matches any blocked domain using O(1) frozenset lookups.
Walks up the hostname's suffix chain: for "tracker.ads.doubleclick.net",
checks "tracker.ads.doubleclick.net", "ads.doubleclick.net", "doubleclick.net".
:param hostname: The hostname to check.
:param domains: A frozenset of blocked domain names.
:return: True if the hostname or any of its parent domains is in the blocked set.
"""
if hostname in domains:
return True
idx = hostname.find(".")
while idx != -1:
suffix = hostname[idx + 1 :]
if "." in suffix and suffix in domains:
return True
idx = hostname.find(".", idx + 1)
return False
def create_intercept_handler(disable_resources: bool, blocked_domains: Optional[Set[str]] = None) -> Callable:
"""Create a route handler that blocks both resource types and specific domains.
@@ -27,7 +48,7 @@ def create_intercept_handler(disable_resources: bool, blocked_domains: Optional[
:return: A sync route handler function.
"""
disabled_resources = EXTRA_RESOURCES if disable_resources else set()
domains = blocked_domains or set()
domains = frozenset(blocked_domains) if blocked_domains else frozenset()
def handler(route: Route):
if route.request.resource_type in disabled_resources:
@@ -35,7 +56,7 @@ def create_intercept_handler(disable_resources: bool, blocked_domains: Optional[
route.abort()
elif domains:
hostname = urlparse(route.request.url).hostname or ""
if any(hostname == d or hostname.endswith("." + d) for d in domains):
if _is_domain_blocked(hostname, domains):
log.debug(f'Blocking request to blocked domain "{hostname}" ({route.request.url})')
route.abort()
else:
@@ -54,7 +75,7 @@ def create_async_intercept_handler(disable_resources: bool, blocked_domains: Opt
:return: An async route handler function.
"""
disabled_resources = EXTRA_RESOURCES if disable_resources else set()
domains = blocked_domains or set()
domains = frozenset(blocked_domains) if blocked_domains else frozenset()
async def handler(route: async_Route):
if route.request.resource_type in disabled_resources:
@@ -62,7 +83,7 @@ def create_async_intercept_handler(disable_resources: bool, blocked_domains: Opt
await route.abort()
elif domains:
hostname = urlparse(route.request.url).hostname or ""
if any(hostname == d or hostname.endswith("." + d) for d in domains):
if _is_domain_blocked(hostname, domains):
log.debug(f'Blocking request to blocked domain "{hostname}" ({route.request.url})')
await route.abort()
else:
scrapling/fetchers/chrome.py
+8 -2
View File
@@ -15,13 +15,16 @@ class DynamicFetcher(BaseFetcher):
:param headless: Run the browser in headless/hidden (default), or headful/visible mode.
:param disable_resources: Drop requests for unnecessary resources for a speed boost.
:param blocked_domains: A set of domain names to block requests to. Subdomains are also matched (e.g., ``"example.com"`` blocks ``"sub.example.com"`` too).
:param block_ads: Block requests to ~3,500 known ad/tracking domains. Can be combined with ``blocked_domains``.
:param dns_over_https: Route DNS queries through Cloudflare's DNS-over-HTTPS to prevent DNS leaks when using proxies.
:param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it.
:param cookies: Set cookies for the next request.
:param network_idle: Wait for the page until there are no network connections for at least 500 ms.
:param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute.
:param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the Response object.
:param page_action: Added for automation. A function that takes the `page` object and does the automation you need.
:param page_action: Added for automation. A function that takes the `page` object, runs after navigation, and does the automation you need.
:param page_setup: A function that takes the `page` object, runs before navigation. Use it to register event listeners or routes that must be set up before the page loads.
:param wait_selector: Wait for a specific CSS selector to be in a specific state.
:param init_script: An absolute path to a JavaScript file to be executed on page creation with this request.
:param locale: Set the locale for the browser if wanted. Defaults to the system default locale.
@@ -55,13 +58,16 @@ class DynamicFetcher(BaseFetcher):
:param headless: Run the browser in headless/hidden (default), or headful/visible mode.
:param disable_resources: Drop requests for unnecessary resources for a speed boost.
:param blocked_domains: A set of domain names to block requests to. Subdomains are also matched (e.g., ``"example.com"`` blocks ``"sub.example.com"`` too).
:param block_ads: Block requests to ~3,500 known ad/tracking domains. Can be combined with ``blocked_domains``.
:param dns_over_https: Route DNS queries through Cloudflare's DNS-over-HTTPS to prevent DNS leaks when using proxies.
:param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it.
:param cookies: Set cookies for the next request.
:param network_idle: Wait for the page until there are no network connections for at least 500 ms.
:param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute.
:param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the Response object.
:param page_action: Added for automation. A function that takes the `page` object and does the automation you need.
:param page_action: Added for automation. A function that takes the `page` object, runs after navigation, and does the automation you need.
:param page_setup: A function that takes the `page` object, runs before navigation. Use it to register event listeners or routes that must be set up before the page loads.
:param wait_selector: Wait for a specific CSS selector to be in a specific state.
:param init_script: An absolute path to a JavaScript file to be executed on page creation with this request.
:param locale: Set the locale for the browser if wanted. Defaults to the system default locale.
scrapling/fetchers/stealth_chrome.py
+8 -2
View File
@@ -20,12 +20,15 @@ class StealthyFetcher(BaseFetcher):
:param disable_resources: Drop requests for unnecessary resources for a speed boost.
Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
:param blocked_domains: A set of domain names to block requests to. Subdomains are also matched (e.g., ``"example.com"`` blocks ``"sub.example.com"`` too).
:param block_ads: Block requests to ~3,500 known ad/tracking domains. Can be combined with ``blocked_domains``.
:param dns_over_https: Route DNS queries through Cloudflare's DNS-over-HTTPS to prevent DNS leaks when using proxies.
:param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it.
:param cookies: Set cookies for the next request.
:param network_idle: Wait for the page until there are no network connections for at least 500 ms.
:param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
:param page_action: Added for automation. A function that takes the `page` object and does the automation you need.
:param page_action: Added for automation. A function that takes the `page` object, runs after navigation, and does the automation you need.
:param page_setup: A function that takes the `page` object, runs before navigation. Use it to register event listeners or routes that must be set up before the page loads.
:param wait_selector: Wait for a specific CSS selector to be in a specific state.
:param init_script: An absolute path to a JavaScript file to be executed on page creation for all pages in this session.
:param locale: Specify user locale, for example, `en-GB`, `de-DE`, etc. Locale will affect navigator.language value, Accept-Language request header value as well as number and date formatting
@@ -69,12 +72,15 @@ class StealthyFetcher(BaseFetcher):
:param disable_resources: Drop requests for unnecessary resources for a speed boost.
Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
:param blocked_domains: A set of domain names to block requests to. Subdomains are also matched (e.g., ``"example.com"`` blocks ``"sub.example.com"`` too).
:param block_ads: Block requests to ~3,500 known ad/tracking domains. Can be combined with ``blocked_domains``.
:param dns_over_https: Route DNS queries through Cloudflare's DNS-over-HTTPS to prevent DNS leaks when using proxies.
:param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it.
:param cookies: Set cookies for the next request.
:param network_idle: Wait for the page until there are no network connections for at least 500 ms.
:param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
:param page_action: Added for automation. A function that takes the `page` object and does the automation you need.
:param page_action: Added for automation. A function that takes the `page` object, runs after navigation, and does the automation you need.
:param page_setup: A function that takes the `page` object, runs before navigation. Use it to register event listeners or routes that must be set up before the page loads.
:param wait_selector: Wait for a specific CSS selector to be in a specific state.
:param init_script: An absolute path to a JavaScript file to be executed on page creation for all pages in this session.
:param locale: Specify user locale, for example, `en-GB`, `de-DE`, etc. Locale will affect navigator.language value, Accept-Language request header value as well as number and date formatting