From be28fe16ec1501e1f534b007b5263ac06b301ebe Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Sun, 12 Apr 2026 18:03:03 +0200 Subject: [PATCH] feat(browsers): add new feature to enable DNS-over-HTTP to prevent DNS leaks --- scrapling/cli.py | 11 +++++++++++ scrapling/core/_shell_signatures.py | 2 ++ scrapling/engines/_browsers/_base.py | 7 +++++++ scrapling/engines/_browsers/_types.py | 1 + scrapling/engines/_browsers/_validators.py | 1 + scrapling/fetchers/chrome.py | 2 ++ scrapling/fetchers/stealth_chrome.py | 2 ++ 7 files changed, 26 insertions(+) diff --git a/scrapling/cli.py b/scrapling/cli.py index 9ca0dd4..3327dcb 100644 --- a/scrapling/cli.py +++ b/scrapling/cli.py @@ -309,6 +309,11 @@ def _common_browser_options(f): default=True, help="Run browser in headless mode (default: True)", ), + option( + "--dns-over-https/--no-dns-over-https", + default=False, + help="Route DNS through Cloudflare's DoH to prevent DNS leaks when using proxies (default: False)", + ), option( "--block-ads/--no-block-ads", default=False, @@ -503,6 +508,7 @@ def __build_browser_kwargs( real_chrome, proxy, parsed_headers, + dns_over_https, block_ads, ) -> Dict[str, Any]: """Build shared kwargs dict for browser-based commands.""" @@ -513,6 +519,7 @@ def __build_browser_kwargs( "timeout": timeout, "locale": locale, "real_chrome": real_chrome, + "dns_over_https": dns_over_https, "block_ads": block_ads, } if wait > 0: @@ -545,6 +552,7 @@ def fetch( proxy, extra_headers, ai_targeted, + dns_over_https, block_ads, ): """Opens up a browser and fetch content using DynamicFetcher.""" @@ -560,6 +568,7 @@ def fetch( real_chrome, proxy, parsed_headers, + dns_over_https, block_ads, ) from scrapling.fetchers import DynamicFetcher @@ -606,6 +615,7 @@ def stealthy_fetch( allow_webgl, hide_canvas, ai_targeted, + dns_over_https, block_ads, ): """Opens up a browser with advanced stealth features and fetch content using StealthyFetcher.""" @@ -621,6 +631,7 @@ def stealthy_fetch( real_chrome, proxy, parsed_headers, + dns_over_https, block_ads, ) kwargs.update( diff --git a/scrapling/core/_shell_signatures.py b/scrapling/core/_shell_signatures.py index afa482d..b2340fb 100644 --- a/scrapling/core/_shell_signatures.py +++ b/scrapling/core/_shell_signatures.py @@ -65,6 +65,7 @@ _FETCH_PARAMS = { "retry_delay": int | float, "capture_xhr": str | None, "executable_path": Optional[str], + "dns_over_https": bool, } _STEALTHY_FETCH_PARAMS = { @@ -97,6 +98,7 @@ _STEALTHY_FETCH_PARAMS = { "retry_delay": int | float, "capture_xhr": str | None, "executable_path": Optional[str], + "dns_over_https": bool, "allow_webgl": bool, "hide_canvas": bool, "block_webrtc": bool, diff --git a/scrapling/engines/_browsers/_base.py b/scrapling/engines/_browsers/_base.py index 2d42b08..c9d390c 100644 --- a/scrapling/engines/_browsers/_base.py +++ b/scrapling/engines/_browsers/_base.py @@ -455,6 +455,13 @@ class BaseSessionMixin: if config.extra_flags or extra_flags: flags = list(set(tuple(flags) + tuple(config.extra_flags or extra_flags or ()))) + if config.dns_over_https: + doh_flag = "--dns-over-https-templates=https://cloudflare-dns.com/dns-query" + if isinstance(flags, list): + flags.append(doh_flag) + else: + flags = list(flags) + [doh_flag] + self._browser_options.update( { "args": flags, diff --git a/scrapling/engines/_browsers/_types.py b/scrapling/engines/_browsers/_types.py index 932b2e2..5030480 100644 --- a/scrapling/engines/_browsers/_types.py +++ b/scrapling/engines/_browsers/_types.py @@ -93,6 +93,7 @@ class PlaywrightSession(TypedDict, total=False): retry_delay: int | float capture_xhr: str | None executable_path: Optional[str] + dns_over_https: bool class PlaywrightFetchParams(TypedDict, total=False): diff --git a/scrapling/engines/_browsers/_validators.py b/scrapling/engines/_browsers/_validators.py index 8d47061..f7bb34e 100644 --- a/scrapling/engines/_browsers/_validators.py +++ b/scrapling/engines/_browsers/_validators.py @@ -90,6 +90,7 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False, weakref=True): retry_delay: Seconds = 1 capture_xhr: str | None = None executable_path: Optional[str] = None + dns_over_https: bool = False def __post_init__(self): # pragma: no cover """Custom validation after msgspec validation""" diff --git a/scrapling/fetchers/chrome.py b/scrapling/fetchers/chrome.py index 0c06aa1..f7795b7 100644 --- a/scrapling/fetchers/chrome.py +++ b/scrapling/fetchers/chrome.py @@ -16,6 +16,7 @@ class DynamicFetcher(BaseFetcher): :param disable_resources: Drop requests for unnecessary resources for a speed boost. :param blocked_domains: A set of domain names to block requests to. Subdomains are also matched (e.g., ``"example.com"`` blocks ``"sub.example.com"`` too). :param block_ads: Block requests to ~3,500 known ad/tracking domains. Can be combined with ``blocked_domains``. + :param dns_over_https: Route DNS queries through Cloudflare's DNS-over-HTTPS to prevent DNS leaks when using proxies. :param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it. :param cookies: Set cookies for the next request. :param network_idle: Wait for the page until there are no network connections for at least 500 ms. @@ -57,6 +58,7 @@ class DynamicFetcher(BaseFetcher): :param disable_resources: Drop requests for unnecessary resources for a speed boost. :param blocked_domains: A set of domain names to block requests to. Subdomains are also matched (e.g., ``"example.com"`` blocks ``"sub.example.com"`` too). :param block_ads: Block requests to ~3,500 known ad/tracking domains. Can be combined with ``blocked_domains``. + :param dns_over_https: Route DNS queries through Cloudflare's DNS-over-HTTPS to prevent DNS leaks when using proxies. :param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it. :param cookies: Set cookies for the next request. :param network_idle: Wait for the page until there are no network connections for at least 500 ms. diff --git a/scrapling/fetchers/stealth_chrome.py b/scrapling/fetchers/stealth_chrome.py index 0c7134c..70ce937 100644 --- a/scrapling/fetchers/stealth_chrome.py +++ b/scrapling/fetchers/stealth_chrome.py @@ -21,6 +21,7 @@ class StealthyFetcher(BaseFetcher): Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`. :param blocked_domains: A set of domain names to block requests to. Subdomains are also matched (e.g., ``"example.com"`` blocks ``"sub.example.com"`` too). :param block_ads: Block requests to ~3,500 known ad/tracking domains. Can be combined with ``blocked_domains``. + :param dns_over_https: Route DNS queries through Cloudflare's DNS-over-HTTPS to prevent DNS leaks when using proxies. :param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it. :param cookies: Set cookies for the next request. :param network_idle: Wait for the page until there are no network connections for at least 500 ms. @@ -71,6 +72,7 @@ class StealthyFetcher(BaseFetcher): Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`. :param blocked_domains: A set of domain names to block requests to. Subdomains are also matched (e.g., ``"example.com"`` blocks ``"sub.example.com"`` too). :param block_ads: Block requests to ~3,500 known ad/tracking domains. Can be combined with ``blocked_domains``. + :param dns_over_https: Route DNS queries through Cloudflare's DNS-over-HTTPS to prevent DNS leaks when using proxies. :param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it. :param cookies: Set cookies for the next request. :param network_idle: Wait for the page until there are no network connections for at least 500 ms.