feat(browsers): add new feature to enable DNS-over-HTTP to prevent DNS leaks

scrapling/cli.py

@@ -309,6 +309,11 @@ def _common_browser_options(f):
             default=True,
             help="Run browser in headless mode (default: True)",
         ),
+        option(
+            "--dns-over-https/--no-dns-over-https",
+            default=False,
+            help="Route DNS through Cloudflare's DoH to prevent DNS leaks when using proxies (default: False)",
+        ),
         option(
             "--block-ads/--no-block-ads",
             default=False,
@@ -503,6 +508,7 @@ def __build_browser_kwargs(
     real_chrome,
     proxy,
     parsed_headers,
+    dns_over_https,
     block_ads,
 ) -> Dict[str, Any]:
     """Build shared kwargs dict for browser-based commands."""
@@ -513,6 +519,7 @@ def __build_browser_kwargs(
         "timeout": timeout,
         "locale": locale,
         "real_chrome": real_chrome,
+        "dns_over_https": dns_over_https,
         "block_ads": block_ads,
     }
     if wait > 0:
@@ -545,6 +552,7 @@ def fetch(
     proxy,
     extra_headers,
     ai_targeted,
+    dns_over_https,
     block_ads,
 ):
     """Opens up a browser and fetch content using DynamicFetcher."""
@@ -560,6 +568,7 @@ def fetch(
         real_chrome,
         proxy,
         parsed_headers,
+        dns_over_https,
         block_ads,
     )
     from scrapling.fetchers import DynamicFetcher
@@ -606,6 +615,7 @@ def stealthy_fetch(
     allow_webgl,
     hide_canvas,
     ai_targeted,
+    dns_over_https,
     block_ads,
 ):
     """Opens up a browser with advanced stealth features and fetch content using StealthyFetcher."""
@@ -621,6 +631,7 @@ def stealthy_fetch(
         real_chrome,
         proxy,
         parsed_headers,
+        dns_over_https,
         block_ads,
     )
     kwargs.update(

scrapling/core/_shell_signatures.py

@@ -65,6 +65,7 @@ _FETCH_PARAMS = {
     "retry_delay": int | float,
     "capture_xhr": str | None,
     "executable_path": Optional[str],
+    "dns_over_https": bool,
 }
 
 _STEALTHY_FETCH_PARAMS = {
@@ -97,6 +98,7 @@ _STEALTHY_FETCH_PARAMS = {
     "retry_delay": int | float,
     "capture_xhr": str | None,
     "executable_path": Optional[str],
+    "dns_over_https": bool,
     "allow_webgl": bool,
     "hide_canvas": bool,
     "block_webrtc": bool,

scrapling/engines/_browsers/_base.py

@@ -455,6 +455,13 @@ class BaseSessionMixin:
             if config.extra_flags or extra_flags:
                 flags = list(set(tuple(flags) + tuple(config.extra_flags or extra_flags or ())))
 
+            if config.dns_over_https:
+                doh_flag = "--dns-over-https-templates=https://cloudflare-dns.com/dns-query"
+                if isinstance(flags, list):
+                    flags.append(doh_flag)
+                else:
+                    flags = list(flags) + [doh_flag]
+
             self._browser_options.update(
                 {
                     "args": flags,

scrapling/engines/_browsers/_types.py

@@ -93,6 +93,7 @@ class PlaywrightSession(TypedDict, total=False):
     retry_delay: int | float
     capture_xhr: str | None
     executable_path: Optional[str]
+    dns_over_https: bool
 
 
 class PlaywrightFetchParams(TypedDict, total=False):

scrapling/engines/_browsers/_validators.py

@@ -90,6 +90,7 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False, weakref=True):
     retry_delay: Seconds = 1
     capture_xhr: str | None = None
     executable_path: Optional[str] = None
+    dns_over_https: bool = False
 
     def __post_init__(self):  # pragma: no cover
         """Custom validation after msgspec validation"""

scrapling/fetchers/chrome.py

@@ -16,6 +16,7 @@ class DynamicFetcher(BaseFetcher):
         :param disable_resources: Drop requests for unnecessary resources for a speed boost.
         :param blocked_domains: A set of domain names to block requests to. Subdomains are also matched (e.g., ``"example.com"`` blocks ``"sub.example.com"`` too).
         :param block_ads: Block requests to ~3,500 known ad/tracking domains. Can be combined with ``blocked_domains``.
+        :param dns_over_https: Route DNS queries through Cloudflare's DNS-over-HTTPS to prevent DNS leaks when using proxies.
         :param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it.
         :param cookies: Set cookies for the next request.
         :param network_idle: Wait for the page until there are no network connections for at least 500 ms.
@@ -57,6 +58,7 @@ class DynamicFetcher(BaseFetcher):
         :param disable_resources: Drop requests for unnecessary resources for a speed boost.
         :param blocked_domains: A set of domain names to block requests to. Subdomains are also matched (e.g., ``"example.com"`` blocks ``"sub.example.com"`` too).
         :param block_ads: Block requests to ~3,500 known ad/tracking domains. Can be combined with ``blocked_domains``.
+        :param dns_over_https: Route DNS queries through Cloudflare's DNS-over-HTTPS to prevent DNS leaks when using proxies.
         :param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it.
         :param cookies: Set cookies for the next request.
         :param network_idle: Wait for the page until there are no network connections for at least 500 ms.

scrapling/fetchers/stealth_chrome.py

@@ -21,6 +21,7 @@ class StealthyFetcher(BaseFetcher):
             Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
         :param blocked_domains: A set of domain names to block requests to. Subdomains are also matched (e.g., ``"example.com"`` blocks ``"sub.example.com"`` too).
         :param block_ads: Block requests to ~3,500 known ad/tracking domains. Can be combined with ``blocked_domains``.
+        :param dns_over_https: Route DNS queries through Cloudflare's DNS-over-HTTPS to prevent DNS leaks when using proxies.
         :param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it.
         :param cookies: Set cookies for the next request.
         :param network_idle: Wait for the page until there are no network connections for at least 500 ms.
@@ -71,6 +72,7 @@ class StealthyFetcher(BaseFetcher):
             Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
         :param blocked_domains: A set of domain names to block requests to. Subdomains are also matched (e.g., ``"example.com"`` blocks ``"sub.example.com"`` too).
         :param block_ads: Block requests to ~3,500 known ad/tracking domains. Can be combined with ``blocked_domains``.
+        :param dns_over_https: Route DNS queries through Cloudflare's DNS-over-HTTPS to prevent DNS leaks when using proxies.
         :param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it.
         :param cookies: Set cookies for the next request.
         :param network_idle: Wait for the page until there are no network connections for at least 500 ms.