feat(browsers): add a new feature to block ads

This is working by aborting all requests to known ads domains.

scrapling/cli.py

@@ -309,6 +309,11 @@ def _common_browser_options(f):
             default=True,
             help="Run browser in headless mode (default: True)",
         ),
+        option(
+            "--block-ads/--no-block-ads",
+            default=False,
+            help="Block requests to known ad and tracker domains (default: False)",
+        ),
     ]
     for decorator in decorators:
         f = decorator(f)
@@ -498,6 +503,7 @@ def __build_browser_kwargs(
     real_chrome,
     proxy,
     parsed_headers,
+    block_ads,
 ) -> Dict[str, Any]:
     """Build shared kwargs dict for browser-based commands."""
     kwargs: Dict[str, Any] = {
@@ -507,6 +513,7 @@ def __build_browser_kwargs(
         "timeout": timeout,
         "locale": locale,
         "real_chrome": real_chrome,
+        "block_ads": block_ads,
     }
     if wait > 0:
         kwargs["wait"] = wait
@@ -538,6 +545,7 @@ def fetch(
     proxy,
     extra_headers,
     ai_targeted,
+    block_ads,
 ):
     """Opens up a browser and fetch content using DynamicFetcher."""
     parsed_headers, _ = _ParseHeaders(extra_headers, False)
@@ -552,6 +560,7 @@ def fetch(
         real_chrome,
         proxy,
         parsed_headers,
+        block_ads,
     )
     from scrapling.fetchers import DynamicFetcher
 
@@ -597,6 +606,7 @@ def stealthy_fetch(
     allow_webgl,
     hide_canvas,
     ai_targeted,
+    block_ads,
 ):
     """Opens up a browser with advanced stealth features and fetch content using StealthyFetcher."""
     parsed_headers, _ = _ParseHeaders(extra_headers, False)
@@ -611,6 +621,7 @@ def stealthy_fetch(
         real_chrome,
         proxy,
         parsed_headers,
+        block_ads,
     )
     kwargs.update(
         {

scrapling/core/_shell_signatures.py

@@ -58,6 +58,7 @@ _FETCH_PARAMS = {
     "cdp_url": Optional[str],
     "useragent": Optional[str],
     "extra_flags": Optional[List[str]],
+    "block_ads": bool,
 }
 
 _STEALTHY_FETCH_PARAMS = {
@@ -84,6 +85,7 @@ _STEALTHY_FETCH_PARAMS = {
     "cdp_url": Optional[str],
     "useragent": Optional[str],
     "extra_flags": Optional[List[str]],
+    "block_ads": bool,
     "allow_webgl": bool,
     "hide_canvas": bool,
     "block_webrtc": bool,

scrapling/engines/_browsers/_types.py

@@ -88,6 +88,7 @@ class PlaywrightSession(TypedDict, total=False):
     useragent: Optional[str]
     extra_flags: Optional[List[str]]
     blocked_domains: Optional[Set[str]]
+    block_ads: bool
     retries: int
     retry_delay: int | float
     capture_xhr: str | None

scrapling/engines/_browsers/_validators.py

@@ -85,6 +85,7 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False, weakref=True):
     useragent: Optional[str] = None
     extra_flags: Optional[List[str]] = None
     blocked_domains: Optional[Set[str]] = None
+    block_ads: bool = False
     retries: RetriesCount = 3
     retry_delay: Seconds = 1
     capture_xhr: str | None = None
@@ -127,6 +128,14 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False, weakref=True):
             if validation_msg:
                 raise ValueError(validation_msg)
 
+        if self.block_ads:
+            from scrapling.engines.toolbelt.ad_domains import AD_DOMAINS
+
+            if self.blocked_domains:
+                self.blocked_domains = self.blocked_domains | set(AD_DOMAINS)
+            else:
+                self.blocked_domains = set(AD_DOMAINS)
+
 
 class StealthConfig(PlaywrightConfig, kw_only=True, frozen=False, weakref=True):
     allow_webgl: bool = True

scrapling/engines/toolbelt/navigation.py

@@ -19,6 +19,27 @@ class ProxyDict(Struct):
     password: str = ""
 
 
+def _is_domain_blocked(hostname: str, domains: frozenset) -> bool:
+    """Check if a hostname matches any blocked domain using O(1) frozenset lookups.
+
+    Walks up the hostname's suffix chain: for "tracker.ads.doubleclick.net",
+    checks "tracker.ads.doubleclick.net", "ads.doubleclick.net", "doubleclick.net".
+
+    :param hostname: The hostname to check.
+    :param domains: A frozenset of blocked domain names.
+    :return: True if the hostname or any of its parent domains is in the blocked set.
+    """
+    if hostname in domains:
+        return True
+    idx = hostname.find(".")
+    while idx != -1:
+        suffix = hostname[idx + 1 :]
+        if "." in suffix and suffix in domains:
+            return True
+        idx = hostname.find(".", idx + 1)
+    return False
+
+
 def create_intercept_handler(disable_resources: bool, blocked_domains: Optional[Set[str]] = None) -> Callable:
     """Create a route handler that blocks both resource types and specific domains.
 
@@ -27,7 +48,7 @@ def create_intercept_handler(disable_resources: bool, blocked_domains: Optional[
     :return: A sync route handler function.
     """
     disabled_resources = EXTRA_RESOURCES if disable_resources else set()
-    domains = blocked_domains or set()
+    domains = frozenset(blocked_domains) if blocked_domains else frozenset()
 
     def handler(route: Route):
         if route.request.resource_type in disabled_resources:
@@ -35,7 +56,7 @@ def create_intercept_handler(disable_resources: bool, blocked_domains: Optional[
             route.abort()
         elif domains:
             hostname = urlparse(route.request.url).hostname or ""
-            if any(hostname == d or hostname.endswith("." + d) for d in domains):
+            if _is_domain_blocked(hostname, domains):
                 log.debug(f'Blocking request to blocked domain "{hostname}" ({route.request.url})')
                 route.abort()
             else:
@@ -54,7 +75,7 @@ def create_async_intercept_handler(disable_resources: bool, blocked_domains: Opt
     :return: An async route handler function.
     """
     disabled_resources = EXTRA_RESOURCES if disable_resources else set()
-    domains = blocked_domains or set()
+    domains = frozenset(blocked_domains) if blocked_domains else frozenset()
 
     async def handler(route: async_Route):
         if route.request.resource_type in disabled_resources:
@@ -62,7 +83,7 @@ def create_async_intercept_handler(disable_resources: bool, blocked_domains: Opt
             await route.abort()
         elif domains:
             hostname = urlparse(route.request.url).hostname or ""
-            if any(hostname == d or hostname.endswith("." + d) for d in domains):
+            if _is_domain_blocked(hostname, domains):
                 log.debug(f'Blocking request to blocked domain "{hostname}" ({route.request.url})')
                 await route.abort()
             else:

scrapling/fetchers/chrome.py

@@ -15,6 +15,7 @@ class DynamicFetcher(BaseFetcher):
         :param headless: Run the browser in headless/hidden (default), or headful/visible mode.
         :param disable_resources: Drop requests for unnecessary resources for a speed boost.
         :param blocked_domains: A set of domain names to block requests to. Subdomains are also matched (e.g., ``"example.com"`` blocks ``"sub.example.com"`` too).
+        :param block_ads: Block requests to ~3,500 known ad/tracking domains. Can be combined with ``blocked_domains``.
         :param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it.
         :param cookies: Set cookies for the next request.
         :param network_idle: Wait for the page until there are no network connections for at least 500 ms.
@@ -55,6 +56,7 @@ class DynamicFetcher(BaseFetcher):
         :param headless: Run the browser in headless/hidden (default), or headful/visible mode.
         :param disable_resources: Drop requests for unnecessary resources for a speed boost.
         :param blocked_domains: A set of domain names to block requests to. Subdomains are also matched (e.g., ``"example.com"`` blocks ``"sub.example.com"`` too).
+        :param block_ads: Block requests to ~3,500 known ad/tracking domains. Can be combined with ``blocked_domains``.
         :param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it.
         :param cookies: Set cookies for the next request.
         :param network_idle: Wait for the page until there are no network connections for at least 500 ms.

scrapling/fetchers/stealth_chrome.py

@@ -20,6 +20,7 @@ class StealthyFetcher(BaseFetcher):
         :param disable_resources: Drop requests for unnecessary resources for a speed boost.
             Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
         :param blocked_domains: A set of domain names to block requests to. Subdomains are also matched (e.g., ``"example.com"`` blocks ``"sub.example.com"`` too).
+        :param block_ads: Block requests to ~3,500 known ad/tracking domains. Can be combined with ``blocked_domains``.
         :param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it.
         :param cookies: Set cookies for the next request.
         :param network_idle: Wait for the page until there are no network connections for at least 500 ms.
@@ -69,6 +70,7 @@ class StealthyFetcher(BaseFetcher):
         :param disable_resources: Drop requests for unnecessary resources for a speed boost.
             Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
         :param blocked_domains: A set of domain names to block requests to. Subdomains are also matched (e.g., ``"example.com"`` blocks ``"sub.example.com"`` too).
+        :param block_ads: Block requests to ~3,500 known ad/tracking domains. Can be combined with ``blocked_domains``.
         :param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it.
         :param cookies: Set cookies for the next request.
         :param network_idle: Wait for the page until there are no network connections for at least 500 ms.