Merge branch 'dev' into fix/fetcher-session-state-corruption

scrapling/engines/_browsers/_base.py

@@ -196,11 +196,14 @@ class SyncSession:
             context_options = self._build_context_with_proxy(proxy)
             context: BrowserContext = self.browser.new_context(**context_options)
 
+            page_info = None
             try:
                 context = self._initialize_context(self._config, context)
                 page_info = self._get_page(timeout, extra_headers, disable_resources, blocked_domains, context=context)
                 yield page_info
             finally:
+                if page_info is not None and page_info in self.page_pool.pages:
+                    self.page_pool.pages.remove(page_info)
                 context.close()
         else:
             # Standard mode: use PagePool with persistent context
@@ -380,6 +383,7 @@ class AsyncSession:
             context_options = self._build_context_with_proxy(proxy)
             context: AsyncBrowserContext = await self.browser.new_context(**context_options)
 
+            page_info = None
             try:
                 context = await self._initialize_context(self._config, context)
                 page_info = await self._get_page(
@@ -387,6 +391,8 @@ class AsyncSession:
                 )
                 yield page_info
             finally:
+                if page_info is not None and page_info in self.page_pool.pages:
+                    self.page_pool.pages.remove(page_info)
                 await context.close()
         else:
             # Standard mode: use PagePool with persistent context
@@ -449,6 +455,13 @@ class BaseSessionMixin:
             if config.extra_flags or extra_flags:
                 flags = list(set(tuple(flags) + tuple(config.extra_flags or extra_flags or ())))
 
+            if config.dns_over_https:
+                doh_flag = "--dns-over-https-templates=https://cloudflare-dns.com/dns-query"
+                if isinstance(flags, list):
+                    flags.append(doh_flag)
+                else:
+                    flags = list(flags) + [doh_flag]
+
             self._browser_options.update(
                 {
                     "args": flags,

scrapling/engines/_browsers/_controllers.py

@@ -47,7 +47,8 @@ class DynamicSession(SyncSession, DynamicSessionMixin):
         :param network_idle: Wait for the page until there are no network connections for at least 500 ms.
         :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
         :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
-        :param page_action: Added for automation. A function that takes the `page` object and does the automation you need.
+        :param page_action: Added for automation. A function that takes the `page` object, runs after navigation, and does the automation you need.
+        :param page_setup: A function that takes the `page` object, runs before navigation. Use it to register event listeners or routes that must be set up before the page loads.
         :param wait_selector: Wait for a specific CSS selector to be in a specific state.
         :param init_script: An absolute path to a JavaScript file to be executed on page creation for all pages in this session.
         :param locale: Specify user locale, for example, `en-GB`, `de-DE`, etc. Locale will affect navigator.language value, Accept-Language request header value as well as number and date formatting
@@ -105,7 +106,8 @@ class DynamicSession(SyncSession, DynamicSessionMixin):
         :param google_search: Enabled by default, Scrapling will set a Google referer header.
         :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
         :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
-        :param page_action: Added for automation. A function that takes the `page` object and does the automation you need.
+        :param page_action: Added for automation. A function that takes the `page` object, runs after navigation, and does the automation you need.
+        :param page_setup: A function that takes the `page` object, runs before navigation. Use it to register event listeners or routes that must be set up before the page loads.
         :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by `google_search` takes priority over the referer set here if used together._
         :param disable_resources: Drop requests for unnecessary resources for a speed boost.
             Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
@@ -152,6 +154,12 @@ class DynamicSession(SyncSession, DynamicSessionMixin):
                     ),
                 )
 
+                if params.page_setup:
+                    try:
+                        params.page_setup(page)
+                    except Exception as e:  # pragma: no cover
+                        log.error(f"Error executing page_setup: {e}")
+
                 try:
                     first_response = page.goto(url, referer=referer)
                     self._wait_for_page_stability(page, params.load_dom, params.network_idle)
@@ -228,7 +236,8 @@ class AsyncDynamicSession(AsyncSession, DynamicSessionMixin):
         :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute.
         :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
         :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
-        :param page_action: Added for automation. A function that takes the `page` object and does the automation you need.
+        :param page_action: Added for automation. A function that takes the `page` object, runs after navigation, and does the automation you need.
+        :param page_setup: A function that takes the `page` object, runs before navigation. Use it to register event listeners or routes that must be set up before the page loads.
         :param wait_selector: Wait for a specific CSS selector to be in a specific state.
         :param init_script: An absolute path to a JavaScript file to be executed on page creation for all pages in this session.
         :param locale: Specify user locale, for example, `en-GB`, `de-DE`, etc. Locale will affect navigator.language value, Accept-Language request header value as well as number and date formatting
@@ -285,7 +294,8 @@ class AsyncDynamicSession(AsyncSession, DynamicSessionMixin):
         :param google_search: Enabled by default, Scrapling will set a Google referer header.
         :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
         :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
-        :param page_action: Added for automation. A function that takes the `page` object and does the automation you need.
+        :param page_action: Added for automation. A function that takes the `page` object, runs after navigation, and does the automation you need.
+        :param page_setup: A function that takes the `page` object, runs before navigation. Use it to register event listeners or routes that must be set up before the page loads.
         :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by `google_search` takes priority over the referer set here if used together._
         :param disable_resources: Drop requests for unnecessary resources for a speed boost.
             Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
@@ -333,6 +343,12 @@ class AsyncDynamicSession(AsyncSession, DynamicSessionMixin):
                     ),
                 )
 
+                if params.page_setup:
+                    try:
+                        await params.page_setup(page)
+                    except Exception as e:  # pragma: no cover
+                        log.error(f"Error executing page_setup: {e}")
+
                 try:
                     first_response = await page.goto(url, referer=referer)
                     await self._wait_for_page_stability(page, params.load_dom, params.network_idle)

scrapling/engines/_browsers/_stealth.py

@@ -47,7 +47,8 @@ class StealthySession(SyncSession, StealthySessionMixin):
         :param network_idle: Wait for the page until there are no network connections for at least 500 ms.
         :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
         :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
-        :param page_action: Added for automation. A function that takes the `page` object and does the automation you need.
+        :param page_action: Added for automation. A function that takes the `page` object, runs after navigation, and does the automation you need.
+        :param page_setup: A function that takes the `page` object, runs before navigation. Use it to register event listeners or routes that must be set up before the page loads.
         :param wait_selector: Wait for a specific CSS selector to be in a specific state.
         :param init_script: An absolute path to a JavaScript file to be executed on page creation for all pages in this session.
         :param locale: Specify user locale, for example, `en-GB`, `de-DE`, etc. Locale will affect navigator.language value, Accept-Language request header value as well as number and date formatting
@@ -187,7 +188,8 @@ class StealthySession(SyncSession, StealthySessionMixin):
         :param google_search: Enabled by default, Scrapling will set a Google referer header.
         :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
         :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
-        :param page_action: Added for automation. A function that takes the `page` object and does the automation you need.
+        :param page_action: Added for automation. A function that takes the `page` object, runs after navigation, and does the automation you need.
+        :param page_setup: A function that takes the `page` object, runs before navigation. Use it to register event listeners or routes that must be set up before the page loads.
         :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by `google_search` takes priority over the referer set here if used together._
         :param disable_resources: Drop requests for unnecessary resources for a speed boost.
             Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
@@ -235,6 +237,12 @@ class StealthySession(SyncSession, StealthySessionMixin):
                     ),
                 )
 
+                if params.page_setup:
+                    try:
+                        params.page_setup(page)
+                    except Exception as e:  # pragma: no cover
+                        log.error(f"Error executing page_setup: {e}")
+
                 try:
                     first_response = page.goto(url, referer=referer)
                     self._wait_for_page_stability(page, params.load_dom, params.network_idle)
@@ -315,7 +323,8 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin):
         :param network_idle: Wait for the page until there are no network connections for at least 500 ms.
         :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
         :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
-        :param page_action: Added for automation. A function that takes the `page` object and does the automation you need.
+        :param page_action: Added for automation. A function that takes the `page` object, runs after navigation, and does the automation you need.
+        :param page_setup: A function that takes the `page` object, runs before navigation. Use it to register event listeners or routes that must be set up before the page loads.
         :param wait_selector: Wait for a specific CSS selector to be in a specific state.
         :param init_script: An absolute path to a JavaScript file to be executed on page creation for all pages in this session.
         :param locale: Specify user locale, for example, `en-GB`, `de-DE`, etc. Locale will affect navigator.language value, Accept-Language request header value as well as number and date formatting
@@ -454,7 +463,8 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin):
         :param google_search: Enabled by default, Scrapling will set a Google referer header.
         :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
         :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
-        :param page_action: Added for automation. A function that takes the `page` object and does the automation you need.
+        :param page_action: Added for automation. A function that takes the `page` object, runs after navigation, and does the automation you need.
+        :param page_setup: A function that takes the `page` object, runs before navigation. Use it to register event listeners or routes that must be set up before the page loads.
         :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by `google_search` takes priority over the referer set here if used together._
         :param disable_resources: Drop requests for unnecessary resources for a speed boost.
             Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
@@ -503,6 +513,12 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin):
                     ),
                 )
 
+                if params.page_setup:
+                    try:
+                        await params.page_setup(page)
+                    except Exception as e:  # pragma: no cover
+                        log.error(f"Error executing page_setup: {e}")
+
                 try:
                     first_response = await page.goto(url, referer=referer)
                     await self._wait_for_page_stability(page, params.load_dom, params.network_idle)

scrapling/engines/_browsers/_types.py

@@ -19,6 +19,7 @@ from scrapling.core._types import (
     TypeAlias,
     SetCookieParam,
     SelectorWaitStates,
+    FollowRedirects,
 )
 from scrapling.engines.toolbelt.proxy_rotation import ProxyRotator
 
@@ -39,7 +40,7 @@ class RequestsSession(TypedDict, total=False):
     headers: Optional[Mapping[str, Optional[str]]]
     retries: Optional[int]
     retry_delay: Optional[int]
-    follow_redirects: Optional[bool]
+    follow_redirects: Optional[FollowRedirects]
     max_redirects: Optional[int]
     verify: Optional[bool]
     cert: Optional[str | Tuple[str, str]]
@@ -73,6 +74,7 @@ class PlaywrightSession(TypedDict, total=False):
     wait: int | float
     timezone_id: str | None
     page_action: Optional[Callable]
+    page_setup: Optional[Callable]
     proxy: Optional[str | Dict[str, str] | Tuple]
     proxy_rotator: Optional[ProxyRotator]
     extra_headers: Optional[Dict[str, str]]
@@ -87,10 +89,12 @@ class PlaywrightSession(TypedDict, total=False):
     useragent: Optional[str]
     extra_flags: Optional[List[str]]
     blocked_domains: Optional[Set[str]]
+    block_ads: bool
     retries: int
     retry_delay: int | float
     capture_xhr: str | None
     executable_path: Optional[str]
+    dns_over_https: bool
 
 
 class PlaywrightFetchParams(TypedDict, total=False):
@@ -102,6 +106,7 @@ class PlaywrightFetchParams(TypedDict, total=False):
     disable_resources: bool
     wait_selector: Optional[str]
     page_action: Optional[Callable]
+    page_setup: Optional[Callable]
     selector_config: Optional[Dict]
     extra_headers: Optional[Dict[str, str]]
     wait_selector_state: SelectorWaitStates

scrapling/engines/_browsers/_validators.py

@@ -53,7 +53,7 @@ def _is_invalid_cdp_url(cdp_url: str) -> bool | str:
 # Type aliases for cleaner annotations
 PagesCount = Annotated[int, Meta(ge=1, le=50)]
 RetriesCount = Annotated[int, Meta(ge=1, le=10)]
-Seconds = Annotated[int, float, Meta(ge=0)]
+Seconds = Annotated[float, Meta(ge=0)]
 
 
 class PlaywrightConfig(Struct, kw_only=True, frozen=False, weakref=True):
@@ -71,6 +71,7 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False, weakref=True):
     wait: Seconds = 0
     timezone_id: str | None = ""
     page_action: Optional[Callable] = None
+    page_setup: Optional[Callable] = None
     proxy: Optional[str | Dict[str, str] | Tuple] = None  # The default value for proxy in Playwright's source is `None`
     proxy_rotator: Optional[ProxyRotator] = None
     extra_headers: Optional[Dict[str, str]] = None
@@ -85,15 +86,19 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False, weakref=True):
     useragent: Optional[str] = None
     extra_flags: Optional[List[str]] = None
     blocked_domains: Optional[Set[str]] = None
+    block_ads: bool = False
     retries: RetriesCount = 3
     retry_delay: Seconds = 1
     capture_xhr: str | None = None
     executable_path: Optional[str] = None
+    dns_over_https: bool = False
 
     def __post_init__(self):  # pragma: no cover
         """Custom validation after msgspec validation"""
         if self.page_action and not callable(self.page_action):
             raise TypeError(f"page_action must be callable, got {type(self.page_action).__name__}")
+        if self.page_setup and not callable(self.page_setup):
+            raise TypeError(f"page_setup must be callable, got {type(self.page_setup).__name__}")
         if self.proxy and self.proxy_rotator:
             raise ValueError(
                 "Cannot use 'proxy_rotator' together with 'proxy'. "
@@ -127,6 +132,14 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False, weakref=True):
             if validation_msg:
                 raise ValueError(validation_msg)
 
+        if self.block_ads:
+            from scrapling.engines.toolbelt.ad_domains import AD_DOMAINS
+
+            if self.blocked_domains:
+                self.blocked_domains = self.blocked_domains | set(AD_DOMAINS)
+            else:
+                self.blocked_domains = set(AD_DOMAINS)
+
 
 class StealthConfig(PlaywrightConfig, kw_only=True, frozen=False, weakref=True):
     allow_webgl: bool = True
@@ -150,6 +163,7 @@ class _fetch_params:
     timeout: Seconds
     wait: Seconds
     page_action: Optional[Callable]
+    page_setup: Optional[Callable]
     extra_headers: Optional[Dict[str, str]]
     disable_resources: bool
     wait_selector: Optional[str]

scrapling/engines/static.py

@@ -20,6 +20,7 @@ from scrapling.core._types import (
     Optional,
     Awaitable,
     SUPPORTED_HTTP_METHODS,
+    FollowRedirects,
 )
 
 from .toolbelt.custom import Response
@@ -77,7 +78,7 @@ class _ConfigurationLogic(ABC):
         self._default_headers = kwargs.get("headers") or {}
         self._default_retries = kwargs.get("retries", 3)
         self._default_retry_delay = kwargs.get("retry_delay", 1)
-        self._default_follow_redirects = kwargs.get("follow_redirects", True)
+        self._default_follow_redirects = kwargs.get("follow_redirects", "safe")
         self._default_max_redirects = kwargs.get("max_redirects", 30)
         self._default_verify = kwargs.get("verify", True)
         self._default_cert = kwargs.get("cert") or None
@@ -250,6 +251,7 @@ class _SyncSessionLogic(_ConfigurationLogic):
                 request_args = self._merge_request_args(stealth=stealth, proxy=proxy, **kwargs)
                 try:
                     response = session.request(method, **request_args)
+                    assert response is not None
                     result = ResponseFactory.from_http_request(response, selector_config, meta={"proxy": proxy})
                     return result
                 except CurlError as e:  # pragma: no cover
@@ -284,7 +286,7 @@ class _SyncSessionLogic(_ConfigurationLogic):
             - headers: Headers to include in the request.
             - cookies: Cookies to use in the request.
             - timeout: Number of seconds to wait before timing out.
-            - follow_redirects: Whether to follow redirects. Defaults to True.
+            - follow_redirects: Whether to follow redirects. Defaults to "safe" (rejects redirects to internal/private IPs).
             - max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited.
             - retries: Number of retry attempts. Defaults to 3.
             - retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second.
@@ -316,7 +318,7 @@ class _SyncSessionLogic(_ConfigurationLogic):
             - headers: Headers to include in the request.
             - cookies: Cookies to use in the request.
             - timeout: Number of seconds to wait before timing out.
-            - follow_redirects: Whether to follow redirects. Defaults to True.
+            - follow_redirects: Whether to follow redirects. Defaults to "safe" (rejects redirects to internal/private IPs).
             - max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited.
             - retries: Number of retry attempts. Defaults to 3.
             - retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second.
@@ -348,7 +350,7 @@ class _SyncSessionLogic(_ConfigurationLogic):
             - headers: Headers to include in the request.
             - cookies: Cookies to use in the request.
             - timeout: Number of seconds to wait before timing out.
-            - follow_redirects: Whether to follow redirects. Defaults to True.
+            - follow_redirects: Whether to follow redirects. Defaults to "safe" (rejects redirects to internal/private IPs).
             - max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited.
             - retries: Number of retry attempts. Defaults to 3.
             - retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second.
@@ -380,7 +382,7 @@ class _SyncSessionLogic(_ConfigurationLogic):
             - headers: Headers to include in the request.
             - cookies: Cookies to use in the request.
             - timeout: Number of seconds to wait before timing out.
-            - follow_redirects: Whether to follow redirects. Defaults to True.
+            - follow_redirects: Whether to follow redirects. Defaults to "safe" (rejects redirects to internal/private IPs).
             - max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited.
             - retries: Number of retry attempts. Defaults to 3.
             - retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second.
@@ -501,7 +503,7 @@ class _ASyncSessionLogic(_ConfigurationLogic):
             - headers: Headers to include in the request.
             - cookies: Cookies to use in the request.
             - timeout: Number of seconds to wait before timing out.
-            - follow_redirects: Whether to follow redirects. Defaults to True.
+            - follow_redirects: Whether to follow redirects. Defaults to "safe" (rejects redirects to internal/private IPs).
             - max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited.
             - retries: Number of retry attempts. Defaults to 3.
             - retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second.
@@ -533,7 +535,7 @@ class _ASyncSessionLogic(_ConfigurationLogic):
             - headers: Headers to include in the request.
             - cookies: Cookies to use in the request.
             - timeout: Number of seconds to wait before timing out.
-            - follow_redirects: Whether to follow redirects. Defaults to True.
+            - follow_redirects: Whether to follow redirects. Defaults to "safe" (rejects redirects to internal/private IPs).
             - max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited.
             - retries: Number of retry attempts. Defaults to 3.
             - retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second.
@@ -565,7 +567,7 @@ class _ASyncSessionLogic(_ConfigurationLogic):
             - headers: Headers to include in the request.
             - cookies: Cookies to use in the request.
             - timeout: Number of seconds to wait before timing out.
-            - follow_redirects: Whether to follow redirects. Defaults to True.
+            - follow_redirects: Whether to follow redirects. Defaults to "safe" (rejects redirects to internal/private IPs).
             - max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited.
             - retries: Number of retry attempts. Defaults to 3.
             - retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second.
@@ -597,7 +599,7 @@ class _ASyncSessionLogic(_ConfigurationLogic):
             - headers: Headers to include in the request.
             - cookies: Cookies to use in the request.
             - timeout: Number of seconds to wait before timing out.
-            - follow_redirects: Whether to follow redirects. Defaults to True.
+            - follow_redirects: Whether to follow redirects. Defaults to "safe" (rejects redirects to internal/private IPs).
             - max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited.
             - retries: Number of retry attempts. Defaults to 3.
             - retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second.
@@ -662,7 +664,7 @@ class FetcherSession:
         headers: Optional[Dict[str, str]] = None,
         retries: Optional[int] = 3,
         retry_delay: Optional[int] = 1,
-        follow_redirects: bool = True,
+        follow_redirects: FollowRedirects = "safe",
         max_redirects: int = 30,
         verify: bool = True,
         cert: Optional[str | Tuple[str, str]] = None,
@@ -681,7 +683,7 @@ class FetcherSession:
         :param headers: Headers to include in the session with every request.
         :param retries: Number of retry attempts. Defaults to 3.
         :param retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second.
-        :param follow_redirects: Whether to follow redirects. Defaults to True.
+        :param follow_redirects: Whether to follow redirects. Defaults to "safe", which follows redirects but rejects those targeting internal/private IPs (SSRF protection). Pass True to follow all redirects without restriction.
         :param max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited.
         :param verify: Whether to verify HTTPS certificates. Defaults to True.
         :param cert: Tuple of (cert, key) filenames for the client certificate.

scrapling/engines/toolbelt/navigation.py

@@ -19,6 +19,27 @@ class ProxyDict(Struct):
     password: str = ""
 
 
+def _is_domain_blocked(hostname: str, domains: frozenset) -> bool:
+    """Check if a hostname matches any blocked domain using O(1) frozenset lookups.
+
+    Walks up the hostname's suffix chain: for "tracker.ads.doubleclick.net",
+    checks "tracker.ads.doubleclick.net", "ads.doubleclick.net", "doubleclick.net".
+
+    :param hostname: The hostname to check.
+    :param domains: A frozenset of blocked domain names.
+    :return: True if the hostname or any of its parent domains is in the blocked set.
+    """
+    if hostname in domains:
+        return True
+    idx = hostname.find(".")
+    while idx != -1:
+        suffix = hostname[idx + 1 :]
+        if "." in suffix and suffix in domains:
+            return True
+        idx = hostname.find(".", idx + 1)
+    return False
+
+
 def create_intercept_handler(disable_resources: bool, blocked_domains: Optional[Set[str]] = None) -> Callable:
     """Create a route handler that blocks both resource types and specific domains.
 
@@ -27,7 +48,7 @@ def create_intercept_handler(disable_resources: bool, blocked_domains: Optional[
     :return: A sync route handler function.
     """
     disabled_resources = EXTRA_RESOURCES if disable_resources else set()
-    domains = blocked_domains or set()
+    domains = frozenset(blocked_domains) if blocked_domains else frozenset()
 
     def handler(route: Route):
         if route.request.resource_type in disabled_resources:
@@ -35,7 +56,7 @@ def create_intercept_handler(disable_resources: bool, blocked_domains: Optional[
             route.abort()
         elif domains:
             hostname = urlparse(route.request.url).hostname or ""
-            if any(hostname == d or hostname.endswith("." + d) for d in domains):
+            if _is_domain_blocked(hostname, domains):
                 log.debug(f'Blocking request to blocked domain "{hostname}" ({route.request.url})')
                 route.abort()
             else:
@@ -54,7 +75,7 @@ def create_async_intercept_handler(disable_resources: bool, blocked_domains: Opt
     :return: An async route handler function.
     """
     disabled_resources = EXTRA_RESOURCES if disable_resources else set()
-    domains = blocked_domains or set()
+    domains = frozenset(blocked_domains) if blocked_domains else frozenset()
 
     async def handler(route: async_Route):
         if route.request.resource_type in disabled_resources:
@@ -62,7 +83,7 @@ def create_async_intercept_handler(disable_resources: bool, blocked_domains: Opt
             await route.abort()
         elif domains:
             hostname = urlparse(route.request.url).hostname or ""
-            if any(hostname == d or hostname.endswith("." + d) for d in domains):
+            if _is_domain_blocked(hostname, domains):
                 log.debug(f'Blocking request to blocked domain "{hostname}" ({route.request.url})')
                 await route.abort()
             else: