Merge branch 'dev' into fix/fetcher-session-state-corruption

scrapling/__init__.py

@@ -1,5 +1,5 @@
 __author__ = "Karim Shoair (karim.shoair@pm.me)"
-__version__ = "0.4.4"
+__version__ = "0.4.7"
 __copyright__ = "Copyright (c) 2024 Karim Shoair"
 
 from typing import Any, TYPE_CHECKING

scrapling/cli.py

@@ -53,6 +53,8 @@ def __Request_and_Save(
     if not output_path.is_absolute():
         output_path = Path.cwd() / output_file
 
+    if ai_targeted:
+        kwargs.setdefault("block_ads", True)
     response = fetcher_func(url, **kwargs)
     Convertor.write_content_to_file(response, str(output_path), css_selector, main_content_only=ai_targeted)
     log.info(f"Content successfully saved to '{output_path}'")
@@ -309,6 +311,16 @@ def _common_browser_options(f):
             default=True,
             help="Run browser in headless mode (default: True)",
         ),
+        option(
+            "--dns-over-https/--no-dns-over-https",
+            default=False,
+            help="Route DNS through Cloudflare's DoH to prevent DNS leaks when using proxies (default: False)",
+        ),
+        option(
+            "--block-ads/--no-block-ads",
+            default=False,
+            help="Block requests to known ad and tracker domains (default: False)",
+        ),
     ]
     for decorator in decorators:
         f = decorator(f)
@@ -498,6 +510,8 @@ def __build_browser_kwargs(
     real_chrome,
     proxy,
     parsed_headers,
+    dns_over_https,
+    block_ads,
 ) -> Dict[str, Any]:
     """Build shared kwargs dict for browser-based commands."""
     kwargs: Dict[str, Any] = {
@@ -507,6 +521,8 @@ def __build_browser_kwargs(
         "timeout": timeout,
         "locale": locale,
         "real_chrome": real_chrome,
+        "dns_over_https": dns_over_https,
+        "block_ads": block_ads,
     }
     if wait > 0:
         kwargs["wait"] = wait
@@ -538,6 +554,8 @@ def fetch(
     proxy,
     extra_headers,
     ai_targeted,
+    dns_over_https,
+    block_ads,
 ):
     """Opens up a browser and fetch content using DynamicFetcher."""
     parsed_headers, _ = _ParseHeaders(extra_headers, False)
@@ -552,6 +570,8 @@ def fetch(
         real_chrome,
         proxy,
         parsed_headers,
+        dns_over_https,
+        block_ads,
     )
     from scrapling.fetchers import DynamicFetcher
 
@@ -597,6 +617,8 @@ def stealthy_fetch(
     allow_webgl,
     hide_canvas,
     ai_targeted,
+    dns_over_https,
+    block_ads,
 ):
     """Opens up a browser with advanced stealth features and fetch content using StealthyFetcher."""
     parsed_headers, _ = _ParseHeaders(extra_headers, False)
@@ -611,6 +633,8 @@ def stealthy_fetch(
         real_chrome,
         proxy,
         parsed_headers,
+        dns_over_https,
+        block_ads,
     )
     kwargs.update(
         {

scrapling/core/_shell_signatures.py

@@ -2,12 +2,14 @@ from scrapling.core._types import (
     Any,
     Dict,
     List,
+    Set,
     Tuple,
     Sequence,
     Callable,
     Optional,
     SetCookieParam,
     SelectorWaitStates,
+    FollowRedirects,
 )
 
 # Parameter definitions for shell function signatures (defined once at module level)
@@ -26,7 +28,7 @@ _REQUESTS_PARAMS = {
     "headers": Any,
     "retries": Optional[int],
     "retry_delay": Optional[int],
-    "follow_redirects": Optional[bool],
+    "follow_redirects": Optional[FollowRedirects],
     "max_redirects": Optional[int],
     "verify": Optional[bool],
     "cert": Optional[str | Tuple[str, str]],
@@ -45,6 +47,7 @@ _FETCH_PARAMS = {
     "wait": int | float,
     "timezone_id": str | None,
     "page_action": Optional[Callable],
+    "page_setup": Optional[Callable],
     "proxy": Optional[str | Dict[str, str] | Tuple],
     "extra_headers": Optional[Dict[str, str]],
     "timeout": int | float,
@@ -57,6 +60,13 @@ _FETCH_PARAMS = {
     "cdp_url": Optional[str],
     "useragent": Optional[str],
     "extra_flags": Optional[List[str]],
+    "blocked_domains": Optional[Set[str]],
+    "block_ads": bool,
+    "retries": int,
+    "retry_delay": int | float,
+    "capture_xhr": str | None,
+    "executable_path": Optional[str],
+    "dns_over_https": bool,
 }
 
 _STEALTHY_FETCH_PARAMS = {
@@ -71,6 +81,7 @@ _STEALTHY_FETCH_PARAMS = {
     "wait": int | float,
     "timezone_id": str | None,
     "page_action": Optional[Callable],
+    "page_setup": Optional[Callable],
     "proxy": Optional[str | Dict[str, str] | Tuple],
     "extra_headers": Optional[Dict[str, str]],
     "timeout": int | float,
@@ -83,6 +94,13 @@ _STEALTHY_FETCH_PARAMS = {
     "cdp_url": Optional[str],
     "useragent": Optional[str],
     "extra_flags": Optional[List[str]],
+    "blocked_domains": Optional[Set[str]],
+    "block_ads": bool,
+    "retries": int,
+    "retry_delay": int | float,
+    "capture_xhr": str | None,
+    "executable_path": Optional[str],
+    "dns_over_https": bool,
     "allow_webgl": bool,
     "hide_canvas": bool,
     "block_webrtc": bool,

scrapling/core/_types.py

@@ -40,6 +40,7 @@ SelectorWaitStates = Literal["attached", "detached", "hidden", "visible"]
 PageLoadStates = Literal["commit", "domcontentloaded", "load", "networkidle"]
 extraction_types = Literal["text", "html", "markdown"]
 StrOrBytes = Union[str, bytes]
+FollowRedirects = Union[bool, Literal["safe", "all", "obeycode", "firstonly"]]
 
 
 # Copied from `playwright._impl._api_structures.SetCookieParam`

scrapling/core/ai.py

@@ -27,6 +27,7 @@ from scrapling.core._types import (
     SetCookieParam,
     extraction_types,
     SelectorWaitStates,
+    FollowRedirects,
 )
 
 SessionType = Literal["dynamic", "stealthy"]
@@ -122,6 +123,7 @@ class ScraplingMCPServer:
     async def open_session(
         self,
         session_type: SessionType,
+        session_id: Optional[str] = None,
         headless: bool = True,
         google_search: bool = True,
         real_chrome: bool = False,
@@ -151,6 +153,7 @@ class ScraplingMCPServer:
         Use close_session to close the session when done, and list_sessions to see all active sessions.
 
         :param session_type: The type of session to open. Use "dynamic" for standard Playwright browser, or "stealthy" for anti-bot bypass with fingerprint spoofing.
+        :param session_id: Optional custom session ID. If not provided, a random 12-character hex ID will be generated. Useful for naming sessions for easier management.
         :param headless: Run the browser in headless/hidden (default), or headful/visible mode.
         :param google_search: Enabled by default, Scrapling will set a Google referer header.
         :param real_chrome: If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch an instance of your browser and use it.
@@ -174,6 +177,12 @@ class ScraplingMCPServer:
         :param solve_cloudflare: (Stealthy only) Solves all types of the Cloudflare's Turnstile/Interstitial challenges.
         :param additional_args: (Stealthy only) Additional arguments to be passed to Playwright's context as additional settings.
         """
+        session_id = session_id or uuid4().hex[:12]
+        if session_id in self._sessions:
+            raise ValueError(
+                f"Session '{session_id}' already exists. Use a different ID or close the existing session first."
+            )
+
         common_kwargs: Dict[str, Any] = dict(
             wait=wait,
             proxy=proxy,
@@ -182,6 +191,7 @@ class ScraplingMCPServer:
             cookies=cookies,
             cdp_url=cdp_url,
             headless=headless,
+            block_ads=True,
             max_pages=max_pages,
             useragent=useragent,
             timezone_id=timezone_id,
@@ -209,7 +219,6 @@ class ScraplingMCPServer:
 
         await session.start()
 
-        session_id = uuid4().hex[:12]
         entry = _SessionEntry(session=session, session_type=session_type)
         self._sessions[session_id] = entry
 
@@ -262,7 +271,7 @@ class ScraplingMCPServer:
         headers: Optional[Mapping[str, Optional[str]]] = None,
         cookies: Optional[Dict[str, str]] = None,
         timeout: Optional[int | float] = 30,
-        follow_redirects: bool = True,
+        follow_redirects: FollowRedirects = "safe",
         max_redirects: int = 30,
         retries: Optional[int] = 3,
         retry_delay: Optional[int] = 1,
@@ -289,7 +298,7 @@ class ScraplingMCPServer:
         :param headers: Headers to include in the request.
         :param cookies: Cookies to use in the request.
         :param timeout: Number of seconds to wait before timing out.
-        :param follow_redirects: Whether to follow redirects. Defaults to True.
+        :param follow_redirects: Whether to follow redirects. Defaults to "safe", which follows redirects but rejects those targeting internal/private IPs (SSRF protection). Pass True to follow all redirects without restriction.
         :param max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited.
         :param retries: Number of retry attempts. Defaults to 3.
         :param retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second.
@@ -335,7 +344,7 @@ class ScraplingMCPServer:
         headers: Optional[Mapping[str, Optional[str]]] = None,
         cookies: Optional[Dict[str, str]] = None,
         timeout: Optional[int | float] = 30,
-        follow_redirects: bool = True,
+        follow_redirects: FollowRedirects = "safe",
         max_redirects: int = 30,
         retries: Optional[int] = 3,
         retry_delay: Optional[int] = 1,
@@ -362,7 +371,7 @@ class ScraplingMCPServer:
         :param headers: Headers to include in the request.
         :param cookies: Cookies to use in the request.
         :param timeout: Number of seconds to wait before timing out.
-        :param follow_redirects: Whether to follow redirects. Defaults to True.
+        :param follow_redirects: Whether to follow redirects. Defaults to "safe", which follows redirects but rejects those targeting internal/private IPs (SSRF protection). Pass True to follow all redirects without restriction.
         :param max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited.
         :param retries: Number of retry attempts. Defaults to 3.
         :param retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second.
@@ -568,6 +577,7 @@ class ScraplingMCPServer:
                 cookies=cookies,
                 cdp_url=cdp_url,
                 headless=headless,
+                block_ads=True,
                 max_pages=len(urls),
                 useragent=useragent,
                 timezone_id=timezone_id,
@@ -776,6 +786,7 @@ class ScraplingMCPServer:
                 timeout=timeout,
                 cookies=cookies,
                 headless=headless,
+                block_ads=True,
                 useragent=useragent,
                 timezone_id=timezone_id,
                 real_chrome=real_chrome,

scrapling/core/mixins.py

@@ -26,7 +26,12 @@ class SelectorsGeneration:
             if target.parent:
                 if target.attrib.get("id"):
                     # id is enough
-                    part = f"#{target.attrib['id']}" if css else f"[@id='{target.attrib['id']}']"
+                    if css:
+                        part = f"#{target.attrib['id']}"
+                    elif full_path:
+                        part = f"*[@id='{target.attrib['id']}']"
+                    else:
+                        part = f"[@id='{target.attrib['id']}']"
                     selectorPath.append(part)
                     if not full_path:
                         return " > ".join(reversed(selectorPath)) if css else "//*" + "/".join(reversed(selectorPath))
@@ -47,7 +52,7 @@ class SelectorsGeneration:
                     if counter[target.tag] > 1:
                         part += f":nth-of-type({counter[target.tag]})" if css else f"[{counter[target.tag]}]"
 
-                selectorPath.append(part)
+                    selectorPath.append(part)
                 target = target.parent
                 if target is None or target.tag == "html":
                     return " > ".join(reversed(selectorPath)) if css else "//" + "/".join(reversed(selectorPath))

scrapling/core/shell.py

@@ -294,7 +294,7 @@ class CurlParser:
             headers=headers,
             cookies=cookies,
             proxy=proxies,
-            follow_redirects=True,  # Scrapling default is True
+            follow_redirects="safe",  # Follows redirects but rejects those to internal/private IPs
         )
 
     def convert2fetcher(self, curl_command: Request | str) -> Optional[Response]:

scrapling/engines/_browsers/_base.py

@@ -196,11 +196,14 @@ class SyncSession:
             context_options = self._build_context_with_proxy(proxy)
             context: BrowserContext = self.browser.new_context(**context_options)
 
+            page_info = None
             try:
                 context = self._initialize_context(self._config, context)
                 page_info = self._get_page(timeout, extra_headers, disable_resources, blocked_domains, context=context)
                 yield page_info
             finally:
+                if page_info is not None and page_info in self.page_pool.pages:
+                    self.page_pool.pages.remove(page_info)
                 context.close()
         else:
             # Standard mode: use PagePool with persistent context
@@ -380,6 +383,7 @@ class AsyncSession:
             context_options = self._build_context_with_proxy(proxy)
             context: AsyncBrowserContext = await self.browser.new_context(**context_options)
 
+            page_info = None
             try:
                 context = await self._initialize_context(self._config, context)
                 page_info = await self._get_page(
@@ -387,6 +391,8 @@ class AsyncSession:
                 )
                 yield page_info
             finally:
+                if page_info is not None and page_info in self.page_pool.pages:
+                    self.page_pool.pages.remove(page_info)
                 await context.close()
         else:
             # Standard mode: use PagePool with persistent context
@@ -449,6 +455,13 @@ class BaseSessionMixin:
             if config.extra_flags or extra_flags:
                 flags = list(set(tuple(flags) + tuple(config.extra_flags or extra_flags or ())))
 
+            if config.dns_over_https:
+                doh_flag = "--dns-over-https-templates=https://cloudflare-dns.com/dns-query"
+                if isinstance(flags, list):
+                    flags.append(doh_flag)
+                else:
+                    flags = list(flags) + [doh_flag]
+
             self._browser_options.update(
                 {
                     "args": flags,

scrapling/engines/_browsers/_controllers.py

@@ -47,7 +47,8 @@ class DynamicSession(SyncSession, DynamicSessionMixin):
         :param network_idle: Wait for the page until there are no network connections for at least 500 ms.
         :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
         :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
-        :param page_action: Added for automation. A function that takes the `page` object and does the automation you need.
+        :param page_action: Added for automation. A function that takes the `page` object, runs after navigation, and does the automation you need.
+        :param page_setup: A function that takes the `page` object, runs before navigation. Use it to register event listeners or routes that must be set up before the page loads.
         :param wait_selector: Wait for a specific CSS selector to be in a specific state.
         :param init_script: An absolute path to a JavaScript file to be executed on page creation for all pages in this session.
         :param locale: Specify user locale, for example, `en-GB`, `de-DE`, etc. Locale will affect navigator.language value, Accept-Language request header value as well as number and date formatting
@@ -105,7 +106,8 @@ class DynamicSession(SyncSession, DynamicSessionMixin):
         :param google_search: Enabled by default, Scrapling will set a Google referer header.
         :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
         :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
-        :param page_action: Added for automation. A function that takes the `page` object and does the automation you need.
+        :param page_action: Added for automation. A function that takes the `page` object, runs after navigation, and does the automation you need.
+        :param page_setup: A function that takes the `page` object, runs before navigation. Use it to register event listeners or routes that must be set up before the page loads.
         :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by `google_search` takes priority over the referer set here if used together._
         :param disable_resources: Drop requests for unnecessary resources for a speed boost.
             Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
@@ -152,6 +154,12 @@ class DynamicSession(SyncSession, DynamicSessionMixin):
                     ),
                 )
 
+                if params.page_setup:
+                    try:
+                        params.page_setup(page)
+                    except Exception as e:  # pragma: no cover
+                        log.error(f"Error executing page_setup: {e}")
+
                 try:
                     first_response = page.goto(url, referer=referer)
                     self._wait_for_page_stability(page, params.load_dom, params.network_idle)
@@ -228,7 +236,8 @@ class AsyncDynamicSession(AsyncSession, DynamicSessionMixin):
         :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute.
         :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
         :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
-        :param page_action: Added for automation. A function that takes the `page` object and does the automation you need.
+        :param page_action: Added for automation. A function that takes the `page` object, runs after navigation, and does the automation you need.
+        :param page_setup: A function that takes the `page` object, runs before navigation. Use it to register event listeners or routes that must be set up before the page loads.
         :param wait_selector: Wait for a specific CSS selector to be in a specific state.
         :param init_script: An absolute path to a JavaScript file to be executed on page creation for all pages in this session.
         :param locale: Specify user locale, for example, `en-GB`, `de-DE`, etc. Locale will affect navigator.language value, Accept-Language request header value as well as number and date formatting
@@ -285,7 +294,8 @@ class AsyncDynamicSession(AsyncSession, DynamicSessionMixin):
         :param google_search: Enabled by default, Scrapling will set a Google referer header.
         :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
         :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
-        :param page_action: Added for automation. A function that takes the `page` object and does the automation you need.
+        :param page_action: Added for automation. A function that takes the `page` object, runs after navigation, and does the automation you need.
+        :param page_setup: A function that takes the `page` object, runs before navigation. Use it to register event listeners or routes that must be set up before the page loads.
         :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by `google_search` takes priority over the referer set here if used together._
         :param disable_resources: Drop requests for unnecessary resources for a speed boost.
             Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
@@ -333,6 +343,12 @@ class AsyncDynamicSession(AsyncSession, DynamicSessionMixin):
                     ),
                 )
 
+                if params.page_setup:
+                    try:
+                        await params.page_setup(page)
+                    except Exception as e:  # pragma: no cover
+                        log.error(f"Error executing page_setup: {e}")
+
                 try:
                     first_response = await page.goto(url, referer=referer)
                     await self._wait_for_page_stability(page, params.load_dom, params.network_idle)

scrapling/engines/_browsers/_stealth.py

@@ -47,7 +47,8 @@ class StealthySession(SyncSession, StealthySessionMixin):
         :param network_idle: Wait for the page until there are no network connections for at least 500 ms.
         :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
         :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
-        :param page_action: Added for automation. A function that takes the `page` object and does the automation you need.
+        :param page_action: Added for automation. A function that takes the `page` object, runs after navigation, and does the automation you need.
+        :param page_setup: A function that takes the `page` object, runs before navigation. Use it to register event listeners or routes that must be set up before the page loads.
         :param wait_selector: Wait for a specific CSS selector to be in a specific state.
         :param init_script: An absolute path to a JavaScript file to be executed on page creation for all pages in this session.
         :param locale: Specify user locale, for example, `en-GB`, `de-DE`, etc. Locale will affect navigator.language value, Accept-Language request header value as well as number and date formatting
@@ -187,7 +188,8 @@ class StealthySession(SyncSession, StealthySessionMixin):
         :param google_search: Enabled by default, Scrapling will set a Google referer header.
         :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
         :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
-        :param page_action: Added for automation. A function that takes the `page` object and does the automation you need.
+        :param page_action: Added for automation. A function that takes the `page` object, runs after navigation, and does the automation you need.
+        :param page_setup: A function that takes the `page` object, runs before navigation. Use it to register event listeners or routes that must be set up before the page loads.
         :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by `google_search` takes priority over the referer set here if used together._
         :param disable_resources: Drop requests for unnecessary resources for a speed boost.
             Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
@@ -235,6 +237,12 @@ class StealthySession(SyncSession, StealthySessionMixin):
                     ),
                 )
 
+                if params.page_setup:
+                    try:
+                        params.page_setup(page)
+                    except Exception as e:  # pragma: no cover
+                        log.error(f"Error executing page_setup: {e}")
+
                 try:
                     first_response = page.goto(url, referer=referer)
                     self._wait_for_page_stability(page, params.load_dom, params.network_idle)
@@ -315,7 +323,8 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin):
         :param network_idle: Wait for the page until there are no network connections for at least 500 ms.
         :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
         :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
-        :param page_action: Added for automation. A function that takes the `page` object and does the automation you need.
+        :param page_action: Added for automation. A function that takes the `page` object, runs after navigation, and does the automation you need.
+        :param page_setup: A function that takes the `page` object, runs before navigation. Use it to register event listeners or routes that must be set up before the page loads.
         :param wait_selector: Wait for a specific CSS selector to be in a specific state.
         :param init_script: An absolute path to a JavaScript file to be executed on page creation for all pages in this session.
         :param locale: Specify user locale, for example, `en-GB`, `de-DE`, etc. Locale will affect navigator.language value, Accept-Language request header value as well as number and date formatting
@@ -454,7 +463,8 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin):
         :param google_search: Enabled by default, Scrapling will set a Google referer header.
         :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
         :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
-        :param page_action: Added for automation. A function that takes the `page` object and does the automation you need.
+        :param page_action: Added for automation. A function that takes the `page` object, runs after navigation, and does the automation you need.
+        :param page_setup: A function that takes the `page` object, runs before navigation. Use it to register event listeners or routes that must be set up before the page loads.
         :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by `google_search` takes priority over the referer set here if used together._
         :param disable_resources: Drop requests for unnecessary resources for a speed boost.
             Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
@@ -503,6 +513,12 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin):
                     ),
                 )
 
+                if params.page_setup:
+                    try:
+                        await params.page_setup(page)
+                    except Exception as e:  # pragma: no cover
+                        log.error(f"Error executing page_setup: {e}")
+
                 try:
                     first_response = await page.goto(url, referer=referer)
                     await self._wait_for_page_stability(page, params.load_dom, params.network_idle)

scrapling/engines/_browsers/_types.py

@@ -19,6 +19,7 @@ from scrapling.core._types import (
     TypeAlias,
     SetCookieParam,
     SelectorWaitStates,
+    FollowRedirects,
 )
 from scrapling.engines.toolbelt.proxy_rotation import ProxyRotator
 
@@ -39,7 +40,7 @@ class RequestsSession(TypedDict, total=False):
     headers: Optional[Mapping[str, Optional[str]]]
     retries: Optional[int]
     retry_delay: Optional[int]
-    follow_redirects: Optional[bool]
+    follow_redirects: Optional[FollowRedirects]
     max_redirects: Optional[int]
     verify: Optional[bool]
     cert: Optional[str | Tuple[str, str]]
@@ -73,6 +74,7 @@ class PlaywrightSession(TypedDict, total=False):
     wait: int | float
     timezone_id: str | None
     page_action: Optional[Callable]
+    page_setup: Optional[Callable]
     proxy: Optional[str | Dict[str, str] | Tuple]
     proxy_rotator: Optional[ProxyRotator]
     extra_headers: Optional[Dict[str, str]]
@@ -87,10 +89,12 @@ class PlaywrightSession(TypedDict, total=False):
     useragent: Optional[str]
     extra_flags: Optional[List[str]]
     blocked_domains: Optional[Set[str]]
+    block_ads: bool
     retries: int
     retry_delay: int | float
     capture_xhr: str | None
     executable_path: Optional[str]
+    dns_over_https: bool
 
 
 class PlaywrightFetchParams(TypedDict, total=False):
@@ -102,6 +106,7 @@ class PlaywrightFetchParams(TypedDict, total=False):
     disable_resources: bool
     wait_selector: Optional[str]
     page_action: Optional[Callable]
+    page_setup: Optional[Callable]
     selector_config: Optional[Dict]
     extra_headers: Optional[Dict[str, str]]
     wait_selector_state: SelectorWaitStates

scrapling/engines/_browsers/_validators.py

@@ -53,7 +53,7 @@ def _is_invalid_cdp_url(cdp_url: str) -> bool | str:
 # Type aliases for cleaner annotations
 PagesCount = Annotated[int, Meta(ge=1, le=50)]
 RetriesCount = Annotated[int, Meta(ge=1, le=10)]
-Seconds = Annotated[int, float, Meta(ge=0)]
+Seconds = Annotated[float, Meta(ge=0)]
 
 
 class PlaywrightConfig(Struct, kw_only=True, frozen=False, weakref=True):
@@ -71,6 +71,7 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False, weakref=True):
     wait: Seconds = 0
     timezone_id: str | None = ""
     page_action: Optional[Callable] = None
+    page_setup: Optional[Callable] = None
     proxy: Optional[str | Dict[str, str] | Tuple] = None  # The default value for proxy in Playwright's source is `None`
     proxy_rotator: Optional[ProxyRotator] = None
     extra_headers: Optional[Dict[str, str]] = None
@@ -85,15 +86,19 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False, weakref=True):
     useragent: Optional[str] = None
     extra_flags: Optional[List[str]] = None
     blocked_domains: Optional[Set[str]] = None
+    block_ads: bool = False
     retries: RetriesCount = 3
     retry_delay: Seconds = 1
     capture_xhr: str | None = None
     executable_path: Optional[str] = None
+    dns_over_https: bool = False
 
     def __post_init__(self):  # pragma: no cover
         """Custom validation after msgspec validation"""
         if self.page_action and not callable(self.page_action):
             raise TypeError(f"page_action must be callable, got {type(self.page_action).__name__}")
+        if self.page_setup and not callable(self.page_setup):
+            raise TypeError(f"page_setup must be callable, got {type(self.page_setup).__name__}")
         if self.proxy and self.proxy_rotator:
             raise ValueError(
                 "Cannot use 'proxy_rotator' together with 'proxy'. "
@@ -127,6 +132,14 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False, weakref=True):
             if validation_msg:
                 raise ValueError(validation_msg)
 
+        if self.block_ads:
+            from scrapling.engines.toolbelt.ad_domains import AD_DOMAINS
+
+            if self.blocked_domains:
+                self.blocked_domains = self.blocked_domains | set(AD_DOMAINS)
+            else:
+                self.blocked_domains = set(AD_DOMAINS)
+
 
 class StealthConfig(PlaywrightConfig, kw_only=True, frozen=False, weakref=True):
     allow_webgl: bool = True
@@ -150,6 +163,7 @@ class _fetch_params:
     timeout: Seconds
     wait: Seconds
     page_action: Optional[Callable]
+    page_setup: Optional[Callable]
     extra_headers: Optional[Dict[str, str]]
     disable_resources: bool
     wait_selector: Optional[str]

scrapling/engines/static.py

@@ -20,6 +20,7 @@ from scrapling.core._types import (
     Optional,
     Awaitable,
     SUPPORTED_HTTP_METHODS,
+    FollowRedirects,
 )
 
 from .toolbelt.custom import Response
@@ -77,7 +78,7 @@ class _ConfigurationLogic(ABC):
         self._default_headers = kwargs.get("headers") or {}
         self._default_retries = kwargs.get("retries", 3)
         self._default_retry_delay = kwargs.get("retry_delay", 1)
-        self._default_follow_redirects = kwargs.get("follow_redirects", True)
+        self._default_follow_redirects = kwargs.get("follow_redirects", "safe")
         self._default_max_redirects = kwargs.get("max_redirects", 30)
         self._default_verify = kwargs.get("verify", True)
         self._default_cert = kwargs.get("cert") or None
@@ -250,6 +251,7 @@ class _SyncSessionLogic(_ConfigurationLogic):
                 request_args = self._merge_request_args(stealth=stealth, proxy=proxy, **kwargs)
                 try:
                     response = session.request(method, **request_args)
+                    assert response is not None
                     result = ResponseFactory.from_http_request(response, selector_config, meta={"proxy": proxy})
                     return result
                 except CurlError as e:  # pragma: no cover
@@ -284,7 +286,7 @@ class _SyncSessionLogic(_ConfigurationLogic):
             - headers: Headers to include in the request.
             - cookies: Cookies to use in the request.
             - timeout: Number of seconds to wait before timing out.
-            - follow_redirects: Whether to follow redirects. Defaults to True.
+            - follow_redirects: Whether to follow redirects. Defaults to "safe" (rejects redirects to internal/private IPs).
             - max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited.
             - retries: Number of retry attempts. Defaults to 3.
             - retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second.
@@ -316,7 +318,7 @@ class _SyncSessionLogic(_ConfigurationLogic):
             - headers: Headers to include in the request.
             - cookies: Cookies to use in the request.
             - timeout: Number of seconds to wait before timing out.
-            - follow_redirects: Whether to follow redirects. Defaults to True.
+            - follow_redirects: Whether to follow redirects. Defaults to "safe" (rejects redirects to internal/private IPs).
             - max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited.
             - retries: Number of retry attempts. Defaults to 3.
             - retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second.
@@ -348,7 +350,7 @@ class _SyncSessionLogic(_ConfigurationLogic):
             - headers: Headers to include in the request.
             - cookies: Cookies to use in the request.
             - timeout: Number of seconds to wait before timing out.
-            - follow_redirects: Whether to follow redirects. Defaults to True.
+            - follow_redirects: Whether to follow redirects. Defaults to "safe" (rejects redirects to internal/private IPs).
             - max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited.
             - retries: Number of retry attempts. Defaults to 3.
             - retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second.
@@ -380,7 +382,7 @@ class _SyncSessionLogic(_ConfigurationLogic):
             - headers: Headers to include in the request.
             - cookies: Cookies to use in the request.
             - timeout: Number of seconds to wait before timing out.
-            - follow_redirects: Whether to follow redirects. Defaults to True.
+            - follow_redirects: Whether to follow redirects. Defaults to "safe" (rejects redirects to internal/private IPs).
             - max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited.
             - retries: Number of retry attempts. Defaults to 3.
             - retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second.
@@ -501,7 +503,7 @@ class _ASyncSessionLogic(_ConfigurationLogic):
             - headers: Headers to include in the request.
             - cookies: Cookies to use in the request.
             - timeout: Number of seconds to wait before timing out.
-            - follow_redirects: Whether to follow redirects. Defaults to True.
+            - follow_redirects: Whether to follow redirects. Defaults to "safe" (rejects redirects to internal/private IPs).
             - max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited.
             - retries: Number of retry attempts. Defaults to 3.
             - retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second.
@@ -533,7 +535,7 @@ class _ASyncSessionLogic(_ConfigurationLogic):
             - headers: Headers to include in the request.
             - cookies: Cookies to use in the request.
             - timeout: Number of seconds to wait before timing out.
-            - follow_redirects: Whether to follow redirects. Defaults to True.
+            - follow_redirects: Whether to follow redirects. Defaults to "safe" (rejects redirects to internal/private IPs).
             - max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited.
             - retries: Number of retry attempts. Defaults to 3.
             - retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second.
@@ -565,7 +567,7 @@ class _ASyncSessionLogic(_ConfigurationLogic):
             - headers: Headers to include in the request.
             - cookies: Cookies to use in the request.
             - timeout: Number of seconds to wait before timing out.
-            - follow_redirects: Whether to follow redirects. Defaults to True.
+            - follow_redirects: Whether to follow redirects. Defaults to "safe" (rejects redirects to internal/private IPs).
             - max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited.
             - retries: Number of retry attempts. Defaults to 3.
             - retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second.
@@ -597,7 +599,7 @@ class _ASyncSessionLogic(_ConfigurationLogic):
             - headers: Headers to include in the request.
             - cookies: Cookies to use in the request.
             - timeout: Number of seconds to wait before timing out.
-            - follow_redirects: Whether to follow redirects. Defaults to True.
+            - follow_redirects: Whether to follow redirects. Defaults to "safe" (rejects redirects to internal/private IPs).
             - max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited.
             - retries: Number of retry attempts. Defaults to 3.
             - retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second.
@@ -662,7 +664,7 @@ class FetcherSession:
         headers: Optional[Dict[str, str]] = None,
         retries: Optional[int] = 3,
         retry_delay: Optional[int] = 1,
-        follow_redirects: bool = True,
+        follow_redirects: FollowRedirects = "safe",
         max_redirects: int = 30,
         verify: bool = True,
         cert: Optional[str | Tuple[str, str]] = None,
@@ -681,7 +683,7 @@ class FetcherSession:
         :param headers: Headers to include in the session with every request.
         :param retries: Number of retry attempts. Defaults to 3.
         :param retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second.
-        :param follow_redirects: Whether to follow redirects. Defaults to True.
+        :param follow_redirects: Whether to follow redirects. Defaults to "safe", which follows redirects but rejects those targeting internal/private IPs (SSRF protection). Pass True to follow all redirects without restriction.
         :param max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited.
         :param verify: Whether to verify HTTPS certificates. Defaults to True.
         :param cert: Tuple of (cert, key) filenames for the client certificate.

scrapling/engines/toolbelt/navigation.py

@@ -19,6 +19,27 @@ class ProxyDict(Struct):
     password: str = ""
 
 
+def _is_domain_blocked(hostname: str, domains: frozenset) -> bool:
+    """Check if a hostname matches any blocked domain using O(1) frozenset lookups.
+
+    Walks up the hostname's suffix chain: for "tracker.ads.doubleclick.net",
+    checks "tracker.ads.doubleclick.net", "ads.doubleclick.net", "doubleclick.net".
+
+    :param hostname: The hostname to check.
+    :param domains: A frozenset of blocked domain names.
+    :return: True if the hostname or any of its parent domains is in the blocked set.
+    """
+    if hostname in domains:
+        return True
+    idx = hostname.find(".")
+    while idx != -1:
+        suffix = hostname[idx + 1 :]
+        if "." in suffix and suffix in domains:
+            return True
+        idx = hostname.find(".", idx + 1)
+    return False
+
+
 def create_intercept_handler(disable_resources: bool, blocked_domains: Optional[Set[str]] = None) -> Callable:
     """Create a route handler that blocks both resource types and specific domains.
 
@@ -27,7 +48,7 @@ def create_intercept_handler(disable_resources: bool, blocked_domains: Optional[
     :return: A sync route handler function.
     """
     disabled_resources = EXTRA_RESOURCES if disable_resources else set()
-    domains = blocked_domains or set()
+    domains = frozenset(blocked_domains) if blocked_domains else frozenset()
 
     def handler(route: Route):
         if route.request.resource_type in disabled_resources:
@@ -35,7 +56,7 @@ def create_intercept_handler(disable_resources: bool, blocked_domains: Optional[
             route.abort()
         elif domains:
             hostname = urlparse(route.request.url).hostname or ""
-            if any(hostname == d or hostname.endswith("." + d) for d in domains):
+            if _is_domain_blocked(hostname, domains):
                 log.debug(f'Blocking request to blocked domain "{hostname}" ({route.request.url})')
                 route.abort()
             else:
@@ -54,7 +75,7 @@ def create_async_intercept_handler(disable_resources: bool, blocked_domains: Opt
     :return: An async route handler function.
     """
     disabled_resources = EXTRA_RESOURCES if disable_resources else set()
-    domains = blocked_domains or set()
+    domains = frozenset(blocked_domains) if blocked_domains else frozenset()
 
     async def handler(route: async_Route):
         if route.request.resource_type in disabled_resources:
@@ -62,7 +83,7 @@ def create_async_intercept_handler(disable_resources: bool, blocked_domains: Opt
             await route.abort()
         elif domains:
             hostname = urlparse(route.request.url).hostname or ""
-            if any(hostname == d or hostname.endswith("." + d) for d in domains):
+            if _is_domain_blocked(hostname, domains):
                 log.debug(f'Blocking request to blocked domain "{hostname}" ({route.request.url})')
                 await route.abort()
             else:

scrapling/fetchers/chrome.py

@@ -15,13 +15,16 @@ class DynamicFetcher(BaseFetcher):
         :param headless: Run the browser in headless/hidden (default), or headful/visible mode.
         :param disable_resources: Drop requests for unnecessary resources for a speed boost.
         :param blocked_domains: A set of domain names to block requests to. Subdomains are also matched (e.g., ``"example.com"`` blocks ``"sub.example.com"`` too).
+        :param block_ads: Block requests to ~3,500 known ad/tracking domains. Can be combined with ``blocked_domains``.
+        :param dns_over_https: Route DNS queries through Cloudflare's DNS-over-HTTPS to prevent DNS leaks when using proxies.
         :param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it.
         :param cookies: Set cookies for the next request.
         :param network_idle: Wait for the page until there are no network connections for at least 500 ms.
         :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute.
         :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
         :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the Response object.
-        :param page_action: Added for automation. A function that takes the `page` object and does the automation you need.
+        :param page_action: Added for automation. A function that takes the `page` object, runs after navigation, and does the automation you need.
+        :param page_setup: A function that takes the `page` object, runs before navigation. Use it to register event listeners or routes that must be set up before the page loads.
         :param wait_selector: Wait for a specific CSS selector to be in a specific state.
         :param init_script: An absolute path to a JavaScript file to be executed on page creation with this request.
         :param locale: Set the locale for the browser if wanted. Defaults to the system default locale.
@@ -55,13 +58,16 @@ class DynamicFetcher(BaseFetcher):
         :param headless: Run the browser in headless/hidden (default), or headful/visible mode.
         :param disable_resources: Drop requests for unnecessary resources for a speed boost.
         :param blocked_domains: A set of domain names to block requests to. Subdomains are also matched (e.g., ``"example.com"`` blocks ``"sub.example.com"`` too).
+        :param block_ads: Block requests to ~3,500 known ad/tracking domains. Can be combined with ``blocked_domains``.
+        :param dns_over_https: Route DNS queries through Cloudflare's DNS-over-HTTPS to prevent DNS leaks when using proxies.
         :param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it.
         :param cookies: Set cookies for the next request.
         :param network_idle: Wait for the page until there are no network connections for at least 500 ms.
         :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute.
         :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
         :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the Response object.
-        :param page_action: Added for automation. A function that takes the `page` object and does the automation you need.
+        :param page_action: Added for automation. A function that takes the `page` object, runs after navigation, and does the automation you need.
+        :param page_setup: A function that takes the `page` object, runs before navigation. Use it to register event listeners or routes that must be set up before the page loads.
         :param wait_selector: Wait for a specific CSS selector to be in a specific state.
         :param init_script: An absolute path to a JavaScript file to be executed on page creation with this request.
         :param locale: Set the locale for the browser if wanted. Defaults to the system default locale.

scrapling/fetchers/stealth_chrome.py

@@ -20,12 +20,15 @@ class StealthyFetcher(BaseFetcher):
         :param disable_resources: Drop requests for unnecessary resources for a speed boost.
             Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
         :param blocked_domains: A set of domain names to block requests to. Subdomains are also matched (e.g., ``"example.com"`` blocks ``"sub.example.com"`` too).
+        :param block_ads: Block requests to ~3,500 known ad/tracking domains. Can be combined with ``blocked_domains``.
+        :param dns_over_https: Route DNS queries through Cloudflare's DNS-over-HTTPS to prevent DNS leaks when using proxies.
         :param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it.
         :param cookies: Set cookies for the next request.
         :param network_idle: Wait for the page until there are no network connections for at least 500 ms.
         :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
         :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
-        :param page_action: Added for automation. A function that takes the `page` object and does the automation you need.
+        :param page_action: Added for automation. A function that takes the `page` object, runs after navigation, and does the automation you need.
+        :param page_setup: A function that takes the `page` object, runs before navigation. Use it to register event listeners or routes that must be set up before the page loads.
         :param wait_selector: Wait for a specific CSS selector to be in a specific state.
         :param init_script: An absolute path to a JavaScript file to be executed on page creation for all pages in this session.
         :param locale: Specify user locale, for example, `en-GB`, `de-DE`, etc. Locale will affect navigator.language value, Accept-Language request header value as well as number and date formatting
@@ -69,12 +72,15 @@ class StealthyFetcher(BaseFetcher):
         :param disable_resources: Drop requests for unnecessary resources for a speed boost.
             Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
         :param blocked_domains: A set of domain names to block requests to. Subdomains are also matched (e.g., ``"example.com"`` blocks ``"sub.example.com"`` too).
+        :param block_ads: Block requests to ~3,500 known ad/tracking domains. Can be combined with ``blocked_domains``.
+        :param dns_over_https: Route DNS queries through Cloudflare's DNS-over-HTTPS to prevent DNS leaks when using proxies.
         :param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it.
         :param cookies: Set cookies for the next request.
         :param network_idle: Wait for the page until there are no network connections for at least 500 ms.
         :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
         :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
-        :param page_action: Added for automation. A function that takes the `page` object and does the automation you need.
+        :param page_action: Added for automation. A function that takes the `page` object, runs after navigation, and does the automation you need.
+        :param page_setup: A function that takes the `page` object, runs before navigation. Use it to register event listeners or routes that must be set up before the page loads.
         :param wait_selector: Wait for a specific CSS selector to be in a specific state.
         :param init_script: An absolute path to a JavaScript file to be executed on page creation for all pages in this session.
         :param locale: Specify user locale, for example, `en-GB`, `de-DE`, etc. Locale will affect navigator.language value, Accept-Language request header value as well as number and date formatting

scrapling/spiders/cache.py

@@ -0,0 +1,79 @@
+from base64 import b64encode, b64decode
+from pathlib import Path
+
+import orjson
+import anyio
+from anyio import Path as AsyncPath
+
+from scrapling.core.utils import log
+from scrapling.core._types import Dict, Optional, Any
+from scrapling.engines.toolbelt.custom import Response
+
+
+class ResponseCacheManager:
+    """Caches HTTP responses to disk for replay during spider development."""
+
+    def __init__(self, cache_dir: str | Path):
+        self._cache_dir = AsyncPath(cache_dir)
+
+    def _cache_path(self, fingerprint: bytes) -> AsyncPath:
+        return self._cache_dir / f"{fingerprint.hex()}.json"
+
+    async def get(self, fingerprint: bytes) -> Optional[Response]:
+        path = self._cache_path(fingerprint)
+        if not await path.exists():
+            return None
+
+        try:
+            async with await anyio.open_file(path, "rb") as f:
+                data: Dict[str, Any] = orjson.loads(await f.read())
+
+            return Response(
+                url=data["url"],
+                content=b64decode(data["content"]),
+                status=data["status"],
+                reason=data["reason"],
+                encoding=data["encoding"],
+                cookies=data["cookies"],
+                headers=data["headers"],
+                request_headers=data["request_headers"],
+                method=data["method"],
+            )
+        except Exception as e:
+            log.warning(f"Failed to read cached response for {fingerprint.hex()}: {e}")
+            return None
+
+    async def put(self, fingerprint: bytes, response: Response, method: str = "GET") -> None:
+        await self._cache_dir.mkdir(parents=True, exist_ok=True)
+        temp_path = self._cache_path(fingerprint).with_suffix(".tmp")
+
+        try:
+            serialized = orjson.dumps(
+                {
+                    "url": response.url,
+                    "content": b64encode(response.body).decode("ascii"),
+                    "status": response.status,
+                    "reason": response.reason,
+                    "encoding": response.encoding,
+                    "cookies": dict(response.cookies) if isinstance(response.cookies, dict) else {},
+                    "headers": dict(response.headers),
+                    "request_headers": dict(response.request_headers),
+                    "method": method,
+                }
+            )
+            async with await anyio.open_file(temp_path, "wb") as f:
+                await f.write(serialized)
+
+            await temp_path.rename(self._cache_path(fingerprint))
+        except Exception as e:
+            if await temp_path.exists():
+                await temp_path.unlink()
+            log.warning(f"Failed to cache response for {fingerprint.hex()}: {e}")
+
+    async def clear(self) -> None:
+        if not await self._cache_dir.exists():
+            return
+        async for entry in self._cache_dir.iterdir():
+            if entry.suffix == ".json":
+                await entry.unlink()
+        log.info(f"Cleared response cache at {self._cache_dir}")

scrapling/spiders/engine.py

@@ -1,16 +1,19 @@
 import json
 import pprint
 from pathlib import Path
+from urllib.parse import urlparse
 
 import anyio
 from anyio import Path as AsyncPath
 from anyio import create_task_group, CapacityLimiter, create_memory_object_stream, EndOfStream
 
 from scrapling.core.utils import log
-from scrapling.spiders.request import Request
 from scrapling.spiders.scheduler import Scheduler
 from scrapling.spiders.session import SessionManager
+from scrapling.spiders.request import Request, Response
+from scrapling.spiders.robotstxt import RobotsTxtManager
 from scrapling.spiders.result import CrawlStats, ItemList
+from scrapling.spiders.cache import ResponseCacheManager
 from scrapling.spiders.checkpoint import CheckpointManager, CheckpointData
 from scrapling.core._types import Dict, Union, Optional, TYPE_CHECKING, Any, AsyncGenerator
 
@@ -41,10 +44,29 @@ class CrawlerEngine:
         )
         self.stats = CrawlStats()
 
+        if self.spider.robots_txt_obey:
+
+            async def _fetch_robots(url: str, sid: str) -> Response:
+                return await self.session_manager.fetch(Request(url, sid=sid))
+
+            self._robots_manager: Optional[RobotsTxtManager] = RobotsTxtManager(_fetch_robots)
+        else:
+            self._robots_manager = None
+
+        if self.spider.development_mode:
+            cache_dir = self.spider.development_cache_dir or f".scrapling_cache/{self.spider.name}"
+            self._cache_manager: Optional[ResponseCacheManager] = ResponseCacheManager(cache_dir)
+            log.warning("Development mode enabled -- responses will be cached to disk and replayed on subsequent runs")
+        else:
+            self._cache_manager = None
+
         self._global_limiter = CapacityLimiter(spider.concurrent_requests)
         self._domain_limiters: dict[str, CapacityLimiter] = {}
         self._allowed_domains: set[str] = spider.allowed_domains or set()
 
+        if self.spider.robots_txt_obey:
+            self._domain_delays: dict[str, float] = {}
+
         self._active_tasks: int = 0
         self._running: bool = False
         self._items: ItemList = ItemList()
@@ -68,11 +90,42 @@ class CrawlerEngine:
                 return True
         return False
 
+    async def _get_domain_delay(self, request: Request) -> float:
+        """Resolve the effective download delay for a domain.
+
+        Takes the max of the spider's configured delay and any robots.txt
+        directives (Crawl-delay / Request-rate). Result is cached per domain.
+        """
+        robots_manager = self._robots_manager
+        if robots_manager is None:
+            return self.spider.download_delay
+
+        domain = request.domain
+
+        if domain in self._domain_delays:
+            return self._domain_delays[domain]
+
+        # For domains covered by _prefetch_robots_txt this is a local parser read.
+        # Domains discovered mid-crawl (not in start_urls) will fetch here.
+        c_delay, r_rate = await robots_manager.get_delay_directives(request.url, request.sid)
+
+        delay = self.spider.download_delay
+
+        if r_rate:
+            req_count, period = r_rate
+            if req_count > 0:
+                delay = max(delay, period / req_count)
+
+        if c_delay is not None:
+            delay = max(delay, c_delay)
+
+        self._domain_delays[domain] = delay
+        return delay
+
     def _rate_limiter(self, domain: str) -> CapacityLimiter:
         """Get or create a per-domain concurrency limiter if enabled, otherwise use the global limiter."""
         if self.spider.concurrent_requests_per_domain:
-            if domain not in self._domain_limiters:
-                self._domain_limiters[domain] = CapacityLimiter(self.spider.concurrent_requests_per_domain)
+            self._domain_limiters.setdefault(domain, CapacityLimiter(self.spider.concurrent_requests_per_domain))
             return self._domain_limiters[domain]
         return self._global_limiter
 
@@ -85,47 +138,8 @@ class CrawlerEngine:
         if not request.sid:
             request.sid = self.session_manager.default_session_id
 
-    async def _process_request(self, request: Request) -> None:
-        """Download and process a single request."""
-        async with self._rate_limiter(request.domain):
-            if self.spider.download_delay:
-                await anyio.sleep(self.spider.download_delay)
-
-            if request._session_kwargs.get("proxy"):
-                self.stats.proxies.append(request._session_kwargs["proxy"])
-            if request._session_kwargs.get("proxies"):
-                self.stats.proxies.append(dict(request._session_kwargs["proxies"]))
-            try:
-                response = await self.session_manager.fetch(request)
-                self.stats.increment_requests_count(request.sid or self.session_manager.default_session_id)
-                self.stats.increment_response_bytes(request.domain, len(response.body))
-                self.stats.increment_status(response.status)
-
-            except Exception as e:
-                self.stats.failed_requests_count += 1
-                await self.spider.on_error(request, e)
-                return
-
-        if await self.spider.is_blocked(response):
-            self.stats.blocked_requests_count += 1
-            if request._retry_count < self.spider.max_blocked_retries:
-                retry_request = request.copy()
-                retry_request._retry_count += 1
-                retry_request.priority -= 1  # Don't retry immediately
-                retry_request.dont_filter = True
-                retry_request._session_kwargs.pop("proxy", None)
-                retry_request._session_kwargs.pop("proxies", None)
-
-                new_request = await self.spider.retry_blocked_request(retry_request, response)
-                self._normalize_request(new_request)
-                await self.scheduler.enqueue(new_request)
-                log.info(
-                    f"Scheduled blocked request for retry ({retry_request._retry_count}/{self.spider.max_blocked_retries}): {request.url}"
-                )
-            else:
-                log.warning(f"Max retries exceeded for blocked request: {request.url}")
-            return
-
+    async def _run_callbacks(self, request: Request, response: Response) -> None:
+        """Dispatch response to the request's callback and process yielded items/requests."""
         callback = request.callback if request.callback else self.spider.parse
         try:
             async for result in callback(response):
@@ -155,6 +169,75 @@ class CrawlerEngine:
             log.error(msg, exc_info=e)
             await self.spider.on_error(request, e)
 
+    async def _process_request(self, request: Request) -> None:
+        """Download and process a single request."""
+        if self._robots_manager:
+            can_fetch = await self._robots_manager.can_fetch(request.url, request.sid)
+            if not can_fetch:
+                self.stats.robots_disallowed_count += 1
+                log.info(f"Request disallowed by robots.txt: {request.url}")
+                return
+            delay = await self._get_domain_delay(request)
+        else:
+            delay = self.spider.download_delay
+
+        if self._cache_manager and request._fp is not None:
+            cached = await self._cache_manager.get(request._fp)
+            if cached is not None:
+                cached.request = request
+                self.stats.cache_hits += 1
+                self.stats.increment_requests_count(request.sid or self.session_manager.default_session_id)
+                self.stats.increment_response_bytes(request.domain, len(cached.body))
+                self.stats.increment_status(cached.status)
+                log.debug(f"Cache hit: {request.url}")
+                await self._run_callbacks(request, cached)
+                return
+
+        async with self._rate_limiter(request.domain):
+            if delay:
+                await anyio.sleep(delay)
+
+            if request._session_kwargs.get("proxy"):
+                self.stats.proxies.append(request._session_kwargs["proxy"])
+            if request._session_kwargs.get("proxies"):
+                self.stats.proxies.append(dict(request._session_kwargs["proxies"]))
+            try:
+                response = await self.session_manager.fetch(request)
+                self.stats.increment_requests_count(request.sid or self.session_manager.default_session_id)
+                self.stats.increment_response_bytes(request.domain, len(response.body))
+                self.stats.increment_status(response.status)
+
+            except Exception as e:
+                self.stats.failed_requests_count += 1
+                await self.spider.on_error(request, e)
+                return
+
+        if self._cache_manager and request._fp is not None:
+            self.stats.cache_misses += 1
+            await self._cache_manager.put(request._fp, response, request._session_kwargs.get("method", "GET"))
+
+        if await self.spider.is_blocked(response):
+            self.stats.blocked_requests_count += 1
+            if request._retry_count < self.spider.max_blocked_retries:
+                retry_request = request.copy()
+                retry_request._retry_count += 1
+                retry_request.priority -= 1  # Don't retry immediately
+                retry_request.dont_filter = True
+                retry_request._session_kwargs.pop("proxy", None)
+                retry_request._session_kwargs.pop("proxies", None)
+
+                new_request = await self.spider.retry_blocked_request(retry_request, response)
+                self._normalize_request(new_request)
+                await self.scheduler.enqueue(new_request)
+                log.info(
+                    f"Scheduled blocked request for retry ({retry_request._retry_count}/{self.spider.max_blocked_retries}): {request.url}"
+                )
+            else:
+                log.warning(f"Max retries exceeded for blocked request: {request.url}")
+            return
+
+        await self._run_callbacks(request, response)
+
     async def _task_wrapper(self, request: Request) -> None:
         """Wrapper to track active task count."""
         try:
@@ -219,6 +302,25 @@ class CrawlerEngine:
 
         return True
 
+    async def _prefetch_robots_txt(self) -> None:
+        """Pre-warm the robots.txt cache before the crawl loop starts.
+
+        Extracts unique domains from start_urls, preserving the original scheme.
+        """
+        if not self._robots_manager or not self.spider.start_urls:
+            return
+
+        # Deduplicate by netloc, preserving the scheme from the first URL per domain
+        seen: set[str] = set()
+        seed_urls: list[str] = []
+        for url in self.spider.start_urls:
+            parsed = urlparse(url)
+            if parsed.netloc not in seen:
+                seen.add(parsed.netloc)
+                seed_urls.append(f"{parsed.scheme}://{parsed.netloc}/")
+
+        await self._robots_manager.prefetch(seed_urls, self.session_manager.default_session_id)
+
     async def crawl(self) -> CrawlStats:
         """Run the spider and return CrawlStats."""
         self._running = True
@@ -227,6 +329,9 @@ class CrawlerEngine:
         self._pause_requested = False
         self._force_stop = False
         self.stats = CrawlStats(start_time=anyio.current_time())
+        self._domain_limiters.clear()
+        if self._robots_manager:
+            self._domain_delays.clear()
 
         # Check for existing checkpoint
         resuming = (await self._restore_from_checkpoint()) if self._checkpoint_system_enabled else False
@@ -238,6 +343,8 @@ class CrawlerEngine:
             self.stats.download_delay = self.spider.download_delay
             await self.spider.on_start(resuming=resuming)
 
+            await self._prefetch_robots_txt()
+
             try:
                 if not resuming:
                     async for request in self.spider.start_requests():
@@ -251,11 +358,7 @@ class CrawlerEngine:
                     while self._running:
                         if self._pause_requested:
                             if self._active_tasks == 0 or self._force_stop:
-                                if self._force_stop:
-                                    log.warning(f"Force stopping with {self._active_tasks} active tasks")
-                                    tg.cancel_scope.cancel()
-
-                                # Only save checkpoint if checkpoint system is enabled
+                                # Save checkpoint before canceling to avoid data loss
                                 if self._checkpoint_system_enabled:
                                     await self._save_checkpoint()
                                     self.paused = True
@@ -263,6 +366,10 @@ class CrawlerEngine:
                                 else:
                                     log.info("Spider stopped gracefully")
 
+                                if self._force_stop:
+                                    log.warning(f"Force stopping with {self._active_tasks} active tasks")
+                                    tg.cancel_scope.cancel()
+
                                 self._running = False
                                 break
 

scrapling/spiders/result.py

@@ -47,6 +47,9 @@ class CrawlStats:
     concurrent_requests_per_domain: int = 0
     failed_requests_count: int = 0
     offsite_requests_count: int = 0
+    robots_disallowed_count: int = 0
+    cache_hits: int = 0
+    cache_misses: int = 0
     response_bytes: int = 0
     items_scraped: int = 0
     items_dropped: int = 0
@@ -95,6 +98,9 @@ class CrawlStats:
             "sessions_requests_count": self.sessions_requests_count,
             "failed_requests_count": self.failed_requests_count,
             "offsite_requests_count": self.offsite_requests_count,
+            "robots_disallowed_count": self.robots_disallowed_count,
+            "cache_hits": self.cache_hits,
+            "cache_misses": self.cache_misses,
             "blocked_requests_count": self.blocked_requests_count,
             "response_status_count": self.response_status_count,
             "response_bytes": self.response_bytes,

scrapling/spiders/robotstxt.py

@@ -0,0 +1,77 @@
+from urllib.parse import urlparse
+
+from anyio import create_task_group
+from protego import Protego
+
+from scrapling.core._types import Dict, Optional, Callable, Awaitable
+from scrapling.core.utils import log
+
+
+class RobotsTxtManager:
+    """Manages fetching, parsing, and caching of robots.txt files."""
+
+    def __init__(self, fetch_fn: Callable[[str, str], Awaitable]):
+        self._fetch_fn = fetch_fn
+        self._cache: Dict[str, Protego] = {}
+
+    async def _get_parser(self, url: str, sid: str) -> Protego:
+        parsed = urlparse(url)
+        domain = parsed.netloc
+
+        if domain in self._cache:
+            return self._cache[domain]
+
+        scheme = parsed.scheme or "https"
+        robots_url = f"{scheme}://{domain}/robots.txt"
+        content = ""
+        try:
+            response = await self._fetch_fn(robots_url, sid)
+            if response.status == 200:
+                content = response.body.decode(response.encoding, errors="replace")
+        except Exception as e:
+            log.warning(f"Failed to fetch robots.txt for {domain}: {e}")
+
+        try:
+            parser = Protego.parse(content)
+        except Exception as e:
+            log.warning(f"Failed to parse robots.txt for {domain}: {e}")
+            parser = Protego.parse("")
+
+        self._cache[domain] = parser
+        return parser
+
+    async def can_fetch(self, url: str, sid: str) -> bool:
+        """Check if a URL can be fetched according to the domain's robots.txt.
+
+        :param url: The full URL to check
+        :param sid: Session ID for fetching robots.txt if not yet cached
+        """
+        parser = await self._get_parser(url, sid)
+        return parser.can_fetch(url, "*")
+
+    async def get_delay_directives(self, url: str, sid: str) -> tuple[Optional[float], Optional[tuple[int, int]]]:
+        """Return both crawl-delay and request-rate in a single parser lookup.
+
+        :param url: Any URL on the domain to check
+        :param sid: Session ID for fetching robots.txt if not yet cached
+        """
+        parser = await self._get_parser(url, sid)
+        c_delay = parser.crawl_delay("*")
+        rate = parser.request_rate("*")
+        return (
+            float(c_delay) if c_delay is not None else None,
+            (rate.requests, rate.seconds) if rate is not None else None,
+        )
+
+    async def prefetch(self, urls: list[str], sid: str) -> None:
+        """Pre-warm the robots.txt cache for a list of seed URLs concurrently.
+
+        :param urls: Seed URLs whose domains should be pre-fetched (one per domain).
+        :param sid: Session ID to use for the robots.txt fetch requests.
+        """
+        if not urls:
+            return
+        log.debug(f"Pre-fetching robots.txt for {len(urls)} domain(s)")
+        async with create_task_group() as tg:
+            for url in urls:
+                tg.start_soon(self._get_parser, url, sid)

scrapling/spiders/spider.py

@@ -72,6 +72,13 @@ class Spider(ABC):
     start_urls: list[str] = []
     allowed_domains: Set[str] = set()
 
+    # Robots.txt compliance
+    robots_txt_obey: bool = False
+
+    # Development mode
+    development_mode: bool = False
+    development_cache_dir: Optional[str] = None
+
     # Concurrency settings
     concurrent_requests: int = 4
     concurrent_requests_per_domain: int = 0