feat(spiders): add a development mode

scrapling/spiders/cache.py

@@ -0,0 +1,79 @@
+from base64 import b64encode, b64decode
+from pathlib import Path
+
+import orjson
+import anyio
+from anyio import Path as AsyncPath
+
+from scrapling.core.utils import log
+from scrapling.core._types import Dict, Optional, Any
+from scrapling.engines.toolbelt.custom import Response
+
+
+class ResponseCacheManager:
+    """Caches HTTP responses to disk for replay during spider development."""
+
+    def __init__(self, cache_dir: str | Path):
+        self._cache_dir = AsyncPath(cache_dir)
+
+    def _cache_path(self, fingerprint: bytes) -> AsyncPath:
+        return self._cache_dir / f"{fingerprint.hex()}.json"
+
+    async def get(self, fingerprint: bytes) -> Optional[Response]:
+        path = self._cache_path(fingerprint)
+        if not await path.exists():
+            return None
+
+        try:
+            async with await anyio.open_file(path, "rb") as f:
+                data: Dict[str, Any] = orjson.loads(await f.read())
+
+            return Response(
+                url=data["url"],
+                content=b64decode(data["content"]),
+                status=data["status"],
+                reason=data["reason"],
+                encoding=data["encoding"],
+                cookies=data["cookies"],
+                headers=data["headers"],
+                request_headers=data["request_headers"],
+                method=data["method"],
+            )
+        except Exception as e:
+            log.warning(f"Failed to read cached response for {fingerprint.hex()}: {e}")
+            return None
+
+    async def put(self, fingerprint: bytes, response: Response, method: str = "GET") -> None:
+        await self._cache_dir.mkdir(parents=True, exist_ok=True)
+        temp_path = self._cache_path(fingerprint).with_suffix(".tmp")
+
+        try:
+            serialized = orjson.dumps(
+                {
+                    "url": response.url,
+                    "content": b64encode(response.body).decode("ascii"),
+                    "status": response.status,
+                    "reason": response.reason,
+                    "encoding": response.encoding,
+                    "cookies": dict(response.cookies) if isinstance(response.cookies, dict) else {},
+                    "headers": dict(response.headers),
+                    "request_headers": dict(response.request_headers),
+                    "method": method,
+                }
+            )
+            async with await anyio.open_file(temp_path, "wb") as f:
+                await f.write(serialized)
+
+            await temp_path.rename(self._cache_path(fingerprint))
+        except Exception as e:
+            if await temp_path.exists():
+                await temp_path.unlink()
+            log.warning(f"Failed to cache response for {fingerprint.hex()}: {e}")
+
+    async def clear(self) -> None:
+        if not await self._cache_dir.exists():
+            return
+        async for entry in self._cache_dir.iterdir():
+            if entry.suffix == ".json":
+                await entry.unlink()
+        log.info(f"Cleared response cache at {self._cache_dir}")