Ahmed Elshahat
f0db3d7d14
CheckpointManager.save() and ResponseCacheManager.put() write to a temp file and then move it into place with Path.rename(). On Windows, os.rename cannot overwrite an existing destination and raises FileExistsError (WinError 183), so every write after the first one fails: checkpoint saving raises and breaks resume, while the development response cache swallows the error and keeps returning the stale entry. Path.replace() (os.replace) overwrites the destination atomically on every platform and behaves identically to rename() on POSIX, so this is a no-op on Linux and macOS and only fixes the broken overwrite on Windows. Add a regression test for the cache overwrite path; the checkpoint overwrite is already covered by test_multiple_saves_overwrite.
80 lines
2.9 KiB
Python
80 lines
2.9 KiB
Python
from base64 import b64encode, b64decode
|
|
from pathlib import Path
|
|
|
|
import orjson
|
|
import anyio
|
|
from anyio import Path as AsyncPath
|
|
|
|
from scrapling.core.utils import log
|
|
from scrapling.core._types import Dict, Optional, Any
|
|
from scrapling.engines.toolbelt.custom import Response
|
|
|
|
|
|
class ResponseCacheManager:
|
|
"""Caches HTTP responses to disk for replay during spider development."""
|
|
|
|
def __init__(self, cache_dir: str | Path):
|
|
self._cache_dir = AsyncPath(cache_dir)
|
|
|
|
def _cache_path(self, fingerprint: bytes) -> AsyncPath:
|
|
return self._cache_dir / f"{fingerprint.hex()}.json"
|
|
|
|
async def get(self, fingerprint: bytes) -> Optional[Response]:
|
|
path = self._cache_path(fingerprint)
|
|
if not await path.exists():
|
|
return None
|
|
|
|
try:
|
|
async with await anyio.open_file(path, "rb") as f:
|
|
data: Dict[str, Any] = orjson.loads(await f.read())
|
|
|
|
return Response(
|
|
url=data["url"],
|
|
content=b64decode(data["content"]),
|
|
status=data["status"],
|
|
reason=data["reason"],
|
|
encoding=data["encoding"],
|
|
cookies=data["cookies"],
|
|
headers=data["headers"],
|
|
request_headers=data["request_headers"],
|
|
method=data["method"],
|
|
)
|
|
except Exception as e:
|
|
log.warning(f"Failed to read cached response for {fingerprint.hex()}: {e}")
|
|
return None
|
|
|
|
async def put(self, fingerprint: bytes, response: Response, method: str = "GET") -> None:
|
|
await self._cache_dir.mkdir(parents=True, exist_ok=True)
|
|
temp_path = self._cache_path(fingerprint).with_suffix(".tmp")
|
|
|
|
try:
|
|
serialized = orjson.dumps(
|
|
{
|
|
"url": response.url,
|
|
"content": b64encode(response.body).decode("ascii"),
|
|
"status": response.status,
|
|
"reason": response.reason,
|
|
"encoding": response.encoding,
|
|
"cookies": dict(response.cookies) if isinstance(response.cookies, dict) else {},
|
|
"headers": dict(response.headers),
|
|
"request_headers": dict(response.request_headers),
|
|
"method": method,
|
|
}
|
|
)
|
|
async with await anyio.open_file(temp_path, "wb") as f:
|
|
await f.write(serialized)
|
|
|
|
await temp_path.replace(self._cache_path(fingerprint))
|
|
except Exception as e:
|
|
if await temp_path.exists():
|
|
await temp_path.unlink()
|
|
log.warning(f"Failed to cache response for {fingerprint.hex()}: {e}")
|
|
|
|
async def clear(self) -> None:
|
|
if not await self._cache_dir.exists():
|
|
return
|
|
async for entry in self._cache_dir.iterdir():
|
|
if entry.suffix == ".json":
|
|
await entry.unlink()
|
|
log.info(f"Cleared response cache at {self._cache_dir}")
|