freedak/Scrapling
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(spiders): use os.replace for atomic checkpoint/cache writes on Windows (#344)

Browse Source
This commit is contained in:
Karim shoair
2026-06-07 15:49:33 +03:00
committed by GitHub
parent 2b2abb3810 4f0a593b7d
commit f7d1e338e0
3 changed files with 23 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scrapling/spiders/cache.py
+1 -1
View File
@@ -64,7 +64,7 @@ class ResponseCacheManager:
async with await anyio.open_file(temp_path, "wb") as f: async with await anyio.open_file(temp_path, "wb") as f:
await f.write(serialized) await f.write(serialized)
await temp_path.rename(self._cache_path(fingerprint)) await temp_path.replace(self._cache_path(fingerprint))
except Exception as e: except Exception as e:
if await temp_path.exists(): if await temp_path.exists():
await temp_path.unlink() await temp_path.unlink()
scrapling/spiders/checkpoint.py
+1 -1
View File
@@ -50,7 +50,7 @@ class CheckpointManager:
async with await anyio.open_file(temp_path, "wb") as f: async with await anyio.open_file(temp_path, "wb") as f:
await f.write(serialized) await f.write(serialized)
await temp_path.rename(self._checkpoint_path) await temp_path.replace(self._checkpoint_path)
log.info(f"Checkpoint saved: {len(data.requests)} requests, {len(data.seen)} seen URLs") log.info(f"Checkpoint saved: {len(data.requests)} requests, {len(data.seen)} seen URLs")
except Exception as e: except Exception as e:
tests/spiders/test_cache.py
+21
View File
@@ -49,6 +49,27 @@ class TestResponseCacheManager:
assert dict(restored.headers) == dict(original.headers) assert dict(restored.headers) == dict(original.headers)
assert dict(restored.request_headers) == dict(original.request_headers) assert dict(restored.request_headers) == dict(original.request_headers)
@pytest.mark.anyio
async def test_put_overwrites_existing_entry(self):
"""Re-caching the same fingerprint must replace the stored response.
Regression test for a Windows-only failure: ``Path.rename`` cannot
overwrite an existing destination on Windows (raising ``WinError 183``),
so the second ``put`` was caught by the error handler, the temp file was
removed, and ``get`` kept returning the stale body. ``Path.replace``
overwrites atomically on every platform.
"""
with tempfile.TemporaryDirectory() as tmpdir:
cache = ResponseCacheManager(tmpdir)
fp = b"\x05" * 20
await cache.put(fp, _make_response(body=b"<html>first</html>"), "GET")
await cache.put(fp, _make_response(body=b"<html>second</html>"), "GET")
restored = await cache.get(fp)
assert restored is not None
assert restored.body == b"<html>second</html>"
@pytest.mark.anyio @pytest.mark.anyio
async def test_get_cache_miss(self): async def test_get_cache_miss(self):
with tempfile.TemporaryDirectory() as tmpdir: with tempfile.TemporaryDirectory() as tmpdir: