freedak/Scrapling
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(spiders): add a development mode

Browse Source
This commit is contained in:
Karim shoair
2026-04-07 04:08:54 +02:00
parent fad9efdc35
commit d1baf1fc46
7 changed files with 399 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scrapling/spiders/cache.py
+79
View File
@@ -0,0 +1,79 @@
from base64 import b64encode, b64decode
from pathlib import Path
import orjson
import anyio
from anyio import Path as AsyncPath
from scrapling.core.utils import log
from scrapling.core._types import Dict, Optional, Any
from scrapling.engines.toolbelt.custom import Response
class ResponseCacheManager:
"""Caches HTTP responses to disk for replay during spider development."""
def __init__(self, cache_dir: str | Path):
self._cache_dir = AsyncPath(cache_dir)
def _cache_path(self, fingerprint: bytes) -> AsyncPath:
return self._cache_dir / f"{fingerprint.hex()}.json"
async def get(self, fingerprint: bytes) -> Optional[Response]:
path = self._cache_path(fingerprint)
if not await path.exists():
return None
try:
async with await anyio.open_file(path, "rb") as f:
data: Dict[str, Any] = orjson.loads(await f.read())
return Response(
url=data["url"],
content=b64decode(data["content"]),
status=data["status"],
reason=data["reason"],
encoding=data["encoding"],
cookies=data["cookies"],
headers=data["headers"],
request_headers=data["request_headers"],
method=data["method"],
)
except Exception as e:
log.warning(f"Failed to read cached response for {fingerprint.hex()}: {e}")
return None
async def put(self, fingerprint: bytes, response: Response, method: str = "GET") -> None:
await self._cache_dir.mkdir(parents=True, exist_ok=True)
temp_path = self._cache_path(fingerprint).with_suffix(".tmp")
try:
serialized = orjson.dumps(
{
"url": response.url,
"content": b64encode(response.body).decode("ascii"),
"status": response.status,
"reason": response.reason,
"encoding": response.encoding,
"cookies": dict(response.cookies) if isinstance(response.cookies, dict) else {},
"headers": dict(response.headers),
"request_headers": dict(response.request_headers),
"method": method,
}
)
async with await anyio.open_file(temp_path, "wb") as f:
await f.write(serialized)
await temp_path.rename(self._cache_path(fingerprint))
except Exception as e:
if await temp_path.exists():
await temp_path.unlink()
log.warning(f"Failed to cache response for {fingerprint.hex()}: {e}")
async def clear(self) -> None:
if not await self._cache_dir.exists():
return
async for entry in self._cache_dir.iterdir():
if entry.suffix == ".json":
await entry.unlink()
log.info(f"Cleared response cache at {self._cache_dir}")
scrapling/spiders/engine.py
+80 -56
View File
@@ -13,6 +13,7 @@ from scrapling.spiders.session import SessionManager
from scrapling.spiders.request import Request, Response
from scrapling.spiders.robotstxt import RobotsTxtManager
from scrapling.spiders.result import CrawlStats, ItemList
from scrapling.spiders.cache import ResponseCacheManager
from scrapling.spiders.checkpoint import CheckpointManager, CheckpointData
from scrapling.core._types import Dict, Union, Optional, TYPE_CHECKING, Any, AsyncGenerator
@@ -52,6 +53,13 @@ class CrawlerEngine:
else:
self._robots_manager = None
if self.spider.development_mode:
cache_dir = self.spider.development_cache_dir or f".scrapling_cache/{self.spider.name}"
self._cache_manager: Optional[ResponseCacheManager] = ResponseCacheManager(cache_dir)
log.warning("Development mode enabled -- responses will be cached to disk and replayed on subsequent runs")
else:
self._cache_manager = None
self._global_limiter = CapacityLimiter(spider.concurrent_requests)
self._domain_limiters: dict[str, CapacityLimiter] = {}
self._allowed_domains: set[str] = spider.allowed_domains or set()
@@ -130,57 +138,8 @@ class CrawlerEngine:
if not request.sid:
request.sid = self.session_manager.default_session_id
async def _process_request(self, request: Request) -> None:
"""Download and process a single request."""
if self._robots_manager:
can_fetch = await self._robots_manager.can_fetch(request.url, request.sid)
if not can_fetch:
self.stats.robots_disallowed_count += 1
log.info(f"Request disallowed by robots.txt: {request.url}")
return
delay = await self._get_domain_delay(request)
else:
delay = self.spider.download_delay
async with self._rate_limiter(request.domain):
if delay:
await anyio.sleep(delay)
if request._session_kwargs.get("proxy"):
self.stats.proxies.append(request._session_kwargs["proxy"])
if request._session_kwargs.get("proxies"):
self.stats.proxies.append(dict(request._session_kwargs["proxies"]))
try:
response = await self.session_manager.fetch(request)
self.stats.increment_requests_count(request.sid or self.session_manager.default_session_id)
self.stats.increment_response_bytes(request.domain, len(response.body))
self.stats.increment_status(response.status)
except Exception as e:
self.stats.failed_requests_count += 1
await self.spider.on_error(request, e)
return
if await self.spider.is_blocked(response):
self.stats.blocked_requests_count += 1
if request._retry_count < self.spider.max_blocked_retries:
retry_request = request.copy()
retry_request._retry_count += 1
retry_request.priority -= 1 # Don't retry immediately
retry_request.dont_filter = True
retry_request._session_kwargs.pop("proxy", None)
retry_request._session_kwargs.pop("proxies", None)
new_request = await self.spider.retry_blocked_request(retry_request, response)
self._normalize_request(new_request)
await self.scheduler.enqueue(new_request)
log.info(
f"Scheduled blocked request for retry ({retry_request._retry_count}/{self.spider.max_blocked_retries}): {request.url}"
)
else:
log.warning(f"Max retries exceeded for blocked request: {request.url}")
return
async def _run_callbacks(self, request: Request, response: Response) -> None:
"""Dispatch response to the request's callback and process yielded items/requests."""
callback = request.callback if request.callback else self.spider.parse
try:
async for result in callback(response):
@@ -210,6 +169,75 @@ class CrawlerEngine:
log.error(msg, exc_info=e)
await self.spider.on_error(request, e)
async def _process_request(self, request: Request) -> None:
"""Download and process a single request."""
if self._robots_manager:
can_fetch = await self._robots_manager.can_fetch(request.url, request.sid)
if not can_fetch:
self.stats.robots_disallowed_count += 1
log.info(f"Request disallowed by robots.txt: {request.url}")
return
delay = await self._get_domain_delay(request)
else:
delay = self.spider.download_delay
if self._cache_manager and request._fp is not None:
cached = await self._cache_manager.get(request._fp)
if cached is not None:
cached.request = request
self.stats.cache_hits += 1
self.stats.increment_requests_count(request.sid or self.session_manager.default_session_id)
self.stats.increment_response_bytes(request.domain, len(cached.body))
self.stats.increment_status(cached.status)
log.debug(f"Cache hit: {request.url}")
await self._run_callbacks(request, cached)
return
async with self._rate_limiter(request.domain):
if delay:
await anyio.sleep(delay)
if request._session_kwargs.get("proxy"):
self.stats.proxies.append(request._session_kwargs["proxy"])
if request._session_kwargs.get("proxies"):
self.stats.proxies.append(dict(request._session_kwargs["proxies"]))
try:
response = await self.session_manager.fetch(request)
self.stats.increment_requests_count(request.sid or self.session_manager.default_session_id)
self.stats.increment_response_bytes(request.domain, len(response.body))
self.stats.increment_status(response.status)
except Exception as e:
self.stats.failed_requests_count += 1
await self.spider.on_error(request, e)
return
if self._cache_manager and request._fp is not None:
self.stats.cache_misses += 1
await self._cache_manager.put(request._fp, response, request._session_kwargs.get("method", "GET"))
if await self.spider.is_blocked(response):
self.stats.blocked_requests_count += 1
if request._retry_count < self.spider.max_blocked_retries:
retry_request = request.copy()
retry_request._retry_count += 1
retry_request.priority -= 1 # Don't retry immediately
retry_request.dont_filter = True
retry_request._session_kwargs.pop("proxy", None)
retry_request._session_kwargs.pop("proxies", None)
new_request = await self.spider.retry_blocked_request(retry_request, response)
self._normalize_request(new_request)
await self.scheduler.enqueue(new_request)
log.info(
f"Scheduled blocked request for retry ({retry_request._retry_count}/{self.spider.max_blocked_retries}): {request.url}"
)
else:
log.warning(f"Max retries exceeded for blocked request: {request.url}")
return
await self._run_callbacks(request, response)
async def _task_wrapper(self, request: Request) -> None:
"""Wrapper to track active task count."""
try:
@@ -330,11 +358,7 @@ class CrawlerEngine:
while self._running:
if self._pause_requested:
if self._active_tasks == 0 or self._force_stop:
# Save checkpoint BEFORE cancelling the scope.
# cancel_scope.cancel() makes all subsequent awaits
# raise Cancelled, which would abort the checkpoint
# write and leave self.paused=False — causing the
# finally block to DELETE the previous checkpoint.
# Save checkpoint before canceling to avoid data loss
if self._checkpoint_system_enabled:
await self._save_checkpoint()
self.paused = True
scrapling/spiders/result.py
+4
View File
@@ -48,6 +48,8 @@ class CrawlStats:
failed_requests_count: int = 0
offsite_requests_count: int = 0
robots_disallowed_count: int = 0
cache_hits: int = 0
cache_misses: int = 0
response_bytes: int = 0
items_scraped: int = 0
items_dropped: int = 0
@@ -97,6 +99,8 @@ class CrawlStats:
"failed_requests_count": self.failed_requests_count,
"offsite_requests_count": self.offsite_requests_count,
"robots_disallowed_count": self.robots_disallowed_count,
"cache_hits": self.cache_hits,
"cache_misses": self.cache_misses,
"blocked_requests_count": self.blocked_requests_count,
"response_status_count": self.response_status_count,
"response_bytes": self.response_bytes,
scrapling/spiders/spider.py
+4
View File
@@ -75,6 +75,10 @@ class Spider(ABC):
# Robots.txt compliance
robots_txt_obey: bool = False
# Development mode
development_mode: bool = False
development_cache_dir: Optional[str] = None
# Concurrency settings
concurrent_requests: int = 4
concurrent_requests_per_domain: int = 0