feat(spiders): add a development mode

2026-04-07 04:08:54 +02:00
parent fad9efdc35
commit d1baf1fc46
7 changed files with 399 additions and 56 deletions
@@ -48,6 +48,8 @@ class CrawlStats:
    failed_requests_count: int = 0
    offsite_requests_count: int = 0
    robots_disallowed_count: int = 0
+    cache_hits: int = 0
+    cache_misses: int = 0
    response_bytes: int = 0
    items_scraped: int = 0
    items_dropped: int = 0
@@ -97,6 +99,8 @@ class CrawlStats:
            "failed_requests_count": self.failed_requests_count,
            "offsite_requests_count": self.offsite_requests_count,
            "robots_disallowed_count": self.robots_disallowed_count,
+            "cache_hits": self.cache_hits,
+            "cache_misses": self.cache_misses,
            "blocked_requests_count": self.blocked_requests_count,
            "response_status_count": self.response_status_count,
            "response_bytes": self.response_bytes,