feat(spiders): integrate robots.txt compliance into the crawl engine

2026-04-03 15:08:33 +02:00
parent 0bbe62fc7f
commit 5c40c6a853
3 changed files with 83 additions and 6 deletions
@@ -47,6 +47,7 @@ class CrawlStats:
    concurrent_requests_per_domain: int = 0
    failed_requests_count: int = 0
    offsite_requests_count: int = 0
+    robots_disallowed_count: int = 0
    response_bytes: int = 0
    items_scraped: int = 0
    items_dropped: int = 0
@@ -95,6 +96,7 @@ class CrawlStats:
            "sessions_requests_count": self.sessions_requests_count,
            "failed_requests_count": self.failed_requests_count,
            "offsite_requests_count": self.offsite_requests_count,
+            "robots_disallowed_count": self.robots_disallowed_count,
            "blocked_requests_count": self.blocked_requests_count,
            "response_status_count": self.response_status_count,
            "response_bytes": self.response_bytes,