feat(spiders): integrate robots.txt compliance into the crawl engine

2026-04-03 15:08:33 +02:00
parent 0bbe62fc7f
commit 5c40c6a853
3 changed files with 83 additions and 6 deletions
@@ -72,6 +72,9 @@ class Spider(ABC):
    start_urls: list[str] = []
    allowed_domains: Set[str] = set()

+    # Robots.txt compliance
+    robots_txt_obey: bool = False
+
    # Concurrency settings
    concurrent_requests: int = 4
    concurrent_requests_per_domain: int = 0