refactor(spiders): Make Robots.txt compliance turned off by default

Scrapy is turning it off by default
2026-04-05 01:55:17 +02:00
parent 556a90f645
commit ea2dd7866b
1 changed files with 1 additions and 1 deletions
@@ -73,7 +73,7 @@ class Spider(ABC):
    allowed_domains: Set[str] = set()

    # Robots.txt compliance
-    robots_txt_obey: bool = True
+    robots_txt_obey: bool = False

    # Concurrency settings
    concurrent_requests: int = 4