feat(spiders): add a development mode

2026-04-07 04:08:54 +02:00
parent fad9efdc35
commit d1baf1fc46
7 changed files with 399 additions and 56 deletions
@@ -75,6 +75,10 @@ class Spider(ABC):
    # Robots.txt compliance
    robots_txt_obey: bool = False

+    # Development mode
+    development_mode: bool = False
+    development_cache_dir: Optional[str] = None
+
    # Concurrency settings
    concurrent_requests: int = 4
    concurrent_requests_per_domain: int = 0