From ea2dd7866b9cfe052279025de790892b15275096 Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Sun, 5 Apr 2026 01:55:17 +0200
Subject: [PATCH] refactor(spiders): Make Robots.txt compliance turned off by
 default

Scrapy is turning it off by default
---
 scrapling/spiders/spider.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapling/spiders/spider.py b/scrapling/spiders/spider.py
index 52afcbd..6aaa24f 100644
--- a/scrapling/spiders/spider.py
+++ b/scrapling/spiders/spider.py
@@ -73,7 +73,7 @@ class Spider(ABC):
     allowed_domains: Set[str] = set()
 
     # Robots.txt compliance
-    robots_txt_obey: bool = True
+    robots_txt_obey: bool = False
 
     # Concurrency settings
     concurrent_requests: int = 4