From a134fdb8cce853f66713e115b81d195ec6e7b0c3 Mon Sep 17 00:00:00 2001
From: Abdullah <52079299+AbdullahY36@users.noreply.github.com>
Date: Sat, 4 Apr 2026 03:10:17 +0200
Subject: [PATCH] feat(spiders): enable robots.txt compliance by default

robots_txt_obey now defaults to True. Spiders must explicitly opt out
with robots_txt_obey = False rather than opt in, making ethical
crawling the default behaviour.

File: scrapling/spiders/spider.py
---
 scrapling/spiders/spider.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapling/spiders/spider.py b/scrapling/spiders/spider.py
index 6aaa24f..52afcbd 100644
--- a/scrapling/spiders/spider.py
+++ b/scrapling/spiders/spider.py
@@ -73,7 +73,7 @@ class Spider(ABC):
     allowed_domains: Set[str] = set()
 
     # Robots.txt compliance
-    robots_txt_obey: bool = False
+    robots_txt_obey: bool = True
 
     # Concurrency settings
     concurrent_requests: int = 4