From 07129ce4b1645f73df25943eb88c215c05e1f14d Mon Sep 17 00:00:00 2001
From: Abdullah <52079299+AbdullahY36@users.noreply.github.com>
Date: Fri, 3 Apr 2026 17:51:00 +0200
Subject: [PATCH] feat(deps): move protego to fetchers optional dependency

protego is only used by the spider framework for robots.txt compliance.

Moving it from core dependencies to the optional 'fetchers' group reduces
the dependency footprint for users who don't need the spider framework.

for pyproject.toml file
---
 pyproject.toml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/pyproject.toml b/pyproject.toml
index 41fb14c..4333d7e 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -66,8 +66,7 @@ dependencies = [
     "orjson>=3.11.8",
     "tld>=0.13.2",
     "w3lib>=2.4.1",
-    "typing_extensions",
-    "protego>=0.4.0",
+    "typing_extensions"
 ]
 
 [project.optional-dependencies]
@@ -79,7 +78,8 @@ fetchers = [
     "browserforge>=1.2.4",
     "apify-fingerprint-datapoints>=0.12.0",
     "msgspec>=0.20.0",
-    "anyio>=4.12.1"
+    "anyio>=4.12.1",
+    "protego>=0.4.0",
 ]
 ai = [
     "mcp>=1.26.0",