feat(deps): add protego for robots.txt parsing and fix pyright type error in static.py

2026-04-03 15:08:33 +02:00
parent cb93c15c5a
commit 1d15349e07
2 changed files with 2 additions and 0 deletions
@@ -67,6 +67,7 @@ dependencies = [
    "tld>=0.13.2",
    "w3lib>=2.4.1",
    "typing_extensions",
+    "protego>=0.4.0",
 ]

 [project.optional-dependencies]