diff --git a/README.md b/README.md index d577a00..d6b6e41 100644 --- a/README.md +++ b/README.md @@ -214,6 +214,7 @@ MySpider().start() - 💾 **Pause & Resume**: Checkpoint-based crawl persistence. Press Ctrl+C for a graceful shutdown; restart to resume from where you left off. - 📡 **Streaming Mode**: Stream scraped items as they arrive via `async for item in spider.stream()` with real-time stats - ideal for UI, pipelines, and long-running crawls. - 🛡️ **Blocked Request Detection**: Automatic detection and retry of blocked requests with customizable logic. +- 🤖 **Robots.txt Compliance**: Optional `robots_txt_obey` flag that respects `Disallow`, `Crawl-delay`, and `Request-rate` directives with per-domain caching. - 📦 **Built-in Export**: Export results through hooks and your own pipeline or the built-in JSON/JSONL with `result.items.to_json()` / `result.items.to_jsonl()` respectively. ### Advanced Websites Fetching with Session Support diff --git a/agent-skill/Scrapling-Skill.zip b/agent-skill/Scrapling-Skill.zip index c89b5d7..b03d956 100644 Binary files a/agent-skill/Scrapling-Skill.zip and b/agent-skill/Scrapling-Skill.zip differ diff --git a/agent-skill/Scrapling-Skill/SKILL.md b/agent-skill/Scrapling-Skill/SKILL.md index 1fddd58..6b3cbf7 100644 --- a/agent-skill/Scrapling-Skill/SKILL.md +++ b/agent-skill/Scrapling-Skill/SKILL.md @@ -1,7 +1,7 @@ --- name: scrapling-official description: Scrape web pages using Scrapling with anti-bot bypass (like Cloudflare Turnstile), stealth headless browsing, spiders framework, adaptive scraping, and JavaScript rendering. Use when asked to scrape, crawl, or extract data from websites; web_fetch fails; the site has anti-bot protections; write Python code to scrape/crawl; or write spiders. -version: "0.4.3" +version: "0.4.4" license: Complete terms in LICENSE.txt metadata: homepage: "https://scrapling.readthedocs.io/en/latest/index.html" @@ -40,7 +40,7 @@ Blazing fast crawls with real-time stats and streaming. Built by Web Scrapers fo Create a virtual Python environment through any way available, like `venv`, then inside the environment do: -`pip install "scrapling[all]>=0.4.3"` +`pip install "scrapling[all]>=0.4.4"` Then do this to download all the browsers' dependencies: @@ -258,6 +258,7 @@ class QuotesSpider(Spider): name = "quotes" start_urls = ["https://quotes.toscrape.com/"] concurrent_requests = 10 + robots_txt_obey = True # Respect robots.txt rules async def parse(self, response: Response): for quote in response.css('.quote'): @@ -379,7 +380,7 @@ This skill encapsulates almost all the published documentation in Markdown, so d ## Guardrails (Always) - Only scrape content you're authorized to access. -- Respect robots.txt and ToS. -- Add delays (download_delay) for large crawls. +- Respect robots.txt and ToS. Use `robots_txt_obey = True` on spiders to enforce this automatically. +- Add delays (`download_delay`) for large crawls. - Don't bypass paywalls or authentication without permission. - Never scrape personal/sensitive data. \ No newline at end of file diff --git a/agent-skill/Scrapling-Skill/examples/README.md b/agent-skill/Scrapling-Skill/examples/README.md index 3f753c5..0c2f031 100644 --- a/agent-skill/Scrapling-Skill/examples/README.md +++ b/agent-skill/Scrapling-Skill/examples/README.md @@ -9,7 +9,7 @@ All examples collect **all 100 quotes across 10 pages**. Make sure Scrapling is installed: ```bash -pip install "scrapling[all]>=0.4.3" +pip install "scrapling[all]>=0.4.4" scrapling install --force ``` diff --git a/agent-skill/Scrapling-Skill/references/spiders/advanced.md b/agent-skill/Scrapling-Skill/references/spiders/advanced.md index 955706b..1244c9e 100644 --- a/agent-skill/Scrapling-Skill/references/spiders/advanced.md +++ b/agent-skill/Scrapling-Skill/references/spiders/advanced.md @@ -9,6 +9,7 @@ The spider system uses three class attributes to control how aggressively it cra | `concurrent_requests` | `4` | Maximum number of requests being processed at the same time | | `concurrent_requests_per_domain` | `0` | Maximum concurrent requests per domain (0 = no per-domain limit) | | `download_delay` | `0.0` | Seconds to wait before each request | +| `robots_txt_obey` | `False` | Respect robots.txt rules (Disallow, Crawl-delay, Request-rate) | ```python class PoliteSpider(Spider): @@ -218,6 +219,7 @@ print(f"Requests: {stats.requests_count}") print(f"Failed: {stats.failed_requests_count}") print(f"Blocked: {stats.blocked_requests_count}") print(f"Offsite filtered: {stats.offsite_requests_count}") +print(f"Robots.txt disallowed: {stats.robots_disallowed_count}") print(f"Items scraped: {stats.items_scraped}") print(f"Items dropped: {stats.items_dropped}") print(f"Response bytes: {stats.response_bytes}") diff --git a/agent-skill/Scrapling-Skill/references/spiders/architecture.md b/agent-skill/Scrapling-Skill/references/spiders/architecture.md index 93b059b..9976f31 100644 --- a/agent-skill/Scrapling-Skill/references/spiders/architecture.md +++ b/agent-skill/Scrapling-Skill/references/spiders/architecture.md @@ -10,7 +10,7 @@ Here's what happens step by step when you run a spider: 1. The **Spider** produces the first batch of `Request` objects. By default, it creates one request for each URL in `start_urls`, but you can override `start_requests()` for custom logic. 2. The **Scheduler** receives requests and places them in a priority queue, and creates fingerprints for them. Higher-priority requests are dequeued first. -3. The **Crawler Engine** asks the **Scheduler** to dequeue the next request, respecting concurrency limits (global and per-domain) and download delays. Once the **Crawler Engine** receives the request, it passes it to the **Session Manager**, which routes it to the correct session based on the request's `sid` (session ID). +3. The **Crawler Engine** asks the **Scheduler** to dequeue the next request, respecting concurrency limits (global and per-domain) and download delays. If `robots_txt_obey` is enabled, the engine checks the domain's robots.txt rules before proceeding -- disallowed requests are dropped silently. Once the **Crawler Engine** receives the request, it passes it to the **Session Manager**, which routes it to the correct session based on the request's `sid` (session ID). 4. The **session** fetches the page and returns a [Response](../fetching/choosing.md#response-object) object to the **Crawler Engine**. The engine records statistics and checks for blocked responses. If the response is blocked, the engine retries the request up to `max_blocked_retries` times. Of course, the blocking detection and the retry logic for blocked requests can be customized. 5. The **Crawler Engine** passes the [Response](../fetching/choosing.md#response-object) to the request's callback. The callback either yields a dictionary, which gets treated as a scraped item, or a follow-up request, which gets sent to the scheduler for queuing. 6. The cycle repeats from step 2 until the scheduler is empty and no tasks are active, or the spider is paused. @@ -82,6 +82,7 @@ If you're coming from Scrapy, here's how Scrapling's spider system maps: | Blocked detection | Through custom middlewares | Built-in `is_blocked()` + `retry_blocked_request()` hooks | | Concurrency | `CONCURRENT_REQUESTS` setting | `concurrent_requests` class attribute | | Domain filtering | `allowed_domains` | `allowed_domains` | +| Robots.txt | `ROBOTSTXT_OBEY` setting | `robots_txt_obey` class attribute | | Pause/Resume | `JOBDIR` setting | `crawldir` constructor argument | | Export | Feed exports | `result.items.to_json()` / `to_jsonl()` or custom through hooks | | Running | `scrapy crawl spider_name` | `MySpider().start()` | diff --git a/agent-skill/Scrapling-Skill/references/spiders/getting-started.md b/agent-skill/Scrapling-Skill/references/spiders/getting-started.md index 7420079..665fd7f 100644 --- a/agent-skill/Scrapling-Skill/references/spiders/getting-started.md +++ b/agent-skill/Scrapling-Skill/references/spiders/getting-started.md @@ -137,3 +137,28 @@ Subdomains are matched automatically, so setting `allowed_domains = {"example.co When a request is filtered out, it's counted in `stats.offsite_requests_count` so you can see how many were dropped. +## Robots.txt Compliance + +Set `robots_txt_obey = True` to make the spider respect robots.txt rules before crawling any domain: + +```python +class PoliteSpider(Spider): + name = "polite" + start_urls = ["https://example.com"] + robots_txt_obey = True + + async def parse(self, response: Response): + for link in response.css("a::attr(href)").getall(): + yield response.follow(link, callback=self.parse) +``` + +When enabled, the spider will: + +1. **Pre-fetch robots.txt** for all domains in `start_urls` before the crawl begins (concurrently). +2. **Check every request** against the domain's robots.txt `Disallow` rules. Disallowed requests are silently dropped and counted in `stats.robots_disallowed_count`. +3. **Respect `Crawl-delay` and `Request-rate` directives** by taking the maximum of the directive and your configured `download_delay`. This means robots.txt delays never reduce your configured delay, only increase it when needed. + +Robots.txt files are fetched using the spider's default session and cached per domain for the entire crawl. Domains discovered mid-crawl (not in `start_urls`) have their robots.txt fetched on the first request to that domain. + +**Note:** `robots_txt_obey` is turned off by default. It does not affect your concurrency settings -- only the delay between requests is adjusted. + diff --git a/docs/README_AR.md b/docs/README_AR.md index 01b6d4b..7cc7193 100644 --- a/docs/README_AR.md +++ b/docs/README_AR.md @@ -209,6 +209,7 @@ MySpider().start() - 💾 **إيقاف واستئناف**: استمرارية الزحف القائمة على Checkpoint. اضغط Ctrl+C للإيقاف بسلاسة؛ أعد التشغيل للاستئناف من حيث توقفت. - 📡 **وضع Streaming**: بث العناصر المستخرجة فور وصولها عبر `async for item in spider.stream()` مع إحصائيات فورية - مثالي لواجهات المستخدم وخطوط الأنابيب وعمليات الزحف الطويلة. - 🛡️ **كشف الطلبات المحظورة**: كشف تلقائي وإعادة محاولة للطلبات المحظورة مع منطق قابل للتخصيص. +- 🤖 **الامتثال لـ robots.txt**: خيار `robots_txt_obey` الاختياري الذي يحترم توجيهات `Disallow` و `Crawl-delay` و `Request-rate` مع التخزين المؤقت لكل نطاق. - 📦 **تصدير مدمج**: صدّر النتائج عبر الخطافات وخط الأنابيب الخاص بك أو JSON/JSONL المدمج مع `result.items.to_json()` / `result.items.to_jsonl()` على التوالي. ### جلب متقدم للمواقع مع دعم الجلسات diff --git a/docs/README_CN.md b/docs/README_CN.md index 61cd255..bfc0b6f 100644 --- a/docs/README_CN.md +++ b/docs/README_CN.md @@ -209,6 +209,7 @@ MySpider().start() - 💾 **暂停与恢复**：基于 Checkpoint 的爬取持久化。按 Ctrl+C 优雅关闭；重启后从上次停止的地方继续。 - 📡 **Streaming 模式**：通过 `async for item in spider.stream()` 以实时统计 Streaming 抓取的数据--非常适合 UI、管道和长时间运行的爬取。 - 🛡️ **被阻止请求检测**：自动检测并重试被阻止的请求，支持自定义逻辑。 +- 🤖 **robots.txt 合规**：可选的 `robots_txt_obey` 标志，支持 `Disallow`、`Crawl-delay` 和 `Request-rate` 指令，并按域名缓存。 - 📦 **内置导出**：通过钩子和您自己的管道导出结果，或使用内置的 JSON/JSONL，分别通过 `result.items.to_json()`/`result.items.to_jsonl()`。 ### 支持 Session 的高级网站获取 diff --git a/docs/README_DE.md b/docs/README_DE.md index 1548260..fbbd7e9 100644 --- a/docs/README_DE.md +++ b/docs/README_DE.md @@ -209,6 +209,7 @@ MySpider().start() - 💾 **Pause & Resume**: Checkpoint-basierte Crawl-Persistenz. Drücken Sie Strg+C für ein kontrolliertes Herunterfahren; starten Sie neu, um dort fortzufahren, wo Sie aufgehört haben. - 📡 **Streaming-Modus**: Gescrapte Elemente in Echtzeit streamen über `async for item in spider.stream()` mit Echtzeit-Statistiken -- ideal für UI, Pipelines und lang laufende Crawls. - 🛡️ **Erkennung blockierter Anfragen**: Automatische Erkennung und Wiederholung blockierter Anfragen mit anpassbarer Logik. +- 🤖 **robots.txt-Konformität**: Optionales `robots_txt_obey`-Flag, das `Disallow`-, `Crawl-delay`- und `Request-rate`-Direktiven mit domainbasiertem Caching respektiert. - 📦 **Integrierter Export**: Ergebnisse über Hooks und Ihre eigene Pipeline oder den integrierten JSON/JSONL-Export mit `result.items.to_json()` / `result.items.to_jsonl()` exportieren. ### Erweitertes Website-Abrufen mit Session-Unterstützung diff --git a/docs/README_ES.md b/docs/README_ES.md index 746da96..fba23a0 100644 --- a/docs/README_ES.md +++ b/docs/README_ES.md @@ -209,6 +209,7 @@ MySpider().start() - 💾 **Pause & Resume**: Persistencia de rastreo basada en Checkpoint. Presiona Ctrl+C para un cierre ordenado; reinicia para continuar desde donde lo dejaste. - 📡 **Modo Streaming**: Transmite elementos extraídos a medida que llegan con `async for item in spider.stream()` con estadísticas en tiempo real - ideal para UI, pipelines y rastreos de larga duración. - 🛡️ **Detección de Solicitudes Bloqueadas**: Detección automática y reintento de solicitudes bloqueadas con lógica personalizable. +- 🤖 **Cumplimiento de robots.txt**: Flag opcional `robots_txt_obey` que respeta las directivas `Disallow`, `Crawl-delay` y `Request-rate` con caché por dominio. - 📦 **Exportación Integrada**: Exporta resultados a través de hooks y tu propio pipeline o el JSON/JSONL integrado con `result.items.to_json()` / `result.items.to_jsonl()` respectivamente. ### Obtención Avanzada de Sitios Web con Soporte de Session diff --git a/docs/README_FR.md b/docs/README_FR.md index ec2fd97..d001bbb 100644 --- a/docs/README_FR.md +++ b/docs/README_FR.md @@ -209,6 +209,7 @@ MySpider().start() - 💾 **Pause & Reprise** : Persistance du crawl basée sur des checkpoints. Appuyez sur Ctrl+C pour un arrêt gracieux ; redémarrez pour reprendre là où vous vous étiez arrêté. - 📡 **Mode streaming** : Diffusez les éléments scrapés en temps réel via `async for item in spider.stream()` avec des statistiques en temps réel - idéal pour les UI, pipelines et crawls de longue durée. - 🛡️ **Détection des requêtes bloquées** : Détection automatique et réessai des requêtes bloquées avec une logique personnalisable. +- 🤖 **Conformité robots.txt** : Flag optionnel `robots_txt_obey` qui respecte les directives `Disallow`, `Crawl-delay` et `Request-rate` avec mise en cache par domaine. - 📦 **Export intégré** : Exportez les résultats via des hooks et votre propre pipeline ou l'export JSON/JSONL intégré avec `result.items.to_json()` / `result.items.to_jsonl()` respectivement. ### Récupération avancée de sites web avec support de sessions diff --git a/docs/README_JP.md b/docs/README_JP.md index 7b6632a..f5c6d37 100644 --- a/docs/README_JP.md +++ b/docs/README_JP.md @@ -209,6 +209,7 @@ MySpider().start() - 💾 **Pause & Resume**：Checkpoint ベースのクロール永続化。Ctrl+C で正常にシャットダウン；再起動すると中断したところから再開。 - 📡 **Streaming モード**：`async for item in spider.stream()` でリアルタイム統計とともにスクレイプされたアイテムを Streaming で受信 - UI、パイプライン、長時間実行クロールに最適。 - 🛡️ **ブロックされたリクエストの検出**：カスタマイズ可能なロジックによるブロックされたリクエストの自動検出とリトライ。 +- 🤖 **robots.txt 準拠**：オプションの `robots_txt_obey` フラグで `Disallow`、`Crawl-delay`、`Request-rate` ディレクティブをドメインごとのキャッシュで遵守。 - 📦 **組み込みエクスポート**：フックや独自のパイプライン、または組み込みの JSON/JSONL で結果をエクスポート。それぞれ`result.items.to_json()` / `result.items.to_jsonl()`を使用。 ### Session サポート付き高度なウェブサイト取得 diff --git a/docs/README_KR.md b/docs/README_KR.md index 1e53fe6..020a3ef 100644 --- a/docs/README_KR.md +++ b/docs/README_KR.md @@ -209,6 +209,7 @@ MySpider().start() - 💾 **일시정지 & 재개**: 체크포인트 기반의 크롤링 영속화. Ctrl+C로 정상 종료하고, 재시작하면 중단된 지점부터 이어갑니다. - 📡 **스트리밍 모드**: `async for item in spider.stream()`으로 스크레이핑된 아이템을 실시간 통계와 함께 스트리밍으로 수신 - UI, 파이프라인, 장시간 크롤링에 적합합니다. - 🛡️ **차단된 요청 감지**: 커스텀 로직을 통한 차단된 요청의 자동 감지 및 재시도를 지원합니다. +- 🤖 **robots.txt 준수**: 선택적 `robots_txt_obey` 플래그로 `Disallow`, `Crawl-delay`, `Request-rate` 지시문을 도메인별 캐싱과 함께 준수합니다. - 📦 **내장 내보내기**: 훅이나 자체 파이프라인, 또는 내장 JSON/JSONL로 결과를 내보냅니다. 각각 `result.items.to_json()` / `result.items.to_jsonl()`을 사용합니다. ### 세션을 지원하는 고급 웹사이트 가져오기 diff --git a/docs/README_RU.md b/docs/README_RU.md index 7233bd5..fdef497 100644 --- a/docs/README_RU.md +++ b/docs/README_RU.md @@ -212,6 +212,7 @@ MySpider().start() - 💾 **Pause & Resume**: Persistence обхода на основе Checkpoint'ов. Нажмите Ctrl+C для мягкой остановки; перезапустите, чтобы продолжить с того места, где вы остановились. - 📡 **Режим Streaming**: Стримьте извлечённые элементы по мере их поступления через `async for item in spider.stream()` со статистикой в реальном времени - идеально для UI, конвейеров и длительных обходов. - 🛡️ **Обнаружение заблокированных запросов**: Автоматическое обнаружение и повторная отправка заблокированных запросов с настраиваемой логикой. +- 🤖 **Соответствие robots.txt**: Опциональный флаг `robots_txt_obey`, который учитывает директивы `Disallow`, `Crawl-delay` и `Request-rate` с кэшированием по доменам. - 📦 **Встроенный экспорт**: Экспортируйте результаты через хуки и собственный конвейер или встроенный JSON/JSONL с `result.items.to_json()` / `result.items.to_jsonl()` соответственно. ### Продвинутая загрузка сайтов с поддержкой Session diff --git a/docs/index.md b/docs/index.md index 01d9921..9ff2d04 100644 --- a/docs/index.md +++ b/docs/index.md @@ -99,6 +99,7 @@ MySpider().start() - 💾 **Pause & Resume**: Checkpoint-based crawl persistence. Press Ctrl+C for a graceful shutdown; restart to resume from where you left off. - 📡 **Streaming Mode**: Stream scraped items as they arrive via `async for item in spider.stream()` with real-time stats - ideal for UI, pipelines, and long-running crawls. - 🛡️ **Blocked Request Detection**: Automatic detection and retry of blocked requests with customizable logic. +- 🤖 **Robots.txt Compliance**: Optional `robots_txt_obey` flag that respects `Disallow`, `Crawl-delay`, and `Request-rate` directives with per-domain caching. - 📦 **Built-in Export**: Export results through hooks and your own pipeline or the built-in JSON/JSONL with `result.items.to_json()` / `result.items.to_jsonl()` respectively. ### Advanced Websites Fetching with Session Support diff --git a/docs/spiders/advanced.md b/docs/spiders/advanced.md index c35c2b1..9b1f5b9 100644 --- a/docs/spiders/advanced.md +++ b/docs/spiders/advanced.md @@ -17,6 +17,7 @@ The spider system uses three class attributes to control how aggressively it cra | `concurrent_requests` | `4` | Maximum number of requests being processed at the same time | | `concurrent_requests_per_domain` | `0` | Maximum concurrent requests per domain (0 = no per-domain limit) | | `download_delay` | `0.0` | Seconds to wait before each request | +| `robots_txt_obey` | `False` | Respect robots.txt rules (Disallow, Crawl-delay, Request-rate) | ```python class PoliteSpider(Spider): @@ -234,6 +235,7 @@ print(f"Requests: {stats.requests_count}") print(f"Failed: {stats.failed_requests_count}") print(f"Blocked: {stats.blocked_requests_count}") print(f"Offsite filtered: {stats.offsite_requests_count}") +print(f"Robots.txt disallowed: {stats.robots_disallowed_count}") print(f"Items scraped: {stats.items_scraped}") print(f"Items dropped: {stats.items_dropped}") print(f"Response bytes: {stats.response_bytes}") diff --git a/docs/spiders/architecture.md b/docs/spiders/architecture.md index 4ccfad2..82d388f 100644 --- a/docs/spiders/architecture.md +++ b/docs/spiders/architecture.md @@ -19,7 +19,7 @@ Here's what happens step by step when you run a spider without many details: 1. The **Spider** produces the first batch of `Request` objects. By default, it creates one request for each URL in `start_urls`, but you can override `start_requests()` for custom logic. 2. The **Scheduler** receives requests and places them in a priority queue, and creates fingerprints for them. Higher-priority requests are dequeued first. -3. The **Crawler Engine** asks the **Scheduler** to dequeue the next request, respecting concurrency limits (global and per-domain) and download delays. Once the **Crawler Engine** receives the request, it passes it to the **Session Manager**, which routes it to the correct session based on the request's `sid` (session ID). +3. The **Crawler Engine** asks the **Scheduler** to dequeue the next request, respecting concurrency limits (global and per-domain) and download delays. If `robots_txt_obey` is enabled, the engine checks the domain's robots.txt rules before proceeding -- disallowed requests are dropped silently. Once the **Crawler Engine** receives the request, it passes it to the **Session Manager**, which routes it to the correct session based on the request's `sid` (session ID). 4. The **session** fetches the page and returns a [Response](../fetching/choosing.md#response-object) object to the **Crawler Engine**. The engine records statistics and checks for blocked responses. If the response is blocked, the engine retries the request up to `max_blocked_retries` times. Of course, the blocking detection and the retry logic for blocked requests can be customized. 5. The **Crawler Engine** passes the [Response](../fetching/choosing.md#response-object) to the request's callback. The callback either yields a dictionary, which gets treated as a scraped item, or a follow-up request, which gets sent to the scheduler for queuing. 6. The cycle repeats from step 2 until the scheduler is empty and no tasks are active, or the spider is paused. @@ -91,6 +91,7 @@ If you're coming from Scrapy, here's how Scrapling's spider system maps: | Blocked detection | Through custom middlewares | Built-in `is_blocked()` + `retry_blocked_request()` hooks | | Concurrency | `CONCURRENT_REQUESTS` setting | `concurrent_requests` class attribute | | Domain filtering | `allowed_domains` | `allowed_domains` | +| Robots.txt | `ROBOTSTXT_OBEY` setting | `robots_txt_obey` class attribute | | Pause/Resume | `JOBDIR` setting | `crawldir` constructor argument | | Export | Feed exports | `result.items.to_json()` / `to_jsonl()` or custom through hooks | | Running | `scrapy crawl spider_name` | `MySpider().start()` | diff --git a/docs/spiders/getting-started.md b/docs/spiders/getting-started.md index e619268..9847c98 100644 --- a/docs/spiders/getting-started.md +++ b/docs/spiders/getting-started.md @@ -149,6 +149,31 @@ Subdomains are matched automatically, so setting `allowed_domains = {"example.co When a request is filtered out, it's counted in `stats.offsite_requests_count` so you can see how many were dropped. +## Robots.txt Compliance + +Set `robots_txt_obey = True` to make the spider respect robots.txt rules before crawling any domain: + +```python +class PoliteSpider(Spider): + name = "polite" + start_urls = ["https://example.com"] + robots_txt_obey = True + + async def parse(self, response: Response): + for link in response.css("a::attr(href)").getall(): + yield response.follow(link, callback=self.parse) +``` + +When enabled, the spider will: + +1. **Pre-fetch robots.txt** for all domains in `start_urls` before the crawl begins (concurrently). +2. **Check every request** against the domain's robots.txt `Disallow` rules. Disallowed requests are silently dropped and counted in `stats.robots_disallowed_count`. +3. **Respect `Crawl-delay` and `Request-rate` directives** by taking the maximum of the directive and your configured `download_delay`. This means robots.txt delays never reduce your configured delay, only increase it when needed. + +Robots.txt files are fetched using the spider's default session and cached per domain for the entire crawl. Domains discovered mid-crawl (not in `start_urls`) have their robots.txt fetched on the first request to that domain. + +**Note:** `robots_txt_obey` is turned off by default to avoid surprising behavior. If you enable it, it does not affect your concurrency settings (`concurrent_requests`, `concurrent_requests_per_domain`) -- only the delay between requests is adjusted. + ## What's Next Now that you have the basics, you can explore: diff --git a/pyproject.toml b/pyproject.toml index 8e468fa..af8f37a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta" [project] name = "scrapling" # Static version instead of a dynamic version so we can get better layer caching while building docker, check the docker file to understand -version = "0.4.3" +version = "0.4.4" description = "Scrapling is an undetectable, powerful, flexible, high-performance Python library that makes Web Scraping easy and effortless as it should be!" readme = {file = "README.md", content-type = "text/markdown"} license = {file = "LICENSE"} @@ -63,22 +63,23 @@ classifiers = [ dependencies = [ "lxml>=6.0.2", "cssselect>=1.4.0", - "orjson>=3.11.7", + "orjson>=3.11.8", "tld>=0.13.2", "w3lib>=2.4.1", - "typing_extensions", + "typing_extensions" ] [project.optional-dependencies] fetchers = [ "click>=8.3.0", - "curl_cffi>=0.14.0", + "curl_cffi>=0.15.0", "playwright==1.58.0", "patchright==1.58.2", "browserforge>=1.2.4", - "apify-fingerprint-datapoints>=0.11.0", + "apify-fingerprint-datapoints>=0.12.0", "msgspec>=0.20.0", - "anyio>=4.12.1" + "anyio>=4.12.1", + "protego>=0.6.0", ] ai = [ "mcp>=1.26.0", diff --git a/scrapling/__init__.py b/scrapling/__init__.py index e95acff..5d20532 100644 --- a/scrapling/__init__.py +++ b/scrapling/__init__.py @@ -1,5 +1,5 @@ __author__ = "Karim Shoair (karim.shoair@pm.me)" -__version__ = "0.4.3" +__version__ = "0.4.4" __copyright__ = "Copyright (c) 2024 Karim Shoair" from typing import Any, TYPE_CHECKING diff --git a/scrapling/engines/_browsers/_base.py b/scrapling/engines/_browsers/_base.py index 16a995d..2d42b08 100644 --- a/scrapling/engines/_browsers/_base.py +++ b/scrapling/engines/_browsers/_base.py @@ -48,9 +48,8 @@ from scrapling.engines.constants import STEALTH_ARGS, HARMFUL_ARGS, DEFAULT_ARGS class SyncSession: _config: "PlaywrightConfig | StealthConfig" _context_options: Dict[str, Any] - - def _build_context_with_proxy(self, proxy: Optional[ProxyType] = None) -> Dict[str, Any]: - raise NotImplementedError # pragma: no cover + if TYPE_CHECKING: + _build_context_with_proxy: Callable[..., Dict[str, Any]] def __init__(self, max_pages: int = 1): self.max_pages = max_pages @@ -197,11 +196,14 @@ class SyncSession: context_options = self._build_context_with_proxy(proxy) context: BrowserContext = self.browser.new_context(**context_options) + page_info = None try: context = self._initialize_context(self._config, context) page_info = self._get_page(timeout, extra_headers, disable_resources, blocked_domains, context=context) yield page_info finally: + if page_info is not None and page_info in self.page_pool.pages: + self.page_pool.pages.remove(page_info) context.close() else: # Standard mode: use PagePool with persistent context @@ -216,9 +218,8 @@ class SyncSession: class AsyncSession: _config: "PlaywrightConfig | StealthConfig" _context_options: Dict[str, Any] - - def _build_context_with_proxy(self, proxy: Optional[ProxyType] = None) -> Dict[str, Any]: - raise NotImplementedError # pragma: no cover + if TYPE_CHECKING: + _build_context_with_proxy: Callable[..., Dict[str, Any]] def __init__(self, max_pages: int = 1): self.max_pages = max_pages @@ -382,6 +383,7 @@ class AsyncSession: context_options = self._build_context_with_proxy(proxy) context: AsyncBrowserContext = await self.browser.new_context(**context_options) + page_info = None try: context = await self._initialize_context(self._config, context) page_info = await self._get_page( @@ -389,6 +391,8 @@ class AsyncSession: ) yield page_info finally: + if page_info is not None and page_info in self.page_pool.pages: + self.page_pool.pages.remove(page_info) await context.close() else: # Standard mode: use PagePool with persistent context diff --git a/scrapling/engines/static.py b/scrapling/engines/static.py index 1f4b09b..a962ccf 100644 --- a/scrapling/engines/static.py +++ b/scrapling/engines/static.py @@ -250,6 +250,7 @@ class _SyncSessionLogic(_ConfigurationLogic): request_args = self._merge_request_args(stealth=stealth, proxy=proxy, **kwargs) try: response = session.request(method, **request_args) + assert response is not None result = ResponseFactory.from_http_request(response, selector_config, meta={"proxy": proxy}) return result except CurlError as e: # pragma: no cover diff --git a/scrapling/spiders/engine.py b/scrapling/spiders/engine.py index d77f838..deac28f 100644 --- a/scrapling/spiders/engine.py +++ b/scrapling/spiders/engine.py @@ -1,15 +1,17 @@ import json import pprint from pathlib import Path +from urllib.parse import urlparse import anyio from anyio import Path as AsyncPath from anyio import create_task_group, CapacityLimiter, create_memory_object_stream, EndOfStream from scrapling.core.utils import log -from scrapling.spiders.request import Request from scrapling.spiders.scheduler import Scheduler from scrapling.spiders.session import SessionManager +from scrapling.spiders.request import Request, Response +from scrapling.spiders.robotstxt import RobotsTxtManager from scrapling.spiders.result import CrawlStats, ItemList from scrapling.spiders.checkpoint import CheckpointManager, CheckpointData from scrapling.core._types import Dict, Union, Optional, TYPE_CHECKING, Any, AsyncGenerator @@ -41,10 +43,22 @@ class CrawlerEngine: ) self.stats = CrawlStats() + if self.spider.robots_txt_obey: + + async def _fetch_robots(url: str, sid: str) -> Response: + return await self.session_manager.fetch(Request(url, sid=sid)) + + self._robots_manager: Optional[RobotsTxtManager] = RobotsTxtManager(_fetch_robots) + else: + self._robots_manager = None + self._global_limiter = CapacityLimiter(spider.concurrent_requests) self._domain_limiters: dict[str, CapacityLimiter] = {} self._allowed_domains: set[str] = spider.allowed_domains or set() + if self.spider.robots_txt_obey: + self._domain_delays: dict[str, float] = {} + self._active_tasks: int = 0 self._running: bool = False self._items: ItemList = ItemList() @@ -68,11 +82,42 @@ class CrawlerEngine: return True return False + async def _get_domain_delay(self, request: Request) -> float: + """Resolve the effective download delay for a domain. + + Takes the max of the spider's configured delay and any robots.txt + directives (Crawl-delay / Request-rate). Result is cached per domain. + """ + robots_manager = self._robots_manager + if robots_manager is None: + return self.spider.download_delay + + domain = request.domain + + if domain in self._domain_delays: + return self._domain_delays[domain] + + # For domains covered by _prefetch_robots_txt this is a local parser read. + # Domains discovered mid-crawl (not in start_urls) will fetch here. + c_delay, r_rate = await robots_manager.get_delay_directives(request.url, request.sid) + + delay = self.spider.download_delay + + if r_rate: + req_count, period = r_rate + if req_count > 0: + delay = max(delay, period / req_count) + + if c_delay is not None: + delay = max(delay, c_delay) + + self._domain_delays[domain] = delay + return delay + def _rate_limiter(self, domain: str) -> CapacityLimiter: """Get or create a per-domain concurrency limiter if enabled, otherwise use the global limiter.""" if self.spider.concurrent_requests_per_domain: - if domain not in self._domain_limiters: - self._domain_limiters[domain] = CapacityLimiter(self.spider.concurrent_requests_per_domain) + self._domain_limiters.setdefault(domain, CapacityLimiter(self.spider.concurrent_requests_per_domain)) return self._domain_limiters[domain] return self._global_limiter @@ -87,9 +132,19 @@ class CrawlerEngine: async def _process_request(self, request: Request) -> None: """Download and process a single request.""" + if self._robots_manager: + can_fetch = await self._robots_manager.can_fetch(request.url, request.sid) + if not can_fetch: + self.stats.robots_disallowed_count += 1 + log.info(f"Request disallowed by robots.txt: {request.url}") + return + delay = await self._get_domain_delay(request) + else: + delay = self.spider.download_delay + async with self._rate_limiter(request.domain): - if self.spider.download_delay: - await anyio.sleep(self.spider.download_delay) + if delay: + await anyio.sleep(delay) if request._session_kwargs.get("proxy"): self.stats.proxies.append(request._session_kwargs["proxy"]) @@ -219,6 +274,25 @@ class CrawlerEngine: return True + async def _prefetch_robots_txt(self) -> None: + """Pre-warm the robots.txt cache before the crawl loop starts. + + Extracts unique domains from start_urls, preserving the original scheme. + """ + if not self._robots_manager or not self.spider.start_urls: + return + + # Deduplicate by netloc, preserving the scheme from the first URL per domain + seen: set[str] = set() + seed_urls: list[str] = [] + for url in self.spider.start_urls: + parsed = urlparse(url) + if parsed.netloc not in seen: + seen.add(parsed.netloc) + seed_urls.append(f"{parsed.scheme}://{parsed.netloc}/") + + await self._robots_manager.prefetch(seed_urls, self.session_manager.default_session_id) + async def crawl(self) -> CrawlStats: """Run the spider and return CrawlStats.""" self._running = True @@ -227,6 +301,9 @@ class CrawlerEngine: self._pause_requested = False self._force_stop = False self.stats = CrawlStats(start_time=anyio.current_time()) + self._domain_limiters.clear() + if self._robots_manager: + self._domain_delays.clear() # Check for existing checkpoint resuming = (await self._restore_from_checkpoint()) if self._checkpoint_system_enabled else False @@ -238,6 +315,8 @@ class CrawlerEngine: self.stats.download_delay = self.spider.download_delay await self.spider.on_start(resuming=resuming) + await self._prefetch_robots_txt() + try: if not resuming: async for request in self.spider.start_requests(): diff --git a/scrapling/spiders/result.py b/scrapling/spiders/result.py index 08a7658..b374152 100644 --- a/scrapling/spiders/result.py +++ b/scrapling/spiders/result.py @@ -47,6 +47,7 @@ class CrawlStats: concurrent_requests_per_domain: int = 0 failed_requests_count: int = 0 offsite_requests_count: int = 0 + robots_disallowed_count: int = 0 response_bytes: int = 0 items_scraped: int = 0 items_dropped: int = 0 @@ -95,6 +96,7 @@ class CrawlStats: "sessions_requests_count": self.sessions_requests_count, "failed_requests_count": self.failed_requests_count, "offsite_requests_count": self.offsite_requests_count, + "robots_disallowed_count": self.robots_disallowed_count, "blocked_requests_count": self.blocked_requests_count, "response_status_count": self.response_status_count, "response_bytes": self.response_bytes, diff --git a/scrapling/spiders/robotstxt.py b/scrapling/spiders/robotstxt.py new file mode 100644 index 0000000..18f4263 --- /dev/null +++ b/scrapling/spiders/robotstxt.py @@ -0,0 +1,77 @@ +from urllib.parse import urlparse + +from anyio import create_task_group +from protego import Protego + +from scrapling.core._types import Dict, Optional, Callable, Awaitable +from scrapling.core.utils import log + + +class RobotsTxtManager: + """Manages fetching, parsing, and caching of robots.txt files.""" + + def __init__(self, fetch_fn: Callable[[str, str], Awaitable]): + self._fetch_fn = fetch_fn + self._cache: Dict[str, Protego] = {} + + async def _get_parser(self, url: str, sid: str) -> Protego: + parsed = urlparse(url) + domain = parsed.netloc + + if domain in self._cache: + return self._cache[domain] + + scheme = parsed.scheme or "https" + robots_url = f"{scheme}://{domain}/robots.txt" + content = "" + try: + response = await self._fetch_fn(robots_url, sid) + if response.status == 200: + content = response.body.decode(response.encoding, errors="replace") + except Exception as e: + log.warning(f"Failed to fetch robots.txt for {domain}: {e}") + + try: + parser = Protego.parse(content) + except Exception as e: + log.warning(f"Failed to parse robots.txt for {domain}: {e}") + parser = Protego.parse("") + + self._cache[domain] = parser + return parser + + async def can_fetch(self, url: str, sid: str) -> bool: + """Check if a URL can be fetched according to the domain's robots.txt. + + :param url: The full URL to check + :param sid: Session ID for fetching robots.txt if not yet cached + """ + parser = await self._get_parser(url, sid) + return parser.can_fetch(url, "*") + + async def get_delay_directives(self, url: str, sid: str) -> tuple[Optional[float], Optional[tuple[int, int]]]: + """Return both crawl-delay and request-rate in a single parser lookup. + + :param url: Any URL on the domain to check + :param sid: Session ID for fetching robots.txt if not yet cached + """ + parser = await self._get_parser(url, sid) + c_delay = parser.crawl_delay("*") + rate = parser.request_rate("*") + return ( + float(c_delay) if c_delay is not None else None, + (rate.requests, rate.seconds) if rate is not None else None, + ) + + async def prefetch(self, urls: list[str], sid: str) -> None: + """Pre-warm the robots.txt cache for a list of seed URLs concurrently. + + :param urls: Seed URLs whose domains should be pre-fetched (one per domain). + :param sid: Session ID to use for the robots.txt fetch requests. + """ + if not urls: + return + log.debug(f"Pre-fetching robots.txt for {len(urls)} domain(s)") + async with create_task_group() as tg: + for url in urls: + tg.start_soon(self._get_parser, url, sid) diff --git a/scrapling/spiders/spider.py b/scrapling/spiders/spider.py index 4f38912..6aaa24f 100644 --- a/scrapling/spiders/spider.py +++ b/scrapling/spiders/spider.py @@ -72,6 +72,9 @@ class Spider(ABC): start_urls: list[str] = [] allowed_domains: Set[str] = set() + # Robots.txt compliance + robots_txt_obey: bool = False + # Concurrency settings concurrent_requests: int = 4 concurrent_requests_per_domain: int = 0 diff --git a/server.json b/server.json index 796f94b..6ef6d9b 100644 --- a/server.json +++ b/server.json @@ -14,12 +14,12 @@ "mimeType": "image/png" } ], - "version": "0.4.3", + "version": "0.4.4", "packages": [ { "registryType": "pypi", "identifier": "scrapling", - "version": "0.4.3", + "version": "0.4.4", "runtimeHint": "uvx", "packageArguments": [ { diff --git a/setup.cfg b/setup.cfg index eb133a5..7dcdf68 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = scrapling -version = 0.4.3 +version = 0.4.4 author = Karim Shoair author_email = karim.shoair@pm.me description = Scrapling is an undetectable, powerful, flexible, high-performance Python library that makes Web Scraping easy and effortless as it should be! diff --git a/tests/fetchers/test_pages.py b/tests/fetchers/test_pages.py index 69dc4ba..b726c74 100644 --- a/tests/fetchers/test_pages.py +++ b/tests/fetchers/test_pages.py @@ -84,6 +84,15 @@ class TestPagePool: with pytest.raises(RuntimeError): pool.add_page(Mock()) + def test_proxy_rotation_pool_leak(self): + pool = PagePool(max_pages=1) + page_info = pool.add_page(Mock()) + assert pool.pages_count == 1 + pool.pages.remove(page_info) + assert pool.pages_count == 0 + pool.add_page(Mock()) + assert pool.pages_count == 1 + def test_cleanup_error_pages(self): diff --git a/tests/spiders/test_engine.py b/tests/spiders/test_engine.py index b7bfd0f..74cf875 100644 --- a/tests/spiders/test_engine.py +++ b/tests/spiders/test_engine.py @@ -8,6 +8,7 @@ import pytest from scrapling.spiders.engine import CrawlerEngine, _dump from scrapling.spiders.request import Request +from scrapling.spiders.robotstxt import RobotsTxtManager from scrapling.spiders.session import SessionManager from scrapling.spiders.result import CrawlStats, ItemList from scrapling.spiders.checkpoint import CheckpointData @@ -22,10 +23,11 @@ from scrapling.core._types import Any, Dict, Set, AsyncGenerator class MockResponse: """Minimal Response stand-in.""" - def __init__(self, status: int = 200, body: bytes = b"ok", url: str = "https://example.com"): + def __init__(self, status: int = 200, body: bytes = b"ok", url: str = "https://example.com", encoding: str = "utf-8"): self.status = status self.body = body self.url = url + self.encoding = encoding self.request: Any = None self.meta: Dict[str, Any] = {} @@ -83,6 +85,8 @@ class MockSpider: is_blocked_fn=None, on_scraped_item_fn=None, retry_blocked_request_fn=None, + robots_txt_obey: bool = False, + start_urls: list[str] | None = None, ): self.concurrent_requests = concurrent_requests self.concurrent_requests_per_domain = concurrent_requests_per_domain @@ -93,6 +97,8 @@ class MockSpider: self.fp_include_headers = fp_include_headers self.fp_keep_fragments = fp_keep_fragments self.name = "test_spider" + self.robots_txt_obey = robots_txt_obey + self.start_urls = start_urls or [] # Tracking lists self.on_start_calls: list[dict] = [] @@ -912,3 +918,71 @@ class TestPauseDuringCrawl: await engine.crawl() assert engine.paused is False + + +# --------------------------------------------------------------------------- +# Tests: _prefetch_robots_txt +# --------------------------------------------------------------------------- + + +class TestPrefetchRobotsTxt: + """_prefetch_robots_txt warms the robots.txt cache before the crawl loop.""" + + @staticmethod + def _make_counting_fetch(): + """Return (fetch_fn, calls_list) where calls_list records every (url, sid) pair.""" + calls: list[tuple[str, str]] = [] + + async def _fetch(url: str, sid: str): + calls.append((url, sid)) + return MockResponse(status=200, body=b"", url=url) + + return _fetch, calls + + @pytest.mark.asyncio + async def test_prefetch_uses_start_urls(self): + fetch_fn, calls = self._make_counting_fetch() + spider = MockSpider(robots_txt_obey=True, start_urls=["https://example.com/page1"]) + engine = _make_engine(spider=spider) + engine._robots_manager = RobotsTxtManager(fetch_fn) + + await engine._prefetch_robots_txt() + + assert len(calls) == 1 + assert calls[0][0] == "https://example.com/robots.txt" + + @pytest.mark.asyncio + async def test_prefetch_noop_when_robots_disabled(self): + fetch_fn, calls = self._make_counting_fetch() + spider = MockSpider(robots_txt_obey=False) + engine = _make_engine(spider=spider) + + assert engine._robots_manager is None + + await engine._prefetch_robots_txt() + + assert calls == [] + + @pytest.mark.asyncio + async def test_prefetch_noop_when_start_urls_empty(self): + fetch_fn, calls = self._make_counting_fetch() + spider = MockSpider(robots_txt_obey=True, start_urls=[]) + engine = _make_engine(spider=spider) + engine._robots_manager = RobotsTxtManager(fetch_fn) + + await engine._prefetch_robots_txt() + + assert calls == [] + + @pytest.mark.asyncio + async def test_prefetch_deduplicates_same_domain_in_start_urls(self): + fetch_fn, calls = self._make_counting_fetch() + spider = MockSpider(robots_txt_obey=True, start_urls=["https://example.com/a", "https://example.com/b"]) + engine = _make_engine(spider=spider) + engine._robots_manager = RobotsTxtManager(fetch_fn) + + await engine._prefetch_robots_txt() + + # set of Request.domain values deduplicates to one task per domain + assert len(calls) == 1 + assert calls[0][0] == "https://example.com/robots.txt" diff --git a/tests/spiders/test_robotstxt.py b/tests/spiders/test_robotstxt.py new file mode 100644 index 0000000..c4050c9 --- /dev/null +++ b/tests/spiders/test_robotstxt.py @@ -0,0 +1,506 @@ +"""Tests for RobotsTxtManager.""" + +import asyncio + +import pytest + +from scrapling.spiders.robotstxt import RobotsTxtManager + + +# --------------------------------------------------------------------------- +# Fixtures and helpers +# --------------------------------------------------------------------------- + + +class MockResponse: + """Minimal response stub matching the shape _get_parser expects.""" + + def __init__(self, status: int = 200, body: bytes = b"", encoding: str = "utf-8"): + self.status = status + self.body = body + self.encoding = encoding + + +def make_fetch_fn(status: int = 200, content: str = "", encoding: str = "utf-8"): + """Return an async fetch callable that returns a fixed response. + + Attaches a `.calls` list so tests can assert how many times it was invoked + and with which arguments. + """ + calls: list[tuple] = [] + + async def _fetch(url: str, sid: str) -> MockResponse: + calls.append((url, sid)) + return MockResponse(status=status, body=content.encode(encoding), encoding=encoding) + + _fetch.calls = calls # type: ignore[attr-defined] + return _fetch + + +# --------------------------------------------------------------------------- +# Shared robots.txt fixtures +# --------------------------------------------------------------------------- + +ROBOTS_BASIC = """\ +User-agent: * +Disallow: /admin/ +Crawl-delay: 2 +""" + +ROBOTS_WITH_RATE = """\ +User-agent: * +Request-rate: 1/10 +Disallow: /private/ +""" + +ROBOTS_ALLOW_OVERRIDE = """\ +User-agent: * +Disallow: /secret/ +Allow: /secret/public.html +""" + +ROBOTS_DISALLOW_ALL = """\ +User-agent: * +Disallow: / +""" + + +# --------------------------------------------------------------------------- +# Tests: can_fetch +# --------------------------------------------------------------------------- + + +class TestCanFetch: + @pytest.mark.asyncio + async def test_allowed_url_returns_true(self): + mgr = RobotsTxtManager(make_fetch_fn(content=ROBOTS_BASIC)) + + assert await mgr.can_fetch("https://example.com/products", "s1") is True + + @pytest.mark.asyncio + async def test_disallowed_url_returns_false(self): + mgr = RobotsTxtManager(make_fetch_fn(content=ROBOTS_BASIC)) + + assert await mgr.can_fetch("https://example.com/admin/", "s1") is False + + @pytest.mark.asyncio + async def test_disallowed_subpath_returns_false(self): + mgr = RobotsTxtManager(make_fetch_fn(content=ROBOTS_BASIC)) + + assert await mgr.can_fetch("https://example.com/admin/users", "s1") is False + + @pytest.mark.asyncio + async def test_root_url_is_allowed(self): + mgr = RobotsTxtManager(make_fetch_fn(content=ROBOTS_BASIC)) + + assert await mgr.can_fetch("https://example.com/", "s1") is True + + @pytest.mark.asyncio + async def test_allow_directive_overrides_disallow(self): + mgr = RobotsTxtManager(make_fetch_fn(content=ROBOTS_ALLOW_OVERRIDE)) + + assert await mgr.can_fetch("https://example.com/secret/public.html", "s1") is True + assert await mgr.can_fetch("https://example.com/secret/private.html", "s1") is False + + @pytest.mark.asyncio + async def test_disallow_all_blocks_every_path(self): + mgr = RobotsTxtManager(make_fetch_fn(content=ROBOTS_DISALLOW_ALL)) + + assert await mgr.can_fetch("https://example.com/", "s1") is False + assert await mgr.can_fetch("https://example.com/page", "s1") is False + assert await mgr.can_fetch("https://example.com/a/b/c", "s1") is False + + @pytest.mark.asyncio + async def test_empty_robots_allows_everything(self): + mgr = RobotsTxtManager(make_fetch_fn(content="")) + + assert await mgr.can_fetch("https://example.com/anything", "s1") is True + assert await mgr.can_fetch("https://example.com/admin/secret", "s1") is True + + @pytest.mark.asyncio + async def test_non_200_response_allows_everything(self): + for status in [403, 404, 500, 503]: + mgr = RobotsTxtManager(make_fetch_fn(status=status)) + result = await mgr.can_fetch("https://example.com/page", "s1") + assert result is True, f"Expected True for HTTP {status}" + + @pytest.mark.asyncio + async def test_fetch_error_allows_everything(self): + async def failing_fetch(url: str, sid: str) -> MockResponse: + raise ConnectionError("network failure") + + mgr = RobotsTxtManager(failing_fetch) + + assert await mgr.can_fetch("https://example.com/page", "s1") is True + + @pytest.mark.asyncio + async def test_wildcard_path_pattern(self): + content = "User-agent: *\nDisallow: /*.pdf$" + mgr = RobotsTxtManager(make_fetch_fn(content=content)) + + assert await mgr.can_fetch("https://example.com/report.pdf", "s1") is False + assert await mgr.can_fetch("https://example.com/report.html", "s1") is True + + @pytest.mark.asyncio + async def test_returns_bool(self): + mgr = RobotsTxtManager(make_fetch_fn(content=ROBOTS_BASIC)) + result = await mgr.can_fetch("https://example.com/", "s1") + assert isinstance(result, bool) + + +# --------------------------------------------------------------------------- +# Tests: get_delay_directives +# --------------------------------------------------------------------------- + + +class TestGetDelayDirectives: + @pytest.mark.asyncio + async def test_returns_crawl_delay_when_set(self): + mgr = RobotsTxtManager(make_fetch_fn(content=ROBOTS_BASIC)) + + c_delay, r_rate = await mgr.get_delay_directives("https://example.com/", "s1") + + assert c_delay == 2.0 + assert isinstance(c_delay, float) + assert r_rate is None + + @pytest.mark.asyncio + async def test_returns_request_rate_when_set(self): + mgr = RobotsTxtManager(make_fetch_fn(content=ROBOTS_WITH_RATE)) + + c_delay, r_rate = await mgr.get_delay_directives("https://example.com/", "s1") + + assert c_delay is None + assert r_rate is not None + assert r_rate == (1, 1) + + @pytest.mark.asyncio + async def test_returns_both_none_when_not_set(self): + content = "User-agent: *\nDisallow: /admin/" + mgr = RobotsTxtManager(make_fetch_fn(content=content)) + + c_delay, r_rate = await mgr.get_delay_directives("https://example.com/", "s1") + + assert c_delay is None + assert r_rate is None + + @pytest.mark.asyncio + async def test_returns_both_none_for_empty_robots(self): + mgr = RobotsTxtManager(make_fetch_fn(content="")) + + c_delay, r_rate = await mgr.get_delay_directives("https://example.com/", "s1") + + assert c_delay is None + assert r_rate is None + + @pytest.mark.asyncio + async def test_returns_both_none_on_fetch_error(self): + async def failing_fetch(url: str, sid: str) -> MockResponse: + raise ConnectionError("network failure") + + mgr = RobotsTxtManager(failing_fetch) + + c_delay, r_rate = await mgr.get_delay_directives("https://example.com/", "s1") + + assert c_delay is None + assert r_rate is None + + @pytest.mark.asyncio + async def test_fractional_crawl_delay(self): + content = "User-agent: *\nCrawl-delay: 0.5" + mgr = RobotsTxtManager(make_fetch_fn(content=content)) + + c_delay, _ = await mgr.get_delay_directives("https://example.com/", "s1") + + assert c_delay == 0.5 + + @pytest.mark.asyncio + async def test_url_path_does_not_affect_result(self): + mgr = RobotsTxtManager(make_fetch_fn(content=ROBOTS_BASIC)) + + r1 = await mgr.get_delay_directives("https://example.com/", "s1") + r2 = await mgr.get_delay_directives("https://example.com/deep/path/page.html", "s1") + + assert r1 == r2 + + +# --------------------------------------------------------------------------- +# Tests: caching behaviour +# --------------------------------------------------------------------------- + + +class TestCachingBehaviour: + @pytest.mark.asyncio + async def test_second_call_same_domain_uses_cache(self): + fetch_fn = make_fetch_fn(content=ROBOTS_BASIC) + mgr = RobotsTxtManager(fetch_fn) + + await mgr.can_fetch("https://example.com/page1", "s1") + await mgr.can_fetch("https://example.com/page2", "s1") + + assert len(fetch_fn.calls) == 1 + + @pytest.mark.asyncio + async def test_all_methods_share_cache(self): + fetch_fn = make_fetch_fn(content=ROBOTS_BASIC) + mgr = RobotsTxtManager(fetch_fn) + + await mgr.can_fetch("https://example.com/", "s1") + await mgr.get_delay_directives("https://example.com/", "s1") + + assert len(fetch_fn.calls) == 1 + + @pytest.mark.asyncio + async def test_different_sids_share_cache_entry(self): + """robots.txt is domain-level — different sessions share the same cached parser.""" + fetch_fn = make_fetch_fn(content=ROBOTS_BASIC) + mgr = RobotsTxtManager(fetch_fn) + + await mgr.can_fetch("https://example.com/", "s1") + await mgr.can_fetch("https://example.com/", "s2") + + assert len(fetch_fn.calls) == 1 + + @pytest.mark.asyncio + async def test_different_domains_use_separate_cache_entries(self): + fetch_fn = make_fetch_fn(content=ROBOTS_BASIC) + mgr = RobotsTxtManager(fetch_fn) + + await mgr.can_fetch("https://example.com/", "s1") + await mgr.can_fetch("https://other.com/", "s1") + + assert len(fetch_fn.calls) == 2 + + @pytest.mark.asyncio + async def test_cache_keyed_by_domain_not_path(self): + fetch_fn = make_fetch_fn(content=ROBOTS_BASIC) + mgr = RobotsTxtManager(fetch_fn) + + await mgr.can_fetch("https://example.com/a/b/c", "s1") + await mgr.can_fetch("https://example.com/x/y/z", "s1") + await mgr.can_fetch("https://example.com/admin/", "s1") + + assert len(fetch_fn.calls) == 1 + + @pytest.mark.asyncio + async def test_sid_is_passed_to_fetch_fn(self): + fetch_fn = make_fetch_fn(content=ROBOTS_BASIC) + mgr = RobotsTxtManager(fetch_fn) + + await mgr.can_fetch("https://example.com/", "my_session") + + _, received_sid = fetch_fn.calls[0] + assert received_sid == "my_session" + + +# --------------------------------------------------------------------------- +# Tests: robots.txt URL construction +# --------------------------------------------------------------------------- + + +class TestRobotsTxtUrlConstruction: + @pytest.mark.asyncio + async def test_http_scheme_preserved(self): + fetch_fn = make_fetch_fn(content="") + mgr = RobotsTxtManager(fetch_fn) + + await mgr.can_fetch("http://example.com/page", "s1") + + fetched_url, _ = fetch_fn.calls[0] + assert fetched_url == "http://example.com/robots.txt" + + @pytest.mark.asyncio + async def test_https_scheme_preserved(self): + fetch_fn = make_fetch_fn(content="") + mgr = RobotsTxtManager(fetch_fn) + + await mgr.can_fetch("https://example.com/page", "s1") + + fetched_url, _ = fetch_fn.calls[0] + assert fetched_url == "https://example.com/robots.txt" + + @pytest.mark.asyncio + async def test_fetched_at_domain_root_regardless_of_request_path(self): + fetch_fn = make_fetch_fn(content="") + mgr = RobotsTxtManager(fetch_fn) + + await mgr.can_fetch("https://example.com/deep/nested/path/page.html", "s1") + + fetched_url, _ = fetch_fn.calls[0] + assert fetched_url == "https://example.com/robots.txt" + + @pytest.mark.asyncio + async def test_port_included_in_url(self): + fetch_fn = make_fetch_fn(content="") + mgr = RobotsTxtManager(fetch_fn) + + await mgr.can_fetch("http://example.com:8080/page", "s1") + + fetched_url, _ = fetch_fn.calls[0] + assert fetched_url == "http://example.com:8080/robots.txt" + + @pytest.mark.asyncio + async def test_different_ports_treated_as_different_domains(self): + fetch_fn = make_fetch_fn(content="") + mgr = RobotsTxtManager(fetch_fn) + + await mgr.can_fetch("http://example.com:8000/page", "s1") + await mgr.can_fetch("http://example.com:9000/page", "s1") + + assert len(fetch_fn.calls) == 2 + urls = [call[0] for call in fetch_fn.calls] + assert "http://example.com:8000/robots.txt" in urls + assert "http://example.com:9000/robots.txt" in urls + + +# --------------------------------------------------------------------------- +# Tests: encoding +# --------------------------------------------------------------------------- + + +class TestEncoding: + @pytest.mark.asyncio + async def test_non_utf8_body_decoded_with_response_encoding(self): + content = "User-agent: *\nDisallow: /admin/\nCrawl-delay: 3" + body = content.encode("latin-1") + + async def fetch_fn(url: str, sid: str) -> MockResponse: + return MockResponse(status=200, body=body, encoding="latin-1") + + mgr = RobotsTxtManager(fetch_fn) + c_delay, _ = await mgr.get_delay_directives("https://example.com/", "s1") + + assert c_delay == 3.0 + + @pytest.mark.asyncio + async def test_bytes_body_decoded_correctly(self): + content = "User-agent: *\nDisallow: /private/" + body = content.encode("utf-8") + + async def fetch_fn(url: str, sid: str) -> MockResponse: + return MockResponse(status=200, body=body, encoding="utf-8") + + mgr = RobotsTxtManager(fetch_fn) + + assert await mgr.can_fetch("https://example.com/private/", "s1") is False + assert await mgr.can_fetch("https://example.com/public/", "s1") is True + + + +# --------------------------------------------------------------------------- +# Tests: concurrent access +# --------------------------------------------------------------------------- + + +class TestCacheAndConcurrency: + @pytest.mark.asyncio + async def test_cached_domain_not_refetched(self): + """Once a domain is cached, subsequent calls return the cached parser without fetching.""" + fetch_count = 0 + + async def counting_fetch(url: str, sid: str) -> MockResponse: + nonlocal fetch_count + fetch_count += 1 + return MockResponse(status=200, body=ROBOTS_BASIC.encode(), encoding="utf-8") + + mgr = RobotsTxtManager(counting_fetch) + + # First call fetches and caches + await mgr.can_fetch("https://example.com/page1", "s1") + # Subsequent calls hit the cache + for i in range(7): + await mgr.can_fetch(f"https://example.com/page{i + 2}", "s1") + + assert fetch_count == 1 + + @pytest.mark.asyncio + async def test_concurrent_calls_different_domains_fetch_independently(self): + fetch_count = 0 + + async def slow_fetch(url: str, sid: str) -> MockResponse: + nonlocal fetch_count + fetch_count += 1 + await asyncio.sleep(0.01) + return MockResponse(status=200, body=b"", encoding="utf-8") + + mgr = RobotsTxtManager(slow_fetch) + + await asyncio.gather( + mgr.can_fetch("https://alpha.com/", "s1"), + mgr.can_fetch("https://beta.com/", "s1"), + mgr.can_fetch("https://gamma.com/", "s1"), + ) + + assert fetch_count == 3 + + @pytest.mark.asyncio + async def test_concurrent_calls_consistent_results(self): + """All concurrent callers should see the same allow/disallow result.""" + mgr = RobotsTxtManager(make_fetch_fn(content=ROBOTS_BASIC)) + + results = await asyncio.gather(*[ + mgr.can_fetch("https://example.com/admin/", "s1") + for _ in range(6) + ]) + + assert all(r is False for r in results) + + @pytest.mark.asyncio + async def test_different_sids_share_cache_after_first_fetch(self): + """After the first fetch, all sessions share the cached parser regardless of sid.""" + fetch_count = 0 + + async def counting_fetch(url: str, sid: str) -> MockResponse: + nonlocal fetch_count + fetch_count += 1 + return MockResponse(status=200, body=b"", encoding="utf-8") + + mgr = RobotsTxtManager(counting_fetch) + + # First call fetches and caches + await mgr.can_fetch("https://example.com/", "s1") + # s2 and s3 hit the cache — no additional fetches + await mgr.can_fetch("https://example.com/", "s2") + await mgr.can_fetch("https://example.com/", "s3") + + assert fetch_count == 1 + + +# --------------------------------------------------------------------------- +# Tests: prefetch +# --------------------------------------------------------------------------- + + +class TestPrefetch: + @pytest.mark.asyncio + async def test_prefetch_fetches_all_domains(self): + fetch_fn = make_fetch_fn(content=ROBOTS_BASIC) + mgr = RobotsTxtManager(fetch_fn) + + await mgr.prefetch(["https://a.com/", "https://b.com/", "https://c.com/"], "s1") + + assert len(fetch_fn.calls) == 3 + fetched = {url for url, _ in fetch_fn.calls} + assert fetched == {"https://a.com/robots.txt", "https://b.com/robots.txt", "https://c.com/robots.txt"} + + @pytest.mark.asyncio + async def test_prefetch_warms_cache_for_subsequent_calls(self): + fetch_fn = make_fetch_fn(content=ROBOTS_BASIC) + mgr = RobotsTxtManager(fetch_fn) + + await mgr.prefetch(["https://example.com/"], "s1") + assert len(fetch_fn.calls) == 1 + + # Any subsequent call for the same domain hits the cache + await mgr.can_fetch("https://example.com/products", "s1") + await mgr.can_fetch("https://example.com/products", "s2") + assert len(fetch_fn.calls) == 1 + + @pytest.mark.asyncio + async def test_prefetch_empty_list_is_noop(self): + fetch_fn = make_fetch_fn(content=ROBOTS_BASIC) + mgr = RobotsTxtManager(fetch_fn) + + await mgr.prefetch([], "s1") + + assert len(fetch_fn.calls) == 0