diff --git a/README.md b/README.md index ab5fb8a..408cfef 100644 --- a/README.md +++ b/README.md @@ -215,6 +215,7 @@ MySpider().start() - 📡 **Streaming Mode**: Stream scraped items as they arrive via `async for item in spider.stream()` with real-time stats - ideal for UI, pipelines, and long-running crawls. - 🛡️ **Blocked Request Detection**: Automatic detection and retry of blocked requests with customizable logic. - 🤖 **Robots.txt Compliance**: Optional `robots_txt_obey` flag that respects `Disallow`, `Crawl-delay`, and `Request-rate` directives with per-domain caching. +- 🧪 **Development Mode**: Cache responses to disk on the first run and replay them on subsequent runs - iterate on your `parse()` logic without re-hitting the target servers. - 📦 **Built-in Export**: Export results through hooks and your own pipeline or the built-in JSON/JSONL with `result.items.to_json()` / `result.items.to_jsonl()` respectively. ### Advanced Websites Fetching with Session Support diff --git a/agent-skill/Scrapling-Skill.zip b/agent-skill/Scrapling-Skill.zip index b03d956..c43d0df 100644 Binary files a/agent-skill/Scrapling-Skill.zip and b/agent-skill/Scrapling-Skill.zip differ diff --git a/agent-skill/Scrapling-Skill/SKILL.md b/agent-skill/Scrapling-Skill/SKILL.md index 6b3cbf7..23f1de4 100644 --- a/agent-skill/Scrapling-Skill/SKILL.md +++ b/agent-skill/Scrapling-Skill/SKILL.md @@ -1,7 +1,7 @@ --- name: scrapling-official description: Scrape web pages using Scrapling with anti-bot bypass (like Cloudflare Turnstile), stealth headless browsing, spiders framework, adaptive scraping, and JavaScript rendering. Use when asked to scrape, crawl, or extract data from websites; web_fetch fails; the site has anti-bot protections; write Python code to scrape/crawl; or write spiders. -version: "0.4.4" +version: "0.4.5" license: Complete terms in LICENSE.txt metadata: homepage: "https://scrapling.readthedocs.io/en/latest/index.html" @@ -40,7 +40,7 @@ Blazing fast crawls with real-time stats and streaming. Built by Web Scrapers fo Create a virtual Python environment through any way available, like `venv`, then inside the environment do: -`pip install "scrapling[all]>=0.4.4"` +`pip install "scrapling[all]>=0.4.5"` Then do this to download all the browsers' dependencies: @@ -104,7 +104,7 @@ Those options are shared between the 4 HTTP request commands: | --proxy | TEXT | Proxy URL in format "http://username:password@host:port" | | -s, --css-selector | TEXT | CSS selector to extract specific content from the page. It returns all matches. | | -p, --params | TEXT | Query parameters in format "key=value" (can be used multiple times) | -| --follow-redirects / --no-follow-redirects | None | Whether to follow redirects (default: True) | +| --follow-redirects / --no-follow-redirects | None | Whether to follow redirects (default: "safe", rejects redirects to internal/private IPs) | | --verify / --no-verify | None | Whether to verify SSL certificates (default: True) | | --impersonate | TEXT | Browser to impersonate. Can be a single browser (e.g., Chrome) or a comma-separated list for random selection (e.g., Chrome, Firefox, Safari). | | --stealthy-headers / --no-stealthy-headers | None | Use stealthy browser headers (default: True) | @@ -302,6 +302,8 @@ QuotesSpider(crawldir="./crawl_data").start() ``` Press Ctrl+C to pause gracefully - progress is saved automatically. Later, when you start the spider again, pass the same `crawldir`, and it will resume from where it stopped. +While iterating on a spider's `parse()` logic, set `development_mode = True` on the spider class to cache responses to disk on the first run and replay them on subsequent runs - so you can re-run the spider as many times as you want without re-hitting the target servers. The cache lives in `.scrapling_cache/{spider.name}/` by default and can be overridden with `development_cache_dir`. Don't ship a spider with this enabled. + ### Advanced Parsing & Navigation ```python from scrapling.fetchers import Fetcher diff --git a/agent-skill/Scrapling-Skill/examples/README.md b/agent-skill/Scrapling-Skill/examples/README.md index 0c2f031..344dc0f 100644 --- a/agent-skill/Scrapling-Skill/examples/README.md +++ b/agent-skill/Scrapling-Skill/examples/README.md @@ -9,7 +9,7 @@ All examples collect **all 100 quotes across 10 pages**. Make sure Scrapling is installed: ```bash -pip install "scrapling[all]>=0.4.4" +pip install "scrapling[all]>=0.4.5" scrapling install --force ``` diff --git a/agent-skill/Scrapling-Skill/references/fetching/static.md b/agent-skill/Scrapling-Skill/references/fetching/static.md index fa4d2a5..4115483 100644 --- a/agent-skill/Scrapling-Skill/references/fetching/static.md +++ b/agent-skill/Scrapling-Skill/references/fetching/static.md @@ -15,7 +15,7 @@ All methods for making requests here share some arguments, so let's discuss them - **url**: The targeted URL - **stealthy_headers**: If enabled (default), it creates and adds real browser headers. It also sets a Google referer header. -- **follow_redirects**: As the name implies, tell the fetcher to follow redirections. **Enabled by default** +- **follow_redirects**: Controls redirect behavior. **Defaults to `"safe"`**, which follows redirects but rejects those targeting internal/private IPs (SSRF protection). Pass `True` to follow all redirects without restriction, or `False` to disable redirects entirely. - **timeout**: The number of seconds to wait for each request to be finished. **Defaults to 30 seconds**. - **retries**: The number of retries that the fetcher will do for failed requests. **Defaults to three retries**. - **retry_delay**: Number of seconds to wait between retry attempts. **Defaults to 1 second**. @@ -50,7 +50,7 @@ Examples are the best way to explain this: >>> from scrapling.fetchers import Fetcher >>> # Basic GET >>> page = Fetcher.get('https://example.com') ->>> page = Fetcher.get('https://scrapling.requestcatcher.com/get', stealthy_headers=True, follow_redirects=True) +>>> page = Fetcher.get('https://scrapling.requestcatcher.com/get', stealthy_headers=True) >>> page = Fetcher.get('https://scrapling.requestcatcher.com/get', proxy='http://username:password@localhost:8030') >>> # With parameters >>> page = Fetcher.get('https://example.com/search', params={'q': 'query'}) @@ -69,7 +69,7 @@ And for asynchronous requests, it's a small adjustment >>> from scrapling.fetchers import AsyncFetcher >>> # Basic GET >>> page = await AsyncFetcher.get('https://example.com') ->>> page = await AsyncFetcher.get('https://scrapling.requestcatcher.com/get', stealthy_headers=True, follow_redirects=True) +>>> page = await AsyncFetcher.get('https://scrapling.requestcatcher.com/get', stealthy_headers=True) >>> page = await AsyncFetcher.get('https://scrapling.requestcatcher.com/get', proxy='http://username:password@localhost:8030') >>> # With parameters >>> page = await AsyncFetcher.get('https://example.com/search', params={'q': 'query'}) @@ -105,7 +105,7 @@ The `page` object in all cases is a [Response](choosing.md#response-object) obje >>> from scrapling.fetchers import Fetcher >>> # Basic POST >>> page = Fetcher.post('https://scrapling.requestcatcher.com/post', data={'key': 'value'}, params={'q': 'query'}) ->>> page = Fetcher.post('https://scrapling.requestcatcher.com/post', data={'key': 'value'}, stealthy_headers=True, follow_redirects=True) +>>> page = Fetcher.post('https://scrapling.requestcatcher.com/post', data={'key': 'value'}, stealthy_headers=True) >>> page = Fetcher.post('https://scrapling.requestcatcher.com/post', data={'key': 'value'}, proxy='http://username:password@localhost:8030', impersonate="chrome") >>> # Another example of form-encoded data >>> page = Fetcher.post('https://example.com/submit', data={'username': 'user', 'password': 'pass'}, http3=True) @@ -117,7 +117,7 @@ And for asynchronous requests, it's a small adjustment >>> from scrapling.fetchers import AsyncFetcher >>> # Basic POST >>> page = await AsyncFetcher.post('https://scrapling.requestcatcher.com/post', data={'key': 'value'}) ->>> page = await AsyncFetcher.post('https://scrapling.requestcatcher.com/post', data={'key': 'value'}, stealthy_headers=True, follow_redirects=True) +>>> page = await AsyncFetcher.post('https://scrapling.requestcatcher.com/post', data={'key': 'value'}, stealthy_headers=True) >>> page = await AsyncFetcher.post('https://scrapling.requestcatcher.com/post', data={'key': 'value'}, proxy='http://username:password@localhost:8030', impersonate="chrome") >>> # Another example of form-encoded data >>> page = await AsyncFetcher.post('https://example.com/submit', data={'username': 'user', 'password': 'pass'}, http3=True) @@ -129,7 +129,7 @@ And for asynchronous requests, it's a small adjustment >>> from scrapling.fetchers import Fetcher >>> # Basic PUT >>> page = Fetcher.put('https://example.com/update', data={'status': 'updated'}) ->>> page = Fetcher.put('https://example.com/update', data={'status': 'updated'}, stealthy_headers=True, follow_redirects=True, impersonate="chrome") +>>> page = Fetcher.put('https://example.com/update', data={'status': 'updated'}, stealthy_headers=True, impersonate="chrome") >>> page = Fetcher.put('https://example.com/update', data={'status': 'updated'}, proxy='http://username:password@localhost:8030') >>> # Another example of form-encoded data >>> page = Fetcher.put("https://scrapling.requestcatcher.com/put", data={'key': ['value1', 'value2']}) @@ -139,7 +139,7 @@ And for asynchronous requests, it's a small adjustment >>> from scrapling.fetchers import AsyncFetcher >>> # Basic PUT >>> page = await AsyncFetcher.put('https://example.com/update', data={'status': 'updated'}) ->>> page = await AsyncFetcher.put('https://example.com/update', data={'status': 'updated'}, stealthy_headers=True, follow_redirects=True, impersonate="chrome") +>>> page = await AsyncFetcher.put('https://example.com/update', data={'status': 'updated'}, stealthy_headers=True, impersonate="chrome") >>> page = await AsyncFetcher.put('https://example.com/update', data={'status': 'updated'}, proxy='http://username:password@localhost:8030') >>> # Another example of form-encoded data >>> page = await AsyncFetcher.put("https://scrapling.requestcatcher.com/put", data={'key': ['value1', 'value2']}) @@ -149,14 +149,14 @@ And for asynchronous requests, it's a small adjustment ```python >>> from scrapling.fetchers import Fetcher >>> page = Fetcher.delete('https://example.com/resource/123') ->>> page = Fetcher.delete('https://example.com/resource/123', stealthy_headers=True, follow_redirects=True, impersonate="chrome") +>>> page = Fetcher.delete('https://example.com/resource/123', stealthy_headers=True, impersonate="chrome") >>> page = Fetcher.delete('https://example.com/resource/123', proxy='http://username:password@localhost:8030') ``` And for asynchronous requests, it's a small adjustment ```python >>> from scrapling.fetchers import AsyncFetcher >>> page = await AsyncFetcher.delete('https://example.com/resource/123') ->>> page = await AsyncFetcher.delete('https://example.com/resource/123', stealthy_headers=True, follow_redirects=True, impersonate="chrome") +>>> page = await AsyncFetcher.delete('https://example.com/resource/123', stealthy_headers=True, impersonate="chrome") >>> page = await AsyncFetcher.delete('https://example.com/resource/123', proxy='http://username:password@localhost:8030') ``` diff --git a/agent-skill/Scrapling-Skill/references/mcp-server.md b/agent-skill/Scrapling-Skill/references/mcp-server.md index 0a63642..7bbc86b 100644 --- a/agent-skill/Scrapling-Skill/references/mcp-server.md +++ b/agent-skill/Scrapling-Skill/references/mcp-server.md @@ -27,7 +27,7 @@ Fast HTTP GET with browser fingerprint impersonation (TLS, headers). Suitable fo | `retry_delay` | int | 1 | Seconds between retries | | `stealthy_headers` | bool | true | Generate realistic browser headers and Google referer | | `http3` | bool | false | Use HTTP/3 (may conflict with `impersonate`) | -| `follow_redirects` | bool | true | Follow HTTP redirects | +| `follow_redirects` | bool or "safe" | "safe" | Follow redirects. "safe" rejects redirects to internal/private IPs | | `max_redirects` | int | 30 | Max redirects (-1 for unlimited) | | `headers` | dict or null | null | Custom request headers | | `cookies` | dict or null | null | Request cookies | diff --git a/agent-skill/Scrapling-Skill/references/spiders/advanced.md b/agent-skill/Scrapling-Skill/references/spiders/advanced.md index 1244c9e..654e17c 100644 --- a/agent-skill/Scrapling-Skill/references/spiders/advanced.md +++ b/agent-skill/Scrapling-Skill/references/spiders/advanced.md @@ -85,6 +85,49 @@ async def on_start(self, resuming: bool = False): self.logger.info("Starting fresh crawl") ``` +## Development Mode + +When you're iterating on a spider's `parse()` logic, re-hitting the target servers on every run is slow and noisy. Development mode caches every response to disk on the first run and replays them from disk on subsequent runs, so you can tweak your selectors and re-run the spider as many times as you want without making a single network request. + +Enable it by setting `development_mode = True` on your spider: + +```python +class MySpider(Spider): + name = "my_spider" + start_urls = ["https://example.com"] + development_mode = True + + async def parse(self, response: Response): + yield {"title": response.css("title::text").get("")} +``` + +The first run fetches normally and stores each response on disk. Every subsequent run serves the same requests from the cache, skipping the network entirely. + +### Cache Location + +By default, responses are cached in `.scrapling_cache/{spider.name}/` relative to the current working directory (where you ran the spider from, **not** where the spider script lives). You can override the location with `development_cache_dir`: + +```python +class MySpider(Spider): + name = "my_spider" + start_urls = ["https://example.com"] + development_mode = True + development_cache_dir = "/tmp/my_spider_cache" +``` + +### How It Works + +1. **Cache key**: Each response is keyed by the request's fingerprint, so any change to fingerprint-affecting attributes (`fp_include_kwargs`, `fp_include_headers`, `fp_keep_fragments`) will produce a fresh fetch. +2. **Storage format**: One JSON file per response, named `{fingerprint_hex}.json`. The body is base64-encoded so binary content is preserved exactly. Writes are atomic (temp file + rename). +3. **Replay**: On a cache hit, the engine skips the network entirely, including `download_delay`, rate limiting, and the `is_blocked()` retry path. The cached response goes straight to your callback. +4. **Stats**: Cached requests still count toward `requests_count`, `response_bytes`, and the per-status counters, so your stat output looks the same as a normal crawl. Two extra counters, `cache_hits` and `cache_misses`, let you see how the cache performed. + +### Clearing the Cache + +There's no automatic expiration. To force a fresh crawl, delete the cache directory or call the manager's `clear()` method directly. + +**Warning:** Development mode is meant for development, not production. Cached responses never expire, and replay bypasses rate limiting and blocked-request retries. Don't ship a spider with `development_mode = True`. + ## Streaming For long-running spiders or applications that need real-time access to scraped items, use the `stream()` method instead of `start()`: @@ -220,6 +263,8 @@ print(f"Failed: {stats.failed_requests_count}") print(f"Blocked: {stats.blocked_requests_count}") print(f"Offsite filtered: {stats.offsite_requests_count}") print(f"Robots.txt disallowed: {stats.robots_disallowed_count}") +print(f"Cache hits: {stats.cache_hits}") +print(f"Cache misses: {stats.cache_misses}") print(f"Items scraped: {stats.items_scraped}") print(f"Items dropped: {stats.items_dropped}") print(f"Response bytes: {stats.response_bytes}") diff --git a/agent-skill/Scrapling-Skill/references/spiders/architecture.md b/agent-skill/Scrapling-Skill/references/spiders/architecture.md index 9976f31..54186a1 100644 --- a/agent-skill/Scrapling-Skill/references/spiders/architecture.md +++ b/agent-skill/Scrapling-Skill/references/spiders/architecture.md @@ -60,6 +60,10 @@ When a request comes in, the Session Manager routes it to the correct session ba An optional system that, if enabled, saves the crawler's state (pending requests + seen URL fingerprints) to a pickle file on disk. Writes are atomic (temp file + rename) to prevent corruption. Checkpoints are saved periodically at a configurable interval and on graceful shutdown. Upon successful completion (not paused), checkpoint files are automatically cleaned up. +### Response Cache + +An optional cache that, when development mode is enabled, stores every fetched response on disk and replays it on subsequent runs. Each response is keyed by request fingerprint and serialized as JSON (with the body base64-encoded so binary content survives). It's meant for iterating on `parse()` logic without re-hitting the target servers, not for production use. + ### Output Scraped items are collected in an `ItemList` (a list subclass with `to_json()` and `to_jsonl()` export methods). Crawl statistics are tracked in a `CrawlStats` dataclass which contains a lot of useful info. diff --git a/docs/README_AR.md b/docs/README_AR.md index a66ed18..c0de787 100644 --- a/docs/README_AR.md +++ b/docs/README_AR.md @@ -210,6 +210,7 @@ MySpider().start() - 📡 **وضع Streaming**: بث العناصر المستخرجة فور وصولها عبر `async for item in spider.stream()` مع إحصائيات فورية - مثالي لواجهات المستخدم وخطوط الأنابيب وعمليات الزحف الطويلة. - 🛡️ **كشف الطلبات المحظورة**: كشف تلقائي وإعادة محاولة للطلبات المحظورة مع منطق قابل للتخصيص. - 🤖 **الامتثال لـ robots.txt**: خيار `robots_txt_obey` الاختياري الذي يحترم توجيهات `Disallow` و `Crawl-delay` و `Request-rate` مع التخزين المؤقت لكل نطاق. +- 🧪 **وضع التطوير**: تخزين الاستجابات على القرص في التشغيل الأول وإعادة تشغيلها في التشغيلات اللاحقة - كرّر العمل على منطق `parse()` دون الحاجة لإرسال طلبات جديدة إلى الخوادم المستهدفة. - 📦 **تصدير مدمج**: صدّر النتائج عبر الخطافات وخط الأنابيب الخاص بك أو JSON/JSONL المدمج مع `result.items.to_json()` / `result.items.to_jsonl()` على التوالي. ### جلب متقدم للمواقع مع دعم الجلسات diff --git a/docs/README_CN.md b/docs/README_CN.md index d75f32b..6cb9d4e 100644 --- a/docs/README_CN.md +++ b/docs/README_CN.md @@ -210,6 +210,7 @@ MySpider().start() - 📡 **Streaming 模式**：通过 `async for item in spider.stream()` 以实时统计 Streaming 抓取的数据--非常适合 UI、管道和长时间运行的爬取。 - 🛡️ **被阻止请求检测**：自动检测并重试被阻止的请求，支持自定义逻辑。 - 🤖 **robots.txt 合规**：可选的 `robots_txt_obey` 标志，支持 `Disallow`、`Crawl-delay` 和 `Request-rate` 指令，并按域名缓存。 +- 🧪 **开发模式**：首次运行时将响应缓存到磁盘，后续运行时直接回放 - 在不重新请求目标服务器的情况下迭代你的 `parse()` 逻辑。 - 📦 **内置导出**：通过钩子和您自己的管道导出结果，或使用内置的 JSON/JSONL，分别通过 `result.items.to_json()`/`result.items.to_jsonl()`。 ### 支持 Session 的高级网站获取 diff --git a/docs/README_DE.md b/docs/README_DE.md index 5acbf15..82d336f 100644 --- a/docs/README_DE.md +++ b/docs/README_DE.md @@ -210,6 +210,7 @@ MySpider().start() - 📡 **Streaming-Modus**: Gescrapte Elemente in Echtzeit streamen über `async for item in spider.stream()` mit Echtzeit-Statistiken -- ideal für UI, Pipelines und lang laufende Crawls. - 🛡️ **Erkennung blockierter Anfragen**: Automatische Erkennung und Wiederholung blockierter Anfragen mit anpassbarer Logik. - 🤖 **robots.txt-Konformität**: Optionales `robots_txt_obey`-Flag, das `Disallow`-, `Crawl-delay`- und `Request-rate`-Direktiven mit domainbasiertem Caching respektiert. +- 🧪 **Entwicklungsmodus**: Antworten beim ersten Lauf auf der Festplatte zwischenspeichern und bei weiteren Läufen erneut abspielen - iterieren Sie an Ihrer `parse()`-Logik, ohne die Zielserver erneut abzufragen. - 📦 **Integrierter Export**: Ergebnisse über Hooks und Ihre eigene Pipeline oder den integrierten JSON/JSONL-Export mit `result.items.to_json()` / `result.items.to_jsonl()` exportieren. ### Erweitertes Website-Abrufen mit Session-Unterstützung diff --git a/docs/README_ES.md b/docs/README_ES.md index f2cd179..58dccb8 100644 --- a/docs/README_ES.md +++ b/docs/README_ES.md @@ -210,6 +210,7 @@ MySpider().start() - 📡 **Modo Streaming**: Transmite elementos extraídos a medida que llegan con `async for item in spider.stream()` con estadísticas en tiempo real - ideal para UI, pipelines y rastreos de larga duración. - 🛡️ **Detección de Solicitudes Bloqueadas**: Detección automática y reintento de solicitudes bloqueadas con lógica personalizable. - 🤖 **Cumplimiento de robots.txt**: Flag opcional `robots_txt_obey` que respeta las directivas `Disallow`, `Crawl-delay` y `Request-rate` con caché por dominio. +- 🧪 **Modo de Desarrollo**: Almacena las respuestas en disco en la primera ejecución y las reproduce en ejecuciones posteriores - itera sobre tu lógica de `parse()` sin volver a consultar los servidores objetivo. - 📦 **Exportación Integrada**: Exporta resultados a través de hooks y tu propio pipeline o el JSON/JSONL integrado con `result.items.to_json()` / `result.items.to_jsonl()` respectivamente. ### Obtención Avanzada de Sitios Web con Soporte de Session diff --git a/docs/README_FR.md b/docs/README_FR.md index ce5ac11..a452210 100644 --- a/docs/README_FR.md +++ b/docs/README_FR.md @@ -210,6 +210,7 @@ MySpider().start() - 📡 **Mode streaming** : Diffusez les éléments scrapés en temps réel via `async for item in spider.stream()` avec des statistiques en temps réel - idéal pour les UI, pipelines et crawls de longue durée. - 🛡️ **Détection des requêtes bloquées** : Détection automatique et réessai des requêtes bloquées avec une logique personnalisable. - 🤖 **Conformité robots.txt** : Flag optionnel `robots_txt_obey` qui respecte les directives `Disallow`, `Crawl-delay` et `Request-rate` avec mise en cache par domaine. +- 🧪 **Mode développement** : Mettez les réponses en cache sur le disque lors de la première exécution et rejouez-les lors des exécutions suivantes - itérez sur votre logique `parse()` sans solliciter à nouveau les serveurs cibles. - 📦 **Export intégré** : Exportez les résultats via des hooks et votre propre pipeline ou l'export JSON/JSONL intégré avec `result.items.to_json()` / `result.items.to_jsonl()` respectivement. ### Récupération avancée de sites web avec support de sessions diff --git a/docs/README_JP.md b/docs/README_JP.md index 7c89cb3..9a2a29d 100644 --- a/docs/README_JP.md +++ b/docs/README_JP.md @@ -210,6 +210,7 @@ MySpider().start() - 📡 **Streaming モード**：`async for item in spider.stream()` でリアルタイム統計とともにスクレイプされたアイテムを Streaming で受信 - UI、パイプライン、長時間実行クロールに最適。 - 🛡️ **ブロックされたリクエストの検出**：カスタマイズ可能なロジックによるブロックされたリクエストの自動検出とリトライ。 - 🤖 **robots.txt 準拠**：オプションの `robots_txt_obey` フラグで `Disallow`、`Crawl-delay`、`Request-rate` ディレクティブをドメインごとのキャッシュで遵守。 +- 🧪 **開発モード**：初回実行時にレスポンスをディスクにキャッシュし、以降の実行ではそれを再生 - ターゲットサーバーに再リクエストすることなく `parse()` ロジックを反復開発できます。 - 📦 **組み込みエクスポート**：フックや独自のパイプライン、または組み込みの JSON/JSONL で結果をエクスポート。それぞれ`result.items.to_json()` / `result.items.to_jsonl()`を使用。 ### Session サポート付き高度なウェブサイト取得 diff --git a/docs/README_KR.md b/docs/README_KR.md index b74d877..6d79389 100644 --- a/docs/README_KR.md +++ b/docs/README_KR.md @@ -210,6 +210,7 @@ MySpider().start() - 📡 **스트리밍 모드**: `async for item in spider.stream()`으로 스크레이핑된 아이템을 실시간 통계와 함께 스트리밍으로 수신 - UI, 파이프라인, 장시간 크롤링에 적합합니다. - 🛡️ **차단된 요청 감지**: 커스텀 로직을 통한 차단된 요청의 자동 감지 및 재시도를 지원합니다. - 🤖 **robots.txt 준수**: 선택적 `robots_txt_obey` 플래그로 `Disallow`, `Crawl-delay`, `Request-rate` 지시문을 도메인별 캐싱과 함께 준수합니다. +- 🧪 **개발 모드**: 첫 실행 시 응답을 디스크에 캐싱하고 이후 실행에서는 캐시된 응답을 재생합니다 - 대상 서버에 다시 요청하지 않고 `parse()` 로직을 반복 개발할 수 있습니다. - 📦 **내장 내보내기**: 훅이나 자체 파이프라인, 또는 내장 JSON/JSONL로 결과를 내보냅니다. 각각 `result.items.to_json()` / `result.items.to_jsonl()`을 사용합니다. ### 세션을 지원하는 고급 웹사이트 가져오기 diff --git a/docs/README_RU.md b/docs/README_RU.md index 2d636a4..0fa251f 100644 --- a/docs/README_RU.md +++ b/docs/README_RU.md @@ -213,6 +213,7 @@ MySpider().start() - 📡 **Режим Streaming**: Стримьте извлечённые элементы по мере их поступления через `async for item in spider.stream()` со статистикой в реальном времени - идеально для UI, конвейеров и длительных обходов. - 🛡️ **Обнаружение заблокированных запросов**: Автоматическое обнаружение и повторная отправка заблокированных запросов с настраиваемой логикой. - 🤖 **Соответствие robots.txt**: Опциональный флаг `robots_txt_obey`, который учитывает директивы `Disallow`, `Crawl-delay` и `Request-rate` с кэшированием по доменам. +- 🧪 **Режим разработки**: Кэшируйте ответы на диск при первом запуске и воспроизводите их при последующих запусках - итерируйте над логикой `parse()`, не отправляя повторные запросы к целевым серверам. - 📦 **Встроенный экспорт**: Экспортируйте результаты через хуки и собственный конвейер или встроенный JSON/JSONL с `result.items.to_json()` / `result.items.to_jsonl()` соответственно. ### Продвинутая загрузка сайтов с поддержкой Session diff --git a/docs/fetching/static.md b/docs/fetching/static.md index 587ebd1..7d2a522 100644 --- a/docs/fetching/static.md +++ b/docs/fetching/static.md @@ -21,7 +21,7 @@ All methods for making requests here share some arguments, so let's discuss them - **url**: The targeted URL - **stealthy_headers**: If enabled (default), it creates and adds real browser headers. It also sets a Google referer header. -- **follow_redirects**: As the name implies, tell the fetcher to follow redirections. **Enabled by default** +- **follow_redirects**: Controls redirect behavior. **Defaults to `"safe"`**, which follows redirects but rejects those targeting internal/private IPs (SSRF protection). Pass `True` to follow all redirects without restriction, or `False` to disable redirects entirely. - **timeout**: The number of seconds to wait for each request to be finished. **Defaults to 30 seconds**. - **retries**: The number of retries that the fetcher will do for failed requests. **Defaults to three retries**. - **retry_delay**: Number of seconds to wait between retry attempts. **Defaults to 1 second**. @@ -57,7 +57,7 @@ Examples are the best way to explain this: >>> from scrapling.fetchers import Fetcher >>> # Basic GET >>> page = Fetcher.get('https://example.com') ->>> page = Fetcher.get('https://scrapling.requestcatcher.com/get', stealthy_headers=True, follow_redirects=True) +>>> page = Fetcher.get('https://scrapling.requestcatcher.com/get', stealthy_headers=True) >>> page = Fetcher.get('https://scrapling.requestcatcher.com/get', proxy='http://username:password@localhost:8030') >>> # With parameters >>> page = Fetcher.get('https://example.com/search', params={'q': 'query'}) @@ -76,7 +76,7 @@ And for asynchronous requests, it's a small adjustment >>> from scrapling.fetchers import AsyncFetcher >>> # Basic GET >>> page = await AsyncFetcher.get('https://example.com') ->>> page = await AsyncFetcher.get('https://scrapling.requestcatcher.com/get', stealthy_headers=True, follow_redirects=True) +>>> page = await AsyncFetcher.get('https://scrapling.requestcatcher.com/get', stealthy_headers=True) >>> page = await AsyncFetcher.get('https://scrapling.requestcatcher.com/get', proxy='http://username:password@localhost:8030') >>> # With parameters >>> page = await AsyncFetcher.get('https://example.com/search', params={'q': 'query'}) @@ -112,7 +112,7 @@ Needless to say, the `page` object in all cases is [Response](choosing.md#respon >>> from scrapling.fetchers import Fetcher >>> # Basic POST >>> page = Fetcher.post('https://scrapling.requestcatcher.com/post', data={'key': 'value'}, params={'q': 'query'}) ->>> page = Fetcher.post('https://scrapling.requestcatcher.com/post', data={'key': 'value'}, stealthy_headers=True, follow_redirects=True) +>>> page = Fetcher.post('https://scrapling.requestcatcher.com/post', data={'key': 'value'}, stealthy_headers=True) >>> page = Fetcher.post('https://scrapling.requestcatcher.com/post', data={'key': 'value'}, proxy='http://username:password@localhost:8030', impersonate="chrome") >>> # Another example of form-encoded data >>> page = Fetcher.post('https://example.com/submit', data={'username': 'user', 'password': 'pass'}, http3=True) @@ -124,7 +124,7 @@ And for asynchronous requests, it's a small adjustment >>> from scrapling.fetchers import AsyncFetcher >>> # Basic POST >>> page = await AsyncFetcher.post('https://scrapling.requestcatcher.com/post', data={'key': 'value'}) ->>> page = await AsyncFetcher.post('https://scrapling.requestcatcher.com/post', data={'key': 'value'}, stealthy_headers=True, follow_redirects=True) +>>> page = await AsyncFetcher.post('https://scrapling.requestcatcher.com/post', data={'key': 'value'}, stealthy_headers=True) >>> page = await AsyncFetcher.post('https://scrapling.requestcatcher.com/post', data={'key': 'value'}, proxy='http://username:password@localhost:8030', impersonate="chrome") >>> # Another example of form-encoded data >>> page = await AsyncFetcher.post('https://example.com/submit', data={'username': 'user', 'password': 'pass'}, http3=True) @@ -136,7 +136,7 @@ And for asynchronous requests, it's a small adjustment >>> from scrapling.fetchers import Fetcher >>> # Basic PUT >>> page = Fetcher.put('https://example.com/update', data={'status': 'updated'}) ->>> page = Fetcher.put('https://example.com/update', data={'status': 'updated'}, stealthy_headers=True, follow_redirects=True, impersonate="chrome") +>>> page = Fetcher.put('https://example.com/update', data={'status': 'updated'}, stealthy_headers=True, impersonate="chrome") >>> page = Fetcher.put('https://example.com/update', data={'status': 'updated'}, proxy='http://username:password@localhost:8030') >>> # Another example of form-encoded data >>> page = Fetcher.put("https://scrapling.requestcatcher.com/put", data={'key': ['value1', 'value2']}) @@ -146,7 +146,7 @@ And for asynchronous requests, it's a small adjustment >>> from scrapling.fetchers import AsyncFetcher >>> # Basic PUT >>> page = await AsyncFetcher.put('https://example.com/update', data={'status': 'updated'}) ->>> page = await AsyncFetcher.put('https://example.com/update', data={'status': 'updated'}, stealthy_headers=True, follow_redirects=True, impersonate="chrome") +>>> page = await AsyncFetcher.put('https://example.com/update', data={'status': 'updated'}, stealthy_headers=True, impersonate="chrome") >>> page = await AsyncFetcher.put('https://example.com/update', data={'status': 'updated'}, proxy='http://username:password@localhost:8030') >>> # Another example of form-encoded data >>> page = await AsyncFetcher.put("https://scrapling.requestcatcher.com/put", data={'key': ['value1', 'value2']}) @@ -156,14 +156,14 @@ And for asynchronous requests, it's a small adjustment ```python >>> from scrapling.fetchers import Fetcher >>> page = Fetcher.delete('https://example.com/resource/123') ->>> page = Fetcher.delete('https://example.com/resource/123', stealthy_headers=True, follow_redirects=True, impersonate="chrome") +>>> page = Fetcher.delete('https://example.com/resource/123', stealthy_headers=True, impersonate="chrome") >>> page = Fetcher.delete('https://example.com/resource/123', proxy='http://username:password@localhost:8030') ``` And for asynchronous requests, it's a small adjustment ```python >>> from scrapling.fetchers import AsyncFetcher >>> page = await AsyncFetcher.delete('https://example.com/resource/123') ->>> page = await AsyncFetcher.delete('https://example.com/resource/123', stealthy_headers=True, follow_redirects=True, impersonate="chrome") +>>> page = await AsyncFetcher.delete('https://example.com/resource/123', stealthy_headers=True, impersonate="chrome") >>> page = await AsyncFetcher.delete('https://example.com/resource/123', proxy='http://username:password@localhost:8030') ``` diff --git a/docs/index.md b/docs/index.md index 9ff2d04..ae41d6c 100644 --- a/docs/index.md +++ b/docs/index.md @@ -100,6 +100,7 @@ MySpider().start() - 📡 **Streaming Mode**: Stream scraped items as they arrive via `async for item in spider.stream()` with real-time stats - ideal for UI, pipelines, and long-running crawls. - 🛡️ **Blocked Request Detection**: Automatic detection and retry of blocked requests with customizable logic. - 🤖 **Robots.txt Compliance**: Optional `robots_txt_obey` flag that respects `Disallow`, `Crawl-delay`, and `Request-rate` directives with per-domain caching. +- 🧪 **Development Mode**: Cache responses to disk on the first run and replay them on subsequent runs - iterate on your `parse()` logic without re-hitting the target servers. - 📦 **Built-in Export**: Export results through hooks and your own pipeline or the built-in JSON/JSONL with `result.items.to_json()` / `result.items.to_jsonl()` respectively. ### Advanced Websites Fetching with Session Support diff --git a/docs/overview.md b/docs/overview.md index b0d1a4e..c62c7c3 100644 --- a/docs/overview.md +++ b/docs/overview.md @@ -264,7 +264,7 @@ page = Fetcher.get('https://scrapling.requestcatcher.com/get', impersonate="chro With that out of the way, here's how to do all HTTP methods: ```python >>> from scrapling.fetchers import Fetcher ->>> page = Fetcher.get('https://scrapling.requestcatcher.com/get', stealthy_headers=True, follow_redirects=True) +>>> page = Fetcher.get('https://scrapling.requestcatcher.com/get', stealthy_headers=True) >>> page = Fetcher.post('https://scrapling.requestcatcher.com/post', data={'key': 'value'}, proxy='http://username:password@localhost:8030') >>> page = Fetcher.put('https://scrapling.requestcatcher.com/put', data={'key': 'value'}) >>> page = Fetcher.delete('https://scrapling.requestcatcher.com/delete') @@ -272,7 +272,7 @@ With that out of the way, here's how to do all HTTP methods: For Async requests, you will replace the import like below: ```python >>> from scrapling.fetchers import AsyncFetcher ->>> page = await AsyncFetcher.get('https://scrapling.requestcatcher.com/get', stealthy_headers=True, follow_redirects=True) +>>> page = await AsyncFetcher.get('https://scrapling.requestcatcher.com/get', stealthy_headers=True) >>> page = await AsyncFetcher.post('https://scrapling.requestcatcher.com/post', data={'key': 'value'}, proxy='http://username:password@localhost:8030') >>> page = await AsyncFetcher.put('https://scrapling.requestcatcher.com/put', data={'key': 'value'}) >>> page = await AsyncFetcher.delete('https://scrapling.requestcatcher.com/delete') diff --git a/docs/spiders/advanced.md b/docs/spiders/advanced.md index 9b1f5b9..98e930a 100644 --- a/docs/spiders/advanced.md +++ b/docs/spiders/advanced.md @@ -97,6 +97,51 @@ async def on_start(self, resuming: bool = False): self.logger.info("Starting fresh crawl") ``` +## Development Mode + +When you're iterating on a spider's `parse()` logic, re-hitting the target servers on every run is slow and noisy. Development mode caches every response to disk on the first run and replays them from disk on subsequent runs, so you can tweak your selectors and re-run the spider as many times as you want without making a single network request. + +Enable it by setting `development_mode = True` on your spider: + +```python +class MySpider(Spider): + name = "my_spider" + start_urls = ["https://example.com"] + development_mode = True + + async def parse(self, response: Response): + yield {"title": response.css("title::text").get("")} +``` + +The first run fetches normally and stores each response on disk. Every subsequent run serves the same requests from the cache, skipping the network entirely. + +### Cache Location + +By default, responses are cached in `.scrapling_cache/{spider.name}/` relative to the current working directory (where you ran the spider from, **not** where the spider script lives). You can override the location with `development_cache_dir`: + +```python +class MySpider(Spider): + name = "my_spider" + start_urls = ["https://example.com"] + development_mode = True + development_cache_dir = "/tmp/my_spider_cache" +``` + +### How It Works + +1. **Cache key**: Each response is keyed by the request's fingerprint, so any change to fingerprint-affecting attributes (`fp_include_kwargs`, `fp_include_headers`, `fp_keep_fragments`) will produce a fresh fetch. +2. **Storage format**: One JSON file per response, named `{fingerprint_hex}.json`. The body is base64-encoded so binary content is preserved exactly. Writes are atomic (temp file + rename). +3. **Replay**: On a cache hit, the engine skips the network entirely, including `download_delay`, rate limiting, and the `is_blocked()` retry path. The cached response goes straight to your callback. +4. **Stats**: Cached requests still count toward `requests_count`, `response_bytes`, and the per-status counters, so your stat output looks the same as a normal crawl. Two extra counters, `cache_hits` and `cache_misses`, let you see how the cache performed. + +### Clearing the Cache + +There's no automatic expiration. To force a fresh crawl, delete the cache directory or call the manager's `clear()` method directly. + +!!! warning + + Development mode is meant for development, not production. Cached responses never expire, and replay bypasses rate limiting and blocked-request retries. Don't ship a spider with `development_mode = True`. + ## Streaming For long-running spiders or applications that need real-time access to scraped items, use the `stream()` method instead of `start()`: @@ -236,6 +281,8 @@ print(f"Failed: {stats.failed_requests_count}") print(f"Blocked: {stats.blocked_requests_count}") print(f"Offsite filtered: {stats.offsite_requests_count}") print(f"Robots.txt disallowed: {stats.robots_disallowed_count}") +print(f"Cache hits: {stats.cache_hits}") +print(f"Cache misses: {stats.cache_misses}") print(f"Items scraped: {stats.items_scraped}") print(f"Items dropped: {stats.items_dropped}") print(f"Response bytes: {stats.response_bytes}") diff --git a/docs/spiders/architecture.md b/docs/spiders/architecture.md index 82d388f..dc360b8 100644 --- a/docs/spiders/architecture.md +++ b/docs/spiders/architecture.md @@ -69,6 +69,10 @@ When a request comes in, the Session Manager routes it to the correct session ba An optional system that, if enabled, saves the crawler's state (pending requests + seen URL fingerprints) to a pickle file on disk. Writes are atomic (temp file + rename) to prevent corruption. Checkpoints are saved periodically at a configurable interval and on graceful shutdown. Upon successful completion (not paused), checkpoint files are automatically cleaned up. +### Response Cache + +An optional cache that, when development mode is enabled, stores every fetched response on disk and replays it on subsequent runs. Each response is keyed by request fingerprint and serialized as JSON (with the body base64-encoded so binary content survives). It's meant for iterating on `parse()` logic without re-hitting the target servers, not for production use. + ### Output Scraped items are collected in an `ItemList` (a list subclass with `to_json()` and `to_jsonl()` export methods). Crawl statistics are tracked in a `CrawlStats` dataclass which contains a lot of useful info. diff --git a/pyproject.toml b/pyproject.toml index af8f37a..6ac5d9c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta" [project] name = "scrapling" # Static version instead of a dynamic version so we can get better layer caching while building docker, check the docker file to understand -version = "0.4.4" +version = "0.4.5" description = "Scrapling is an undetectable, powerful, flexible, high-performance Python library that makes Web Scraping easy and effortless as it should be!" readme = {file = "README.md", content-type = "text/markdown"} license = {file = "LICENSE"} diff --git a/scrapling/__init__.py b/scrapling/__init__.py index 5d20532..a121db4 100644 --- a/scrapling/__init__.py +++ b/scrapling/__init__.py @@ -1,5 +1,5 @@ __author__ = "Karim Shoair (karim.shoair@pm.me)" -__version__ = "0.4.4" +__version__ = "0.4.5" __copyright__ = "Copyright (c) 2024 Karim Shoair" from typing import Any, TYPE_CHECKING diff --git a/scrapling/core/_shell_signatures.py b/scrapling/core/_shell_signatures.py index d9a42fc..9778318 100644 --- a/scrapling/core/_shell_signatures.py +++ b/scrapling/core/_shell_signatures.py @@ -8,6 +8,7 @@ from scrapling.core._types import ( Optional, SetCookieParam, SelectorWaitStates, + FollowRedirects, ) # Parameter definitions for shell function signatures (defined once at module level) @@ -26,7 +27,7 @@ _REQUESTS_PARAMS = { "headers": Any, "retries": Optional[int], "retry_delay": Optional[int], - "follow_redirects": Optional[bool], + "follow_redirects": Optional[FollowRedirects], "max_redirects": Optional[int], "verify": Optional[bool], "cert": Optional[str | Tuple[str, str]], diff --git a/scrapling/core/_types.py b/scrapling/core/_types.py index 25ddb5b..fc8a075 100644 --- a/scrapling/core/_types.py +++ b/scrapling/core/_types.py @@ -40,6 +40,7 @@ SelectorWaitStates = Literal["attached", "detached", "hidden", "visible"] PageLoadStates = Literal["commit", "domcontentloaded", "load", "networkidle"] extraction_types = Literal["text", "html", "markdown"] StrOrBytes = Union[str, bytes] +FollowRedirects = Union[bool, Literal["safe", "all", "obeycode", "firstonly"]] # Copied from `playwright._impl._api_structures.SetCookieParam` diff --git a/scrapling/core/ai.py b/scrapling/core/ai.py index 208f3e7..9b33821 100644 --- a/scrapling/core/ai.py +++ b/scrapling/core/ai.py @@ -27,6 +27,7 @@ from scrapling.core._types import ( SetCookieParam, extraction_types, SelectorWaitStates, + FollowRedirects, ) SessionType = Literal["dynamic", "stealthy"] @@ -262,7 +263,7 @@ class ScraplingMCPServer: headers: Optional[Mapping[str, Optional[str]]] = None, cookies: Optional[Dict[str, str]] = None, timeout: Optional[int | float] = 30, - follow_redirects: bool = True, + follow_redirects: FollowRedirects = "safe", max_redirects: int = 30, retries: Optional[int] = 3, retry_delay: Optional[int] = 1, @@ -289,7 +290,7 @@ class ScraplingMCPServer: :param headers: Headers to include in the request. :param cookies: Cookies to use in the request. :param timeout: Number of seconds to wait before timing out. - :param follow_redirects: Whether to follow redirects. Defaults to True. + :param follow_redirects: Whether to follow redirects. Defaults to "safe", which follows redirects but rejects those targeting internal/private IPs (SSRF protection). Pass True to follow all redirects without restriction. :param max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited. :param retries: Number of retry attempts. Defaults to 3. :param retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second. @@ -335,7 +336,7 @@ class ScraplingMCPServer: headers: Optional[Mapping[str, Optional[str]]] = None, cookies: Optional[Dict[str, str]] = None, timeout: Optional[int | float] = 30, - follow_redirects: bool = True, + follow_redirects: FollowRedirects = "safe", max_redirects: int = 30, retries: Optional[int] = 3, retry_delay: Optional[int] = 1, @@ -362,7 +363,7 @@ class ScraplingMCPServer: :param headers: Headers to include in the request. :param cookies: Cookies to use in the request. :param timeout: Number of seconds to wait before timing out. - :param follow_redirects: Whether to follow redirects. Defaults to True. + :param follow_redirects: Whether to follow redirects. Defaults to "safe", which follows redirects but rejects those targeting internal/private IPs (SSRF protection). Pass True to follow all redirects without restriction. :param max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited. :param retries: Number of retry attempts. Defaults to 3. :param retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second. diff --git a/scrapling/core/shell.py b/scrapling/core/shell.py index 14a25fe..cb805c6 100644 --- a/scrapling/core/shell.py +++ b/scrapling/core/shell.py @@ -294,7 +294,7 @@ class CurlParser: headers=headers, cookies=cookies, proxy=proxies, - follow_redirects=True, # Scrapling default is True + follow_redirects="safe", # Follows redirects but rejects those to internal/private IPs ) def convert2fetcher(self, curl_command: Request | str) -> Optional[Response]: diff --git a/scrapling/engines/_browsers/_types.py b/scrapling/engines/_browsers/_types.py index 6ab40b9..2741407 100644 --- a/scrapling/engines/_browsers/_types.py +++ b/scrapling/engines/_browsers/_types.py @@ -19,6 +19,7 @@ from scrapling.core._types import ( TypeAlias, SetCookieParam, SelectorWaitStates, + FollowRedirects, ) from scrapling.engines.toolbelt.proxy_rotation import ProxyRotator @@ -39,7 +40,7 @@ class RequestsSession(TypedDict, total=False): headers: Optional[Mapping[str, Optional[str]]] retries: Optional[int] retry_delay: Optional[int] - follow_redirects: Optional[bool] + follow_redirects: Optional[FollowRedirects] max_redirects: Optional[int] verify: Optional[bool] cert: Optional[str | Tuple[str, str]] diff --git a/scrapling/engines/static.py b/scrapling/engines/static.py index a962ccf..fae9f83 100644 --- a/scrapling/engines/static.py +++ b/scrapling/engines/static.py @@ -20,6 +20,7 @@ from scrapling.core._types import ( Optional, Awaitable, SUPPORTED_HTTP_METHODS, + FollowRedirects, ) from .toolbelt.custom import Response @@ -77,7 +78,7 @@ class _ConfigurationLogic(ABC): self._default_headers = kwargs.get("headers") or {} self._default_retries = kwargs.get("retries", 3) self._default_retry_delay = kwargs.get("retry_delay", 1) - self._default_follow_redirects = kwargs.get("follow_redirects", True) + self._default_follow_redirects = kwargs.get("follow_redirects", "safe") self._default_max_redirects = kwargs.get("max_redirects", 30) self._default_verify = kwargs.get("verify", True) self._default_cert = kwargs.get("cert") or None @@ -285,7 +286,7 @@ class _SyncSessionLogic(_ConfigurationLogic): - headers: Headers to include in the request. - cookies: Cookies to use in the request. - timeout: Number of seconds to wait before timing out. - - follow_redirects: Whether to follow redirects. Defaults to True. + - follow_redirects: Whether to follow redirects. Defaults to "safe" (rejects redirects to internal/private IPs). - max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited. - retries: Number of retry attempts. Defaults to 3. - retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second. @@ -317,7 +318,7 @@ class _SyncSessionLogic(_ConfigurationLogic): - headers: Headers to include in the request. - cookies: Cookies to use in the request. - timeout: Number of seconds to wait before timing out. - - follow_redirects: Whether to follow redirects. Defaults to True. + - follow_redirects: Whether to follow redirects. Defaults to "safe" (rejects redirects to internal/private IPs). - max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited. - retries: Number of retry attempts. Defaults to 3. - retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second. @@ -349,7 +350,7 @@ class _SyncSessionLogic(_ConfigurationLogic): - headers: Headers to include in the request. - cookies: Cookies to use in the request. - timeout: Number of seconds to wait before timing out. - - follow_redirects: Whether to follow redirects. Defaults to True. + - follow_redirects: Whether to follow redirects. Defaults to "safe" (rejects redirects to internal/private IPs). - max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited. - retries: Number of retry attempts. Defaults to 3. - retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second. @@ -381,7 +382,7 @@ class _SyncSessionLogic(_ConfigurationLogic): - headers: Headers to include in the request. - cookies: Cookies to use in the request. - timeout: Number of seconds to wait before timing out. - - follow_redirects: Whether to follow redirects. Defaults to True. + - follow_redirects: Whether to follow redirects. Defaults to "safe" (rejects redirects to internal/private IPs). - max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited. - retries: Number of retry attempts. Defaults to 3. - retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second. @@ -502,7 +503,7 @@ class _ASyncSessionLogic(_ConfigurationLogic): - headers: Headers to include in the request. - cookies: Cookies to use in the request. - timeout: Number of seconds to wait before timing out. - - follow_redirects: Whether to follow redirects. Defaults to True. + - follow_redirects: Whether to follow redirects. Defaults to "safe" (rejects redirects to internal/private IPs). - max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited. - retries: Number of retry attempts. Defaults to 3. - retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second. @@ -534,7 +535,7 @@ class _ASyncSessionLogic(_ConfigurationLogic): - headers: Headers to include in the request. - cookies: Cookies to use in the request. - timeout: Number of seconds to wait before timing out. - - follow_redirects: Whether to follow redirects. Defaults to True. + - follow_redirects: Whether to follow redirects. Defaults to "safe" (rejects redirects to internal/private IPs). - max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited. - retries: Number of retry attempts. Defaults to 3. - retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second. @@ -566,7 +567,7 @@ class _ASyncSessionLogic(_ConfigurationLogic): - headers: Headers to include in the request. - cookies: Cookies to use in the request. - timeout: Number of seconds to wait before timing out. - - follow_redirects: Whether to follow redirects. Defaults to True. + - follow_redirects: Whether to follow redirects. Defaults to "safe" (rejects redirects to internal/private IPs). - max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited. - retries: Number of retry attempts. Defaults to 3. - retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second. @@ -598,7 +599,7 @@ class _ASyncSessionLogic(_ConfigurationLogic): - headers: Headers to include in the request. - cookies: Cookies to use in the request. - timeout: Number of seconds to wait before timing out. - - follow_redirects: Whether to follow redirects. Defaults to True. + - follow_redirects: Whether to follow redirects. Defaults to "safe" (rejects redirects to internal/private IPs). - max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited. - retries: Number of retry attempts. Defaults to 3. - retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second. @@ -663,7 +664,7 @@ class FetcherSession: headers: Optional[Dict[str, str]] = None, retries: Optional[int] = 3, retry_delay: Optional[int] = 1, - follow_redirects: bool = True, + follow_redirects: FollowRedirects = "safe", max_redirects: int = 30, verify: bool = True, cert: Optional[str | Tuple[str, str]] = None, @@ -682,7 +683,7 @@ class FetcherSession: :param headers: Headers to include in the session with every request. :param retries: Number of retry attempts. Defaults to 3. :param retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second. - :param follow_redirects: Whether to follow redirects. Defaults to True. + :param follow_redirects: Whether to follow redirects. Defaults to "safe", which follows redirects but rejects those targeting internal/private IPs (SSRF protection). Pass True to follow all redirects without restriction. :param max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited. :param verify: Whether to verify HTTPS certificates. Defaults to True. :param cert: Tuple of (cert, key) filenames for the client certificate. diff --git a/scrapling/spiders/cache.py b/scrapling/spiders/cache.py new file mode 100644 index 0000000..40d39d3 --- /dev/null +++ b/scrapling/spiders/cache.py @@ -0,0 +1,79 @@ +from base64 import b64encode, b64decode +from pathlib import Path + +import orjson +import anyio +from anyio import Path as AsyncPath + +from scrapling.core.utils import log +from scrapling.core._types import Dict, Optional, Any +from scrapling.engines.toolbelt.custom import Response + + +class ResponseCacheManager: + """Caches HTTP responses to disk for replay during spider development.""" + + def __init__(self, cache_dir: str | Path): + self._cache_dir = AsyncPath(cache_dir) + + def _cache_path(self, fingerprint: bytes) -> AsyncPath: + return self._cache_dir / f"{fingerprint.hex()}.json" + + async def get(self, fingerprint: bytes) -> Optional[Response]: + path = self._cache_path(fingerprint) + if not await path.exists(): + return None + + try: + async with await anyio.open_file(path, "rb") as f: + data: Dict[str, Any] = orjson.loads(await f.read()) + + return Response( + url=data["url"], + content=b64decode(data["content"]), + status=data["status"], + reason=data["reason"], + encoding=data["encoding"], + cookies=data["cookies"], + headers=data["headers"], + request_headers=data["request_headers"], + method=data["method"], + ) + except Exception as e: + log.warning(f"Failed to read cached response for {fingerprint.hex()}: {e}") + return None + + async def put(self, fingerprint: bytes, response: Response, method: str = "GET") -> None: + await self._cache_dir.mkdir(parents=True, exist_ok=True) + temp_path = self._cache_path(fingerprint).with_suffix(".tmp") + + try: + serialized = orjson.dumps( + { + "url": response.url, + "content": b64encode(response.body).decode("ascii"), + "status": response.status, + "reason": response.reason, + "encoding": response.encoding, + "cookies": dict(response.cookies) if isinstance(response.cookies, dict) else {}, + "headers": dict(response.headers), + "request_headers": dict(response.request_headers), + "method": method, + } + ) + async with await anyio.open_file(temp_path, "wb") as f: + await f.write(serialized) + + await temp_path.rename(self._cache_path(fingerprint)) + except Exception as e: + if await temp_path.exists(): + await temp_path.unlink() + log.warning(f"Failed to cache response for {fingerprint.hex()}: {e}") + + async def clear(self) -> None: + if not await self._cache_dir.exists(): + return + async for entry in self._cache_dir.iterdir(): + if entry.suffix == ".json": + await entry.unlink() + log.info(f"Cleared response cache at {self._cache_dir}") diff --git a/scrapling/spiders/engine.py b/scrapling/spiders/engine.py index deac28f..6db6866 100644 --- a/scrapling/spiders/engine.py +++ b/scrapling/spiders/engine.py @@ -13,6 +13,7 @@ from scrapling.spiders.session import SessionManager from scrapling.spiders.request import Request, Response from scrapling.spiders.robotstxt import RobotsTxtManager from scrapling.spiders.result import CrawlStats, ItemList +from scrapling.spiders.cache import ResponseCacheManager from scrapling.spiders.checkpoint import CheckpointManager, CheckpointData from scrapling.core._types import Dict, Union, Optional, TYPE_CHECKING, Any, AsyncGenerator @@ -52,6 +53,13 @@ class CrawlerEngine: else: self._robots_manager = None + if self.spider.development_mode: + cache_dir = self.spider.development_cache_dir or f".scrapling_cache/{self.spider.name}" + self._cache_manager: Optional[ResponseCacheManager] = ResponseCacheManager(cache_dir) + log.warning("Development mode enabled -- responses will be cached to disk and replayed on subsequent runs") + else: + self._cache_manager = None + self._global_limiter = CapacityLimiter(spider.concurrent_requests) self._domain_limiters: dict[str, CapacityLimiter] = {} self._allowed_domains: set[str] = spider.allowed_domains or set() @@ -130,57 +138,8 @@ class CrawlerEngine: if not request.sid: request.sid = self.session_manager.default_session_id - async def _process_request(self, request: Request) -> None: - """Download and process a single request.""" - if self._robots_manager: - can_fetch = await self._robots_manager.can_fetch(request.url, request.sid) - if not can_fetch: - self.stats.robots_disallowed_count += 1 - log.info(f"Request disallowed by robots.txt: {request.url}") - return - delay = await self._get_domain_delay(request) - else: - delay = self.spider.download_delay - - async with self._rate_limiter(request.domain): - if delay: - await anyio.sleep(delay) - - if request._session_kwargs.get("proxy"): - self.stats.proxies.append(request._session_kwargs["proxy"]) - if request._session_kwargs.get("proxies"): - self.stats.proxies.append(dict(request._session_kwargs["proxies"])) - try: - response = await self.session_manager.fetch(request) - self.stats.increment_requests_count(request.sid or self.session_manager.default_session_id) - self.stats.increment_response_bytes(request.domain, len(response.body)) - self.stats.increment_status(response.status) - - except Exception as e: - self.stats.failed_requests_count += 1 - await self.spider.on_error(request, e) - return - - if await self.spider.is_blocked(response): - self.stats.blocked_requests_count += 1 - if request._retry_count < self.spider.max_blocked_retries: - retry_request = request.copy() - retry_request._retry_count += 1 - retry_request.priority -= 1 # Don't retry immediately - retry_request.dont_filter = True - retry_request._session_kwargs.pop("proxy", None) - retry_request._session_kwargs.pop("proxies", None) - - new_request = await self.spider.retry_blocked_request(retry_request, response) - self._normalize_request(new_request) - await self.scheduler.enqueue(new_request) - log.info( - f"Scheduled blocked request for retry ({retry_request._retry_count}/{self.spider.max_blocked_retries}): {request.url}" - ) - else: - log.warning(f"Max retries exceeded for blocked request: {request.url}") - return - + async def _run_callbacks(self, request: Request, response: Response) -> None: + """Dispatch response to the request's callback and process yielded items/requests.""" callback = request.callback if request.callback else self.spider.parse try: async for result in callback(response): @@ -210,6 +169,75 @@ class CrawlerEngine: log.error(msg, exc_info=e) await self.spider.on_error(request, e) + async def _process_request(self, request: Request) -> None: + """Download and process a single request.""" + if self._robots_manager: + can_fetch = await self._robots_manager.can_fetch(request.url, request.sid) + if not can_fetch: + self.stats.robots_disallowed_count += 1 + log.info(f"Request disallowed by robots.txt: {request.url}") + return + delay = await self._get_domain_delay(request) + else: + delay = self.spider.download_delay + + if self._cache_manager and request._fp is not None: + cached = await self._cache_manager.get(request._fp) + if cached is not None: + cached.request = request + self.stats.cache_hits += 1 + self.stats.increment_requests_count(request.sid or self.session_manager.default_session_id) + self.stats.increment_response_bytes(request.domain, len(cached.body)) + self.stats.increment_status(cached.status) + log.debug(f"Cache hit: {request.url}") + await self._run_callbacks(request, cached) + return + + async with self._rate_limiter(request.domain): + if delay: + await anyio.sleep(delay) + + if request._session_kwargs.get("proxy"): + self.stats.proxies.append(request._session_kwargs["proxy"]) + if request._session_kwargs.get("proxies"): + self.stats.proxies.append(dict(request._session_kwargs["proxies"])) + try: + response = await self.session_manager.fetch(request) + self.stats.increment_requests_count(request.sid or self.session_manager.default_session_id) + self.stats.increment_response_bytes(request.domain, len(response.body)) + self.stats.increment_status(response.status) + + except Exception as e: + self.stats.failed_requests_count += 1 + await self.spider.on_error(request, e) + return + + if self._cache_manager and request._fp is not None: + self.stats.cache_misses += 1 + await self._cache_manager.put(request._fp, response, request._session_kwargs.get("method", "GET")) + + if await self.spider.is_blocked(response): + self.stats.blocked_requests_count += 1 + if request._retry_count < self.spider.max_blocked_retries: + retry_request = request.copy() + retry_request._retry_count += 1 + retry_request.priority -= 1 # Don't retry immediately + retry_request.dont_filter = True + retry_request._session_kwargs.pop("proxy", None) + retry_request._session_kwargs.pop("proxies", None) + + new_request = await self.spider.retry_blocked_request(retry_request, response) + self._normalize_request(new_request) + await self.scheduler.enqueue(new_request) + log.info( + f"Scheduled blocked request for retry ({retry_request._retry_count}/{self.spider.max_blocked_retries}): {request.url}" + ) + else: + log.warning(f"Max retries exceeded for blocked request: {request.url}") + return + + await self._run_callbacks(request, response) + async def _task_wrapper(self, request: Request) -> None: """Wrapper to track active task count.""" try: @@ -330,11 +358,7 @@ class CrawlerEngine: while self._running: if self._pause_requested: if self._active_tasks == 0 or self._force_stop: - if self._force_stop: - log.warning(f"Force stopping with {self._active_tasks} active tasks") - tg.cancel_scope.cancel() - - # Only save checkpoint if checkpoint system is enabled + # Save checkpoint before canceling to avoid data loss if self._checkpoint_system_enabled: await self._save_checkpoint() self.paused = True @@ -342,6 +366,10 @@ class CrawlerEngine: else: log.info("Spider stopped gracefully") + if self._force_stop: + log.warning(f"Force stopping with {self._active_tasks} active tasks") + tg.cancel_scope.cancel() + self._running = False break diff --git a/scrapling/spiders/result.py b/scrapling/spiders/result.py index b374152..3710750 100644 --- a/scrapling/spiders/result.py +++ b/scrapling/spiders/result.py @@ -48,6 +48,8 @@ class CrawlStats: failed_requests_count: int = 0 offsite_requests_count: int = 0 robots_disallowed_count: int = 0 + cache_hits: int = 0 + cache_misses: int = 0 response_bytes: int = 0 items_scraped: int = 0 items_dropped: int = 0 @@ -97,6 +99,8 @@ class CrawlStats: "failed_requests_count": self.failed_requests_count, "offsite_requests_count": self.offsite_requests_count, "robots_disallowed_count": self.robots_disallowed_count, + "cache_hits": self.cache_hits, + "cache_misses": self.cache_misses, "blocked_requests_count": self.blocked_requests_count, "response_status_count": self.response_status_count, "response_bytes": self.response_bytes, diff --git a/scrapling/spiders/spider.py b/scrapling/spiders/spider.py index 6aaa24f..edd9d8b 100644 --- a/scrapling/spiders/spider.py +++ b/scrapling/spiders/spider.py @@ -75,6 +75,10 @@ class Spider(ABC): # Robots.txt compliance robots_txt_obey: bool = False + # Development mode + development_mode: bool = False + development_cache_dir: Optional[str] = None + # Concurrency settings concurrent_requests: int = 4 concurrent_requests_per_domain: int = 0 diff --git a/server.json b/server.json index 6ef6d9b..d1a95ad 100644 --- a/server.json +++ b/server.json @@ -14,12 +14,12 @@ "mimeType": "image/png" } ], - "version": "0.4.4", + "version": "0.4.5", "packages": [ { "registryType": "pypi", "identifier": "scrapling", - "version": "0.4.4", + "version": "0.4.5", "runtimeHint": "uvx", "packageArguments": [ { diff --git a/setup.cfg b/setup.cfg index 7dcdf68..48cddb3 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = scrapling -version = 0.4.4 +version = 0.4.5 author = Karim Shoair author_email = karim.shoair@pm.me description = Scrapling is an undetectable, powerful, flexible, high-performance Python library that makes Web Scraping easy and effortless as it should be! diff --git a/tests/spiders/test_cache.py b/tests/spiders/test_cache.py new file mode 100644 index 0000000..fc9bc59 --- /dev/null +++ b/tests/spiders/test_cache.py @@ -0,0 +1,228 @@ +"""Tests for the ResponseCacheManager and development_mode integration.""" + +import tempfile +from pathlib import Path + +import anyio +import pytest + +from scrapling.spiders.cache import ResponseCacheManager +from scrapling.spiders.engine import CrawlerEngine +from scrapling.spiders.request import Request +from scrapling.spiders.session import SessionManager +from scrapling.engines.toolbelt.custom import Response +from scrapling.core._types import Any, Dict, Set, AsyncGenerator + + +def _make_response(url: str = "https://example.com", body: bytes = b"hello", status: int = 200) -> Response: + return Response( + url=url, + content=body, + status=status, + reason="OK", + encoding="utf-8", + cookies={}, + headers={"content-type": "text/html"}, + request_headers={"user-agent": "test"}, + method="GET", + ) + + +class TestResponseCacheManager: + + @pytest.mark.anyio + async def test_put_get_roundtrip(self): + with tempfile.TemporaryDirectory() as tmpdir: + cache = ResponseCacheManager(tmpdir) + fp = b"\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10\x11\x12\x13\x14" + original = _make_response(body=b"test content") + + await cache.put(fp, original, "GET") + restored = await cache.get(fp) + + assert restored is not None + assert restored.url == original.url + assert restored.body == original.body + assert restored.status == original.status + assert restored.reason == original.reason + assert restored.encoding == original.encoding + assert dict(restored.headers) == dict(original.headers) + assert dict(restored.request_headers) == dict(original.request_headers) + + @pytest.mark.anyio + async def test_get_cache_miss(self): + with tempfile.TemporaryDirectory() as tmpdir: + cache = ResponseCacheManager(tmpdir) + result = await cache.get(b"\x00" * 20) + assert result is None + + @pytest.mark.anyio + async def test_get_corrupt_file(self): + with tempfile.TemporaryDirectory() as tmpdir: + cache = ResponseCacheManager(tmpdir) + fp = b"\xaa" * 20 + corrupt_path = Path(tmpdir) / f"{fp.hex()}.json" + corrupt_path.write_text("not valid json{{{") + + result = await cache.get(fp) + assert result is None + + @pytest.mark.anyio + async def test_clear(self): + with tempfile.TemporaryDirectory() as tmpdir: + cache = ResponseCacheManager(tmpdir) + fp1 = b"\x01" * 20 + fp2 = b"\x02" * 20 + + await cache.put(fp1, _make_response(url="https://a.com"), "GET") + await cache.put(fp2, _make_response(url="https://b.com"), "GET") + + assert await cache.get(fp1) is not None + assert await cache.get(fp2) is not None + + await cache.clear() + + assert await cache.get(fp1) is None + assert await cache.get(fp2) is None + + @pytest.mark.anyio + async def test_creates_cache_dir(self): + with tempfile.TemporaryDirectory() as tmpdir: + nested = Path(tmpdir) / "sub" / "dir" + cache = ResponseCacheManager(str(nested)) + await cache.put(b"\x03" * 20, _make_response(), "GET") + assert nested.exists() + + @pytest.mark.anyio + async def test_preserves_binary_body(self): + with tempfile.TemporaryDirectory() as tmpdir: + cache = ResponseCacheManager(tmpdir) + fp = b"\x04" * 20 + binary_body = bytes(range(256)) + await cache.put(fp, _make_response(body=binary_body), "GET") + restored = await cache.get(fp) + assert restored is not None + assert restored.body == binary_body + + +# --------------------------------------------------------------------------- +# Integration tests +# --------------------------------------------------------------------------- + + +class MockSession: + def __init__(self): + self._is_alive = False + self.fetch_count = 0 + + async def __aenter__(self): + self._is_alive = True + return self + + async def __aexit__(self, *args): + self._is_alive = False + + async def fetch(self, url: str, **kwargs): + self.fetch_count += 1 + return _make_response(url=url, body=b"fetched") + + +class _LogCounterStub: + def get_counts(self) -> Dict[str, int]: + return {"debug": 0, "info": 0, "warning": 0, "error": 0, "critical": 0} + + +class MockSpider: + def __init__(self, cache_dir: str): + self.concurrent_requests = 4 + self.concurrent_requests_per_domain = 0 + self.download_delay = 0.0 + self.max_blocked_retries = 3 + self.allowed_domains: Set[str] = set() + self.fp_include_kwargs = False + self.fp_include_headers = False + self.fp_keep_fragments = False + self.robots_txt_obey = False + self.development_mode = True + self.development_cache_dir = cache_dir + self.start_urls: list[str] = [] + self.name = "test_cache_spider" + self._log_counter = _LogCounterStub() + self.scraped_items: list[dict] = [] + + async def parse(self, response) -> AsyncGenerator[Dict[str, Any] | Request | None, None]: + yield {"url": str(response)} + + async def on_start(self, resuming: bool = False) -> None: + pass + + async def on_close(self) -> None: + pass + + async def on_error(self, request: Request, error: Exception) -> None: + pass + + async def on_scraped_item(self, item: Dict[str, Any]) -> Dict[str, Any] | None: + self.scraped_items.append(item) + return item + + async def is_blocked(self, response) -> bool: + return False + + async def retry_blocked_request(self, request: Request, response) -> Request: + return request + + async def start_requests(self) -> AsyncGenerator[Request, None]: + yield Request("https://example.com/page1", sid="default") + + +class TestDevelopmentModeIntegration: + + @pytest.mark.anyio + async def test_first_run_fetches_and_caches(self): + with tempfile.TemporaryDirectory() as tmpdir: + session = MockSession() + spider = MockSpider(cache_dir=tmpdir) + sm = SessionManager() + sm.add("default", session) + engine = CrawlerEngine(spider, sm) + + await engine.crawl() + + assert session.fetch_count == 1 + assert engine.stats.cache_misses == 1 + assert engine.stats.cache_hits == 0 + assert engine.stats.items_scraped == 1 + + @pytest.mark.anyio + async def test_second_run_uses_cache(self): + with tempfile.TemporaryDirectory() as tmpdir: + session = MockSession() + spider = MockSpider(cache_dir=tmpdir) + sm = SessionManager() + sm.add("default", session) + engine = CrawlerEngine(spider, sm) + + await engine.crawl() + assert session.fetch_count == 1 + + session2 = MockSession() + spider2 = MockSpider(cache_dir=tmpdir) + sm2 = SessionManager() + sm2.add("default", session2) + engine2 = CrawlerEngine(spider2, sm2) + + await engine2.crawl() + assert session2.fetch_count == 0 + assert engine2.stats.cache_hits == 1 + assert engine2.stats.cache_misses == 0 + assert engine2.stats.items_scraped == 1 + + @pytest.mark.anyio + async def test_disabled_by_default(self): + spider = MockSpider(cache_dir="unused") + spider.development_mode = False + sm = SessionManager() + sm.add("default", MockSession()) + engine = CrawlerEngine(spider, sm) + assert engine._cache_manager is None diff --git a/tests/spiders/test_engine.py b/tests/spiders/test_engine.py index 74cf875..10cbfdc 100644 --- a/tests/spiders/test_engine.py +++ b/tests/spiders/test_engine.py @@ -98,6 +98,8 @@ class MockSpider: self.fp_keep_fragments = fp_keep_fragments self.name = "test_spider" self.robots_txt_obey = robots_txt_obey + self.development_mode = False + self.development_cache_dir = None self.start_urls = start_urls or [] # Tracking lists diff --git a/tests/spiders/test_force_stop_checkpoint.py b/tests/spiders/test_force_stop_checkpoint.py new file mode 100644 index 0000000..a26e552 --- /dev/null +++ b/tests/spiders/test_force_stop_checkpoint.py @@ -0,0 +1,282 @@ +"""Tests for force-stop checkpoint preservation in CrawlerEngine. + +Regression tests for the bug where force-stop (second Ctrl+C) called +cancel_scope.cancel() BEFORE saving the checkpoint, causing: +1. _save_checkpoint() to be aborted by anyio's Cancelled exception +2. self.paused never set to True +3. The finally block to DELETE the previous checkpoint (cleanup runs on non-paused exit) + +Total progress loss: user's checkpoint from a long crawl is irrecoverably deleted. +""" + +import tempfile +from pathlib import Path + +import anyio +import pytest + +from scrapling.spiders.engine import CrawlerEngine +from scrapling.spiders.request import Request +from scrapling.spiders.session import SessionManager +from scrapling.spiders.checkpoint import CheckpointManager, CheckpointData +from scrapling.core._types import Any, Dict, Set, AsyncGenerator + + +# --------------------------------------------------------------------------- +# Mock helpers (minimal, matching test_engine.py conventions) +# --------------------------------------------------------------------------- + + +class MockResponse: + def __init__(self, status=200, body=b"ok", url="https://example.com"): + self.status = status + self.body = body + self.url = url + self.request: Any = None + self.meta: Dict[str, Any] = {} + + def __str__(self): + return self.url + + +class MockSession: + def __init__(self, delay: float = 0.0): + self._is_alive = False + self._delay = delay + + async def __aenter__(self): + self._is_alive = True + return self + + async def __aexit__(self, *args): + self._is_alive = False + + async def fetch(self, url: str, **kwargs): + if self._delay: + await anyio.sleep(self._delay) + resp = MockResponse(url=url) + return resp + + +class _LogCounterStub: + def get_counts(self): + return {"debug": 0, "info": 0, "warning": 0, "error": 0, "critical": 0} + + +class SlowSpider: + """Spider with slow-responding requests to simulate in-flight tasks during force-stop.""" + + def __init__(self, num_urls: int = 10): + self.concurrent_requests = 4 + self.concurrent_requests_per_domain = 0 + self.download_delay = 0.0 + self.max_blocked_retries = 3 + self.allowed_domains = set() + self.fp_include_kwargs = False + self.fp_include_headers = False + self.fp_keep_fragments = False + self.robots_txt_obey = False + self.development_mode = False + self.development_cache_dir = None + self.start_urls = [] + self.name = "slow_spider" + self._log_counter = _LogCounterStub() + self._num_urls = num_urls + self.on_start_calls = [] + self.on_close_calls = 0 + + async def parse(self, response) -> AsyncGenerator[Dict[str, Any] | Request | None, None]: + yield {"url": str(response)} + + async def on_start(self, resuming=False): + self.on_start_calls.append({"resuming": resuming}) + + async def on_close(self): + self.on_close_calls += 1 + + async def on_error(self, request, error): + pass + + async def on_scraped_item(self, item): + return item + + async def is_blocked(self, response): + return False + + async def retry_blocked_request(self, request, response): + return request + + async def start_requests(self) -> AsyncGenerator[Request, None]: + for i in range(self._num_urls): + yield Request(f"https://example.com/page/{i}", sid="default") + + +def _make_engine(spider=None, session=None, crawldir=None, interval=300.0): + spider = spider or SlowSpider() + sm = SessionManager() + sm.add("default", session or MockSession()) + return CrawlerEngine(spider, sm, crawldir=crawldir, interval=interval) + + +# --------------------------------------------------------------------------- +# Tests +# --------------------------------------------------------------------------- + + +class TestForceStopCheckpointPreservation: + """Verify checkpoint is saved BEFORE cancel_scope.cancel() on force-stop.""" + + @pytest.mark.anyio + async def test_force_stop_saves_checkpoint_before_cancel(self): + """Core regression test: force-stop must save checkpoint, not delete it.""" + with tempfile.TemporaryDirectory() as tmpdir: + spider = SlowSpider(num_urls=20) + # Use a slow session so tasks are in-flight when we force-stop + session = MockSession(delay=0.5) + engine = _make_engine(spider, session, crawldir=tmpdir, interval=0) + + checkpoint_path = Path(tmpdir) / "checkpoint.pkl" + + async def force_stop_after_delay(): + """Simulate two rapid Ctrl+C presses.""" + # Wait for some tasks to start + await anyio.sleep(0.1) + engine.request_pause() # First Ctrl+C + await anyio.sleep(0.05) + engine.request_pause() # Second Ctrl+C (force stop) + + async with anyio.create_task_group() as tg: + tg.start_soon(force_stop_after_delay) + await engine.crawl() + + # The checkpoint file MUST exist after force-stop + assert checkpoint_path.exists(), ( + "Checkpoint file was not saved (or was deleted) after force-stop. " + "This means the cancel_scope.cancel() ran before _save_checkpoint()." + ) + # Engine must report as paused + assert engine.paused is True + + @pytest.mark.anyio + async def test_graceful_pause_still_saves_checkpoint(self): + """Single Ctrl+C (graceful pause) should save checkpoint as before.""" + with tempfile.TemporaryDirectory() as tmpdir: + spider = SlowSpider(num_urls=5) + session = MockSession(delay=0.3) + engine = _make_engine(spider, session, crawldir=tmpdir, interval=0) + + checkpoint_path = Path(tmpdir) / "checkpoint.pkl" + + async def pause_after_delay(): + await anyio.sleep(0.1) + engine.request_pause() + + async with anyio.create_task_group() as tg: + tg.start_soon(pause_after_delay) + await engine.crawl() + + assert checkpoint_path.exists(), "Checkpoint not saved on graceful pause" + assert engine.paused is True + + @pytest.mark.anyio + async def test_force_stop_checkpoint_is_loadable(self): + """Checkpoint saved during force-stop must be valid and loadable.""" + with tempfile.TemporaryDirectory() as tmpdir: + spider = SlowSpider(num_urls=15) + session = MockSession(delay=0.4) + engine = _make_engine(spider, session, crawldir=tmpdir, interval=0) + + async def force_stop(): + await anyio.sleep(0.1) + engine.request_pause() + await anyio.sleep(0.05) + engine.request_pause() + + async with anyio.create_task_group() as tg: + tg.start_soon(force_stop) + await engine.crawl() + + # Load the checkpoint and verify it's valid + manager = CheckpointManager(tmpdir) + data = await manager.load() + assert data is not None, "Checkpoint data could not be loaded" + assert isinstance(data, CheckpointData) + # seen set should have some entries (requests were enqueued) + assert len(data.seen) > 0 + + @pytest.mark.anyio + async def test_normal_completion_cleans_up_checkpoint(self): + """Normal completion (no pause) should still clean up checkpoint files.""" + with tempfile.TemporaryDirectory() as tmpdir: + spider = SlowSpider(num_urls=2) + session = MockSession(delay=0.0) + engine = _make_engine(spider, session, crawldir=tmpdir, interval=0) + + await engine.crawl() + + checkpoint_path = Path(tmpdir) / "checkpoint.pkl" + # No pause → checkpoint should be cleaned up + assert not checkpoint_path.exists() + assert engine.paused is False + + @pytest.mark.anyio + async def test_force_stop_without_checkpoint_system(self): + """Force-stop without crawldir should not crash.""" + spider = SlowSpider(num_urls=10) + session = MockSession(delay=0.3) + engine = _make_engine(spider, session, crawldir=None) + + async def force_stop(): + await anyio.sleep(0.1) + engine.request_pause() + await anyio.sleep(0.05) + engine.request_pause() + + async with anyio.create_task_group() as tg: + tg.start_soon(force_stop) + await engine.crawl() + + # Should not crash and should not be marked as paused + # (no checkpoint system = no pause state) + assert engine.paused is False + + @pytest.mark.anyio + async def test_force_stop_preserves_existing_checkpoint(self): + """If a checkpoint already exists, force-stop must not delete it.""" + with tempfile.TemporaryDirectory() as tmpdir: + # First run: do a graceful pause to create a checkpoint + spider1 = SlowSpider(num_urls=10) + session1 = MockSession(delay=0.2) + engine1 = _make_engine(spider1, session1, crawldir=tmpdir, interval=0) + + async def pause1(): + await anyio.sleep(0.1) + engine1.request_pause() + + async with anyio.create_task_group() as tg: + tg.start_soon(pause1) + await engine1.crawl() + + checkpoint_path = Path(tmpdir) / "checkpoint.pkl" + assert checkpoint_path.exists(), "First run should create checkpoint" + first_checkpoint_size = checkpoint_path.stat().st_size + + # Second run: force-stop (the fix ensures checkpoint is updated, not deleted) + spider2 = SlowSpider(num_urls=10) + session2 = MockSession(delay=0.3) + engine2 = _make_engine(spider2, session2, crawldir=tmpdir, interval=0) + + async def force_stop2(): + await anyio.sleep(0.1) + engine2.request_pause() + await anyio.sleep(0.05) + engine2.request_pause() + + async with anyio.create_task_group() as tg: + tg.start_soon(force_stop2) + await engine2.crawl() + + # Checkpoint must still exist (updated, not deleted) + assert checkpoint_path.exists(), ( + "Force-stop deleted the checkpoint instead of preserving it" + )