Merge branch 'dev' into fix/fetcher-session-state-corruption
This commit is contained in:
Yuval Dinodia
@@ -173,6 +173,26 @@ MySpider().start()
|
||||
<a href="https://proxy-seller.com/?partner=CU9CAA5TBYFFT2" target="_blank">Proxy-Seller</a> provides reliable proxy infrastructure for web scraping, offering IPv4, IPv6, ISP, Residential, and Mobile proxies with stable performance, broad geo coverage, and flexible plans for business-scale data collection.
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td width="200">
|
||||
<a href="http://mangoproxy.com/?utm_source=D4Vinci&utm_medium=GitHub&utm_campaign=D4Vinci" target="_blank" title="Proxies You Can Rely On: Residential, Server, and Mobile">
|
||||
<img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/MangoProxy.png">
|
||||
</a>
|
||||
</td>
|
||||
<td>
|
||||
<a href="http://mangoproxy.com/?utm_source=D4Vinci&utm_medium=GitHub&utm_campaign=D4Vinci" target="_blank">Stable proxies</a> for scraping, automation, and multi-accounting. Clean IPs, fast response, and reliable performance under load. Built for scalable workflows.
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td width="200">
|
||||
<a href="https://www.swiftproxy.net/?ref=D4Vinci" target="_blank" title="Scalable Solutions for Web Data Access">
|
||||
<img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/SwiftProxy.png">
|
||||
</a>
|
||||
</td>
|
||||
<td>
|
||||
<a href="https://www.swiftproxy.net/?ref=D4Vinci" target="_blank">Swiftproxy</a> provides scalable residential proxies with 80M+ IPs across 195+ countries, delivering fast, reliable connections, automatic rotation, and strong anti-block performance. Free trial available.
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
<i><sub>Do you want to show your ad here? Click [here](https://github.com/sponsors/D4Vinci/sponsorships?tier_id=586646)</sub></i>
|
||||
@@ -186,7 +206,9 @@ MySpider().start()
|
||||
<a href="https://hasdata.com/?utm_source=github&utm_medium=banner&utm_campaign=D4Vinci" target="_blank" title="The web scraping service that actually beats anti-bot systems!"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/hasdata.png"></a>
|
||||
<a href="https://proxyempire.io/?ref=scrapling&utm_source=scrapling" target="_blank" title="Collect The Data Your Project Needs with the Best Residential Proxies"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/ProxyEmpire.png"></a>
|
||||
<a href="https://www.webshare.io/?referral_code=48r2m2cd5uz1" target="_blank" title="The Most Reliable Proxy with Unparalleled Performance"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/webshare.png"></a>
|
||||
<a href="https://browser.cash/?utm_source=D4Vinci&utm_medium=referral" target="_blank" title="Browser Automation & AI Browser Agent Platform"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/browserCash.png"></a>
|
||||
<a href="https://www.crawleo.dev/?utm_source=github&utm_medium=sponsor&utm_campaign=scrapling" target="_blank" title="Supercharge your AI with Real-Time Web Intelligence"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/crawleo.png"></a>
|
||||
<a href="https://www.rapidproxy.io/?ref=d4v" target="_blank" title="Affordable Access to the Proxy World – bypass CAPTCHAs blocks, and avoid additional costs."><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/rapidproxy.jpg"></a>
|
||||
|
||||
|
||||
<!-- /sponsors -->
|
||||
|
||||
@@ -203,6 +225,8 @@ MySpider().start()
|
||||
- 💾 **Pause & Resume**: Checkpoint-based crawl persistence. Press Ctrl+C for a graceful shutdown; restart to resume from where you left off.
|
||||
- 📡 **Streaming Mode**: Stream scraped items as they arrive via `async for item in spider.stream()` with real-time stats - ideal for UI, pipelines, and long-running crawls.
|
||||
- 🛡️ **Blocked Request Detection**: Automatic detection and retry of blocked requests with customizable logic.
|
||||
- 🤖 **Robots.txt Compliance**: Optional `robots_txt_obey` flag that respects `Disallow`, `Crawl-delay`, and `Request-rate` directives with per-domain caching.
|
||||
- 🧪 **Development Mode**: Cache responses to disk on the first run and replay them on subsequent runs - iterate on your `parse()` logic without re-hitting the target servers.
|
||||
- 📦 **Built-in Export**: Export results through hooks and your own pipeline or the built-in JSON/JSONL with `result.items.to_json()` / `result.items.to_jsonl()` respectively.
|
||||
|
||||
### Advanced Websites Fetching with Session Support
|
||||
@@ -211,7 +235,8 @@ MySpider().start()
|
||||
- **Anti-bot Bypass**: Advanced stealth capabilities with `StealthyFetcher` and fingerprint spoofing. Can easily bypass all types of Cloudflare's Turnstile/Interstitial with automation.
|
||||
- **Session Management**: Persistent session support with `FetcherSession`, `StealthySession`, and `DynamicSession` classes for cookie and state management across requests.
|
||||
- **Proxy Rotation**: Built-in `ProxyRotator` with cyclic or custom rotation strategies across all session types, plus per-request proxy overrides.
|
||||
- **Domain Blocking**: Block requests to specific domains (and their subdomains) in browser-based fetchers.
|
||||
- **Domain & Ad Blocking**: Block requests to specific domains (and their subdomains) or enable built-in ad blocking (~3,500 known ad/tracker domains) in browser-based fetchers.
|
||||
- **DNS Leak Prevention**: Optional DNS-over-HTTPS support to route DNS queries through Cloudflare's DoH, preventing DNS leaks when using proxies.
|
||||
- **Async Support**: Complete async support across all fetchers and dedicated async session classes.
|
||||
|
||||
### Adaptive Scraping & AI Integration
|
||||
|
||||
Binary file not shown.
@@ -1,7 +1,7 @@
|
||||
---
|
||||
name: scrapling-official
|
||||
description: Scrape web pages using Scrapling with anti-bot bypass (like Cloudflare Turnstile), stealth headless browsing, spiders framework, adaptive scraping, and JavaScript rendering. Use when asked to scrape, crawl, or extract data from websites; web_fetch fails; the site has anti-bot protections; write Python code to scrape/crawl; or write spiders.
|
||||
version: "0.4.4"
|
||||
version: "0.4.7"
|
||||
license: Complete terms in LICENSE.txt
|
||||
metadata:
|
||||
homepage: "https://scrapling.readthedocs.io/en/latest/index.html"
|
||||
@@ -34,13 +34,13 @@ Blazing fast crawls with real-time stats and streaming. Built by Web Scrapers fo
|
||||
> 2. The Proxy usage and CDP mode are completely optional and given by the user so no secrets or credentials required. Depending on the user usage.
|
||||
> 3. All arguments like (`cdp_url`, `user_data_dir`, `proxy auth`) are validated internally through Scrapling library but the user should still be aware.
|
||||
|
||||
**IMPORTANT**: While using the commandline scraping commands, you MUST use the commandline argument `--ai-targeted` to protect from Prompt Injection!
|
||||
**IMPORTANT**: While using the commandline scraping commands, you MUST use the commandline argument `--ai-targeted` to protect from Prompt Injection! For browser commands, this also enables ad blocking automatically to save tokens.
|
||||
|
||||
## Setup (once)
|
||||
|
||||
Create a virtual Python environment through any way available, like `venv`, then inside the environment do:
|
||||
|
||||
`pip install "scrapling[all]>=0.4.4"`
|
||||
`pip install "scrapling[all]>=0.4.7"`
|
||||
|
||||
Then do this to download all the browsers' dependencies:
|
||||
|
||||
@@ -104,7 +104,7 @@ Those options are shared between the 4 HTTP request commands:
|
||||
| --proxy | TEXT | Proxy URL in format "http://username:password@host:port" |
|
||||
| -s, --css-selector | TEXT | CSS selector to extract specific content from the page. It returns all matches. |
|
||||
| -p, --params | TEXT | Query parameters in format "key=value" (can be used multiple times) |
|
||||
| --follow-redirects / --no-follow-redirects | None | Whether to follow redirects (default: True) |
|
||||
| --follow-redirects / --no-follow-redirects | None | Whether to follow redirects (default: "safe", rejects redirects to internal/private IPs) |
|
||||
| --verify / --no-verify | None | Whether to verify SSL certificates (default: True) |
|
||||
| --impersonate | TEXT | Browser to impersonate. Can be a single browser (e.g., Chrome) or a comma-separated list for random selection (e.g., Chrome, Firefox, Safari). |
|
||||
| --stealthy-headers / --no-stealthy-headers | None | Use stealthy browser headers (default: True) |
|
||||
@@ -156,7 +156,9 @@ Both (`fetch` / `stealthy-fetch`) share options:
|
||||
| --wait-selector | TEXT | CSS selector to wait for before proceeding |
|
||||
| --proxy | TEXT | Proxy URL in format "http://username:password@host:port" |
|
||||
| -H, --extra-headers | TEXT | Extra headers in format "Key: Value" (can be used multiple times) |
|
||||
| --ai-targeted | None | Extract only main content and sanitize hidden elements for AI consumption (default: False) |
|
||||
| --dns-over-https / --no-dns-over-https | None | Route DNS through Cloudflare's DoH to prevent DNS leaks when using proxies (default: False) |
|
||||
| --block-ads / --no-block-ads | None | Block requests to ~3,500 known ad and tracker domains (default: False) |
|
||||
| --ai-targeted | None | Extract only main content and sanitize hidden elements for AI consumption (default: False). Also enables ad blocking automatically. |
|
||||
|
||||
This option is specific to `fetch` only:
|
||||
|
||||
@@ -258,6 +260,7 @@ class QuotesSpider(Spider):
|
||||
name = "quotes"
|
||||
start_urls = ["https://quotes.toscrape.com/"]
|
||||
concurrent_requests = 10
|
||||
robots_txt_obey = True # Respect robots.txt rules
|
||||
|
||||
async def parse(self, response: Response):
|
||||
for quote in response.css('.quote'):
|
||||
@@ -301,6 +304,8 @@ QuotesSpider(crawldir="./crawl_data").start()
|
||||
```
|
||||
Press Ctrl+C to pause gracefully - progress is saved automatically. Later, when you start the spider again, pass the same `crawldir`, and it will resume from where it stopped.
|
||||
|
||||
While iterating on a spider's `parse()` logic, set `development_mode = True` on the spider class to cache responses to disk on the first run and replay them on subsequent runs - so you can re-run the spider as many times as you want without re-hitting the target servers. The cache lives in `.scrapling_cache/{spider.name}/` by default and can be overridden with `development_cache_dir`. Don't ship a spider with this enabled.
|
||||
|
||||
### Advanced Parsing & Navigation
|
||||
```python
|
||||
from scrapling.fetchers import Fetcher
|
||||
@@ -379,7 +384,7 @@ This skill encapsulates almost all the published documentation in Markdown, so d
|
||||
|
||||
## Guardrails (Always)
|
||||
- Only scrape content you're authorized to access.
|
||||
- Respect robots.txt and ToS.
|
||||
- Add delays (download_delay) for large crawls.
|
||||
- Respect robots.txt and ToS. Use `robots_txt_obey = True` on spiders to enforce this automatically.
|
||||
- Add delays (`download_delay`) for large crawls.
|
||||
- Don't bypass paywalls or authentication without permission.
|
||||
- Never scrape personal/sensitive data.
|
||||
@@ -9,7 +9,7 @@ All examples collect **all 100 quotes across 10 pages**.
|
||||
Make sure Scrapling is installed:
|
||||
|
||||
```bash
|
||||
pip install "scrapling[all]>=0.4.4"
|
||||
pip install "scrapling[all]>=0.4.7"
|
||||
scrapling install --force
|
||||
```
|
||||
|
||||
|
||||
@@ -60,7 +60,8 @@ All arguments for `DynamicFetcher` and its session classes:
|
||||
| load_dom | Enabled by default, wait for all JavaScript on page(s) to fully load and execute (wait for the `domcontentloaded` state). | ✔️ |
|
||||
| timeout | The timeout (milliseconds) used in all operations and waits through the page. The default is 30,000 ms (30 seconds). | ✔️ |
|
||||
| wait | The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the `Response` object. | ✔️ |
|
||||
| page_action | Added for automation. Pass a function that takes the `page` object and does the necessary automation. | ✔️ |
|
||||
| page_action | Added for automation. Pass a function that takes the `page` object, runs after navigation, and does the necessary automation. | ✔️ |
|
||||
| page_setup | A function that takes the `page` object, runs before navigation. Use it to register event listeners or routes that must be set up before the page loads. | ✔️ |
|
||||
| wait_selector | Wait for a specific css selector to be in a specific state. | ✔️ |
|
||||
| init_script | An absolute path to a JavaScript file to be executed on page creation for all pages in this session. | ✔️ |
|
||||
| wait_selector_state | Scrapling will wait for the given state to be fulfilled for the selector given with `wait_selector`. _Default state is `attached`._ | ✔️ |
|
||||
@@ -76,13 +77,15 @@ All arguments for `DynamicFetcher` and its session classes:
|
||||
| additional_args | Additional arguments to be passed to Playwright's context as additional settings, and they take higher priority than Scrapling's settings. | ✔️ |
|
||||
| selector_config | A dictionary of custom parsing arguments to be used when creating the final `Selector`/`Response` class. | ✔️ |
|
||||
| blocked_domains | A set of domain names to block requests to. Subdomains are also matched (e.g., `"example.com"` blocks `"sub.example.com"` too). | ✔️ |
|
||||
| block_ads | Block requests to ~3,500 known ad/tracking domains. Can be combined with `blocked_domains`. | ✔️ |
|
||||
| dns_over_https | Route DNS queries through Cloudflare's DNS-over-HTTPS to prevent DNS leaks when using proxies. | ✔️ |
|
||||
| proxy_rotator | A `ProxyRotator` instance for automatic proxy rotation. Cannot be combined with `proxy`. | ✔️ |
|
||||
| retries | Number of retry attempts for failed requests. Defaults to 3. | ✔️ |
|
||||
| retry_delay | Seconds to wait between retry attempts. Defaults to 1. | ✔️ |
|
||||
| capture_xhr | Pass a regex URL pattern string to capture XHR/fetch requests matching it during page load. Captured responses are available via `response.captured_xhr`. Defaults to `None` (disabled). | ✔️ |
|
||||
| executable_path | Absolute path to a custom browser executable to use instead of the bundled Chromium. Useful for non-standard installations or custom browser builds. | ✔️ |
|
||||
|
||||
In session classes, all these arguments can be set globally for the session. Still, you can configure each request individually by passing some of the arguments here that can be configured on the browser tab level like: `google_search`, `timeout`, `wait`, `page_action`, `extra_headers`, `disable_resources`, `wait_selector`, `wait_selector_state`, `network_idle`, `load_dom`, `blocked_domains`, `proxy`, and `selector_config`.
|
||||
In session classes, all these arguments can be set globally for the session. Still, you can configure each request individually by passing some of the arguments here that can be configured on the browser tab level like: `google_search`, `timeout`, `wait`, `page_action`, `page_setup`, `extra_headers`, `disable_resources`, `wait_selector`, `wait_selector_state`, `network_idle`, `load_dom`, `blocked_domains`, `proxy`, and `selector_config`.
|
||||
|
||||
**Notes:**
|
||||
1. The `disable_resources` option made requests ~25% faster in tests for some websites and can help save proxy usage, but be careful with it, as it can cause some websites to never finish loading.
|
||||
@@ -154,6 +157,29 @@ with open(file='main_cover.png', mode='wb') as f:
|
||||
|
||||
The `body` attribute of the `Response` object always returns `bytes`.
|
||||
|
||||
### Pre-Navigation Setup
|
||||
If you need to set up event listeners, routes, or scripts that must be registered before the page navigates, use `page_setup`. This function receives the `page` object and runs before `page.goto()` is called.
|
||||
|
||||
```python
|
||||
from playwright.sync_api import Page
|
||||
|
||||
def capture_websockets(page: Page):
|
||||
page.on("websocket", lambda ws: print(f"WebSocket opened: {ws.url}"))
|
||||
|
||||
page = DynamicFetcher.fetch('https://example.com', page_setup=capture_websockets)
|
||||
```
|
||||
Async version:
|
||||
```python
|
||||
from playwright.async_api import Page
|
||||
|
||||
async def capture_websockets(page: Page):
|
||||
page.on("websocket", lambda ws: print(f"WebSocket opened: {ws.url}"))
|
||||
|
||||
page = await DynamicFetcher.async_fetch('https://example.com', page_setup=capture_websockets)
|
||||
```
|
||||
|
||||
You can combine it with `page_action` -- `page_setup` runs before navigation, `page_action` runs after.
|
||||
|
||||
### Browser Automation
|
||||
This is where your knowledge about [Playwright's Page API](https://playwright.dev/python/docs/api/class-page) comes into play. The function you pass here takes the page object from Playwright's API, performs the desired action, and then the fetcher continues.
|
||||
|
||||
|
||||
@@ -15,7 +15,7 @@ All methods for making requests here share some arguments, so let's discuss them
|
||||
|
||||
- **url**: The targeted URL
|
||||
- **stealthy_headers**: If enabled (default), it creates and adds real browser headers. It also sets a Google referer header.
|
||||
- **follow_redirects**: As the name implies, tell the fetcher to follow redirections. **Enabled by default**
|
||||
- **follow_redirects**: Controls redirect behavior. **Defaults to `"safe"`**, which follows redirects but rejects those targeting internal/private IPs (SSRF protection). Pass `True` to follow all redirects without restriction, or `False` to disable redirects entirely.
|
||||
- **timeout**: The number of seconds to wait for each request to be finished. **Defaults to 30 seconds**.
|
||||
- **retries**: The number of retries that the fetcher will do for failed requests. **Defaults to three retries**.
|
||||
- **retry_delay**: Number of seconds to wait between retry attempts. **Defaults to 1 second**.
|
||||
@@ -50,7 +50,7 @@ Examples are the best way to explain this:
|
||||
>>> from scrapling.fetchers import Fetcher
|
||||
>>> # Basic GET
|
||||
>>> page = Fetcher.get('https://example.com')
|
||||
>>> page = Fetcher.get('https://scrapling.requestcatcher.com/get', stealthy_headers=True, follow_redirects=True)
|
||||
>>> page = Fetcher.get('https://scrapling.requestcatcher.com/get', stealthy_headers=True)
|
||||
>>> page = Fetcher.get('https://scrapling.requestcatcher.com/get', proxy='http://username:password@localhost:8030')
|
||||
>>> # With parameters
|
||||
>>> page = Fetcher.get('https://example.com/search', params={'q': 'query'})
|
||||
@@ -69,7 +69,7 @@ And for asynchronous requests, it's a small adjustment
|
||||
>>> from scrapling.fetchers import AsyncFetcher
|
||||
>>> # Basic GET
|
||||
>>> page = await AsyncFetcher.get('https://example.com')
|
||||
>>> page = await AsyncFetcher.get('https://scrapling.requestcatcher.com/get', stealthy_headers=True, follow_redirects=True)
|
||||
>>> page = await AsyncFetcher.get('https://scrapling.requestcatcher.com/get', stealthy_headers=True)
|
||||
>>> page = await AsyncFetcher.get('https://scrapling.requestcatcher.com/get', proxy='http://username:password@localhost:8030')
|
||||
>>> # With parameters
|
||||
>>> page = await AsyncFetcher.get('https://example.com/search', params={'q': 'query'})
|
||||
@@ -105,7 +105,7 @@ The `page` object in all cases is a [Response](choosing.md#response-object) obje
|
||||
>>> from scrapling.fetchers import Fetcher
|
||||
>>> # Basic POST
|
||||
>>> page = Fetcher.post('https://scrapling.requestcatcher.com/post', data={'key': 'value'}, params={'q': 'query'})
|
||||
>>> page = Fetcher.post('https://scrapling.requestcatcher.com/post', data={'key': 'value'}, stealthy_headers=True, follow_redirects=True)
|
||||
>>> page = Fetcher.post('https://scrapling.requestcatcher.com/post', data={'key': 'value'}, stealthy_headers=True)
|
||||
>>> page = Fetcher.post('https://scrapling.requestcatcher.com/post', data={'key': 'value'}, proxy='http://username:password@localhost:8030', impersonate="chrome")
|
||||
>>> # Another example of form-encoded data
|
||||
>>> page = Fetcher.post('https://example.com/submit', data={'username': 'user', 'password': 'pass'}, http3=True)
|
||||
@@ -117,7 +117,7 @@ And for asynchronous requests, it's a small adjustment
|
||||
>>> from scrapling.fetchers import AsyncFetcher
|
||||
>>> # Basic POST
|
||||
>>> page = await AsyncFetcher.post('https://scrapling.requestcatcher.com/post', data={'key': 'value'})
|
||||
>>> page = await AsyncFetcher.post('https://scrapling.requestcatcher.com/post', data={'key': 'value'}, stealthy_headers=True, follow_redirects=True)
|
||||
>>> page = await AsyncFetcher.post('https://scrapling.requestcatcher.com/post', data={'key': 'value'}, stealthy_headers=True)
|
||||
>>> page = await AsyncFetcher.post('https://scrapling.requestcatcher.com/post', data={'key': 'value'}, proxy='http://username:password@localhost:8030', impersonate="chrome")
|
||||
>>> # Another example of form-encoded data
|
||||
>>> page = await AsyncFetcher.post('https://example.com/submit', data={'username': 'user', 'password': 'pass'}, http3=True)
|
||||
@@ -129,7 +129,7 @@ And for asynchronous requests, it's a small adjustment
|
||||
>>> from scrapling.fetchers import Fetcher
|
||||
>>> # Basic PUT
|
||||
>>> page = Fetcher.put('https://example.com/update', data={'status': 'updated'})
|
||||
>>> page = Fetcher.put('https://example.com/update', data={'status': 'updated'}, stealthy_headers=True, follow_redirects=True, impersonate="chrome")
|
||||
>>> page = Fetcher.put('https://example.com/update', data={'status': 'updated'}, stealthy_headers=True, impersonate="chrome")
|
||||
>>> page = Fetcher.put('https://example.com/update', data={'status': 'updated'}, proxy='http://username:password@localhost:8030')
|
||||
>>> # Another example of form-encoded data
|
||||
>>> page = Fetcher.put("https://scrapling.requestcatcher.com/put", data={'key': ['value1', 'value2']})
|
||||
@@ -139,7 +139,7 @@ And for asynchronous requests, it's a small adjustment
|
||||
>>> from scrapling.fetchers import AsyncFetcher
|
||||
>>> # Basic PUT
|
||||
>>> page = await AsyncFetcher.put('https://example.com/update', data={'status': 'updated'})
|
||||
>>> page = await AsyncFetcher.put('https://example.com/update', data={'status': 'updated'}, stealthy_headers=True, follow_redirects=True, impersonate="chrome")
|
||||
>>> page = await AsyncFetcher.put('https://example.com/update', data={'status': 'updated'}, stealthy_headers=True, impersonate="chrome")
|
||||
>>> page = await AsyncFetcher.put('https://example.com/update', data={'status': 'updated'}, proxy='http://username:password@localhost:8030')
|
||||
>>> # Another example of form-encoded data
|
||||
>>> page = await AsyncFetcher.put("https://scrapling.requestcatcher.com/put", data={'key': ['value1', 'value2']})
|
||||
@@ -149,14 +149,14 @@ And for asynchronous requests, it's a small adjustment
|
||||
```python
|
||||
>>> from scrapling.fetchers import Fetcher
|
||||
>>> page = Fetcher.delete('https://example.com/resource/123')
|
||||
>>> page = Fetcher.delete('https://example.com/resource/123', stealthy_headers=True, follow_redirects=True, impersonate="chrome")
|
||||
>>> page = Fetcher.delete('https://example.com/resource/123', stealthy_headers=True, impersonate="chrome")
|
||||
>>> page = Fetcher.delete('https://example.com/resource/123', proxy='http://username:password@localhost:8030')
|
||||
```
|
||||
And for asynchronous requests, it's a small adjustment
|
||||
```python
|
||||
>>> from scrapling.fetchers import AsyncFetcher
|
||||
>>> page = await AsyncFetcher.delete('https://example.com/resource/123')
|
||||
>>> page = await AsyncFetcher.delete('https://example.com/resource/123', stealthy_headers=True, follow_redirects=True, impersonate="chrome")
|
||||
>>> page = await AsyncFetcher.delete('https://example.com/resource/123', stealthy_headers=True, impersonate="chrome")
|
||||
>>> page = await AsyncFetcher.delete('https://example.com/resource/123', proxy='http://username:password@localhost:8030')
|
||||
```
|
||||
|
||||
|
||||
@@ -38,7 +38,8 @@ Scrapling provides many options with this fetcher and its session classes. Befor
|
||||
| load_dom | Enabled by default, wait for all JavaScript on page(s) to fully load and execute (wait for the `domcontentloaded` state). | ✔️ |
|
||||
| timeout | The timeout (milliseconds) used in all operations and waits through the page. The default is 30,000 ms (30 seconds). | ✔️ |
|
||||
| wait | The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the `Response` object. | ✔️ |
|
||||
| page_action | Added for automation. Pass a function that takes the `page` object and does the necessary automation. | ✔️ |
|
||||
| page_action | Added for automation. Pass a function that takes the `page` object, runs after navigation, and does the necessary automation. | ✔️ |
|
||||
| page_setup | A function that takes the `page` object, runs before navigation. Use it to register event listeners or routes that must be set up before the page loads. | ✔️ |
|
||||
| wait_selector | Wait for a specific css selector to be in a specific state. | ✔️ |
|
||||
| init_script | An absolute path to a JavaScript file to be executed on page creation for all pages in this session. | ✔️ |
|
||||
| wait_selector_state | Scrapling will wait for the given state to be fulfilled for the selector given with `wait_selector`. _Default state is `attached`._ | ✔️ |
|
||||
@@ -58,13 +59,15 @@ Scrapling provides many options with this fetcher and its session classes. Befor
|
||||
| additional_args | Additional arguments to be passed to Playwright's context as additional settings, and they take higher priority than Scrapling's settings. | ✔️ |
|
||||
| selector_config | A dictionary of custom parsing arguments to be used when creating the final `Selector`/`Response` class. | ✔️ |
|
||||
| blocked_domains | A set of domain names to block requests to. Subdomains are also matched (e.g., `"example.com"` blocks `"sub.example.com"` too). | ✔️ |
|
||||
| block_ads | Block requests to ~3,500 known ad/tracking domains. Can be combined with `blocked_domains`. | ✔️ |
|
||||
| dns_over_https | Route DNS queries through Cloudflare's DNS-over-HTTPS to prevent DNS leaks when using proxies. | ✔️ |
|
||||
| proxy_rotator | A `ProxyRotator` instance for automatic proxy rotation. Cannot be combined with `proxy`. | ✔️ |
|
||||
| retries | Number of retry attempts for failed requests. Defaults to 3. | ✔️ |
|
||||
| retry_delay | Seconds to wait between retry attempts. Defaults to 1. | ✔️ |
|
||||
| capture_xhr | Pass a regex URL pattern string to capture XHR/fetch requests matching it during page load. Captured responses are available via `response.captured_xhr`. Defaults to `None` (disabled). | ✔️ |
|
||||
| executable_path | Absolute path to a custom browser executable to use instead of the bundled Chromium. Useful for non-standard installations or custom browser builds. | ✔️ |
|
||||
|
||||
In session classes, all these arguments can be set globally for the session. Still, you can configure each request individually by passing some of the arguments here that can be configured on the browser tab level like: `google_search`, `timeout`, `wait`, `page_action`, `extra_headers`, `disable_resources`, `wait_selector`, `wait_selector_state`, `network_idle`, `load_dom`, `solve_cloudflare`, `blocked_domains`, `proxy`, and `selector_config`.
|
||||
In session classes, all these arguments can be set globally for the session. Still, you can configure each request individually by passing some of the arguments here that can be configured on the browser tab level like: `google_search`, `timeout`, `wait`, `page_action`, `page_setup`, `extra_headers`, `disable_resources`, `wait_selector`, `wait_selector_state`, `network_idle`, `load_dom`, `solve_cloudflare`, `blocked_domains`, `proxy`, and `selector_config`.
|
||||
|
||||
**Notes:**
|
||||
|
||||
|
||||
@@ -27,7 +27,7 @@ Fast HTTP GET with browser fingerprint impersonation (TLS, headers). Suitable fo
|
||||
| `retry_delay` | int | 1 | Seconds between retries |
|
||||
| `stealthy_headers` | bool | true | Generate realistic browser headers and Google referer |
|
||||
| `http3` | bool | false | Use HTTP/3 (may conflict with `impersonate`) |
|
||||
| `follow_redirects` | bool | true | Follow HTTP redirects |
|
||||
| `follow_redirects` | bool or "safe" | "safe" | Follow redirects. "safe" rejects redirects to internal/private IPs |
|
||||
| `max_redirects` | int | 30 | Max redirects (-1 for unlimited) |
|
||||
| `headers` | dict or null | null | Custom request headers |
|
||||
| `cookies` | dict or null | null | Request cookies |
|
||||
@@ -164,6 +164,10 @@ When `main_content_only=true` (the default), the server automatically sanitizes
|
||||
|
||||
Keep `main_content_only=true` for maximum protection.
|
||||
|
||||
## Ad blocking
|
||||
|
||||
All browser-based tools (`fetch`, `bulk_fetch`, `stealthy_fetch`, `bulk_stealthy_fetch`) and persistent sessions (`open_session`) automatically block requests to ~3,500 known ad and tracker domains. This is always enabled in the MCP server to save tokens and speed up page loads. No configuration needed.
|
||||
|
||||
## Setup
|
||||
|
||||
Start the server (stdio transport, used by most MCP clients):
|
||||
|
||||
@@ -9,6 +9,7 @@ The spider system uses three class attributes to control how aggressively it cra
|
||||
| `concurrent_requests` | `4` | Maximum number of requests being processed at the same time |
|
||||
| `concurrent_requests_per_domain` | `0` | Maximum concurrent requests per domain (0 = no per-domain limit) |
|
||||
| `download_delay` | `0.0` | Seconds to wait before each request |
|
||||
| `robots_txt_obey` | `False` | Respect robots.txt rules (Disallow, Crawl-delay, Request-rate) |
|
||||
|
||||
```python
|
||||
class PoliteSpider(Spider):
|
||||
@@ -84,6 +85,49 @@ async def on_start(self, resuming: bool = False):
|
||||
self.logger.info("Starting fresh crawl")
|
||||
```
|
||||
|
||||
## Development Mode
|
||||
|
||||
When you're iterating on a spider's `parse()` logic, re-hitting the target servers on every run is slow and noisy. Development mode caches every response to disk on the first run and replays them from disk on subsequent runs, so you can tweak your selectors and re-run the spider as many times as you want without making a single network request.
|
||||
|
||||
Enable it by setting `development_mode = True` on your spider:
|
||||
|
||||
```python
|
||||
class MySpider(Spider):
|
||||
name = "my_spider"
|
||||
start_urls = ["https://example.com"]
|
||||
development_mode = True
|
||||
|
||||
async def parse(self, response: Response):
|
||||
yield {"title": response.css("title::text").get("")}
|
||||
```
|
||||
|
||||
The first run fetches normally and stores each response on disk. Every subsequent run serves the same requests from the cache, skipping the network entirely.
|
||||
|
||||
### Cache Location
|
||||
|
||||
By default, responses are cached in `.scrapling_cache/{spider.name}/` relative to the current working directory (where you ran the spider from, **not** where the spider script lives). You can override the location with `development_cache_dir`:
|
||||
|
||||
```python
|
||||
class MySpider(Spider):
|
||||
name = "my_spider"
|
||||
start_urls = ["https://example.com"]
|
||||
development_mode = True
|
||||
development_cache_dir = "/tmp/my_spider_cache"
|
||||
```
|
||||
|
||||
### How It Works
|
||||
|
||||
1. **Cache key**: Each response is keyed by the request's fingerprint, so any change to fingerprint-affecting attributes (`fp_include_kwargs`, `fp_include_headers`, `fp_keep_fragments`) will produce a fresh fetch.
|
||||
2. **Storage format**: One JSON file per response, named `{fingerprint_hex}.json`. The body is base64-encoded so binary content is preserved exactly. Writes are atomic (temp file + rename).
|
||||
3. **Replay**: On a cache hit, the engine skips the network entirely, including `download_delay`, rate limiting, and the `is_blocked()` retry path. The cached response goes straight to your callback.
|
||||
4. **Stats**: Cached requests still count toward `requests_count`, `response_bytes`, and the per-status counters, so your stat output looks the same as a normal crawl. Two extra counters, `cache_hits` and `cache_misses`, let you see how the cache performed.
|
||||
|
||||
### Clearing the Cache
|
||||
|
||||
There's no automatic expiration. To force a fresh crawl, delete the cache directory or call the manager's `clear()` method directly.
|
||||
|
||||
**Warning:** Development mode is meant for development, not production. Cached responses never expire, and replay bypasses rate limiting and blocked-request retries. Don't ship a spider with `development_mode = True`.
|
||||
|
||||
## Streaming
|
||||
|
||||
For long-running spiders or applications that need real-time access to scraped items, use the `stream()` method instead of `start()`:
|
||||
@@ -218,6 +262,9 @@ print(f"Requests: {stats.requests_count}")
|
||||
print(f"Failed: {stats.failed_requests_count}")
|
||||
print(f"Blocked: {stats.blocked_requests_count}")
|
||||
print(f"Offsite filtered: {stats.offsite_requests_count}")
|
||||
print(f"Robots.txt disallowed: {stats.robots_disallowed_count}")
|
||||
print(f"Cache hits: {stats.cache_hits}")
|
||||
print(f"Cache misses: {stats.cache_misses}")
|
||||
print(f"Items scraped: {stats.items_scraped}")
|
||||
print(f"Items dropped: {stats.items_dropped}")
|
||||
print(f"Response bytes: {stats.response_bytes}")
|
||||
|
||||
@@ -10,7 +10,7 @@ Here's what happens step by step when you run a spider:
|
||||
|
||||
1. The **Spider** produces the first batch of `Request` objects. By default, it creates one request for each URL in `start_urls`, but you can override `start_requests()` for custom logic.
|
||||
2. The **Scheduler** receives requests and places them in a priority queue, and creates fingerprints for them. Higher-priority requests are dequeued first.
|
||||
3. The **Crawler Engine** asks the **Scheduler** to dequeue the next request, respecting concurrency limits (global and per-domain) and download delays. Once the **Crawler Engine** receives the request, it passes it to the **Session Manager**, which routes it to the correct session based on the request's `sid` (session ID).
|
||||
3. The **Crawler Engine** asks the **Scheduler** to dequeue the next request, respecting concurrency limits (global and per-domain) and download delays. If `robots_txt_obey` is enabled, the engine checks the domain's robots.txt rules before proceeding -- disallowed requests are dropped silently. Once the **Crawler Engine** receives the request, it passes it to the **Session Manager**, which routes it to the correct session based on the request's `sid` (session ID).
|
||||
4. The **session** fetches the page and returns a [Response](../fetching/choosing.md#response-object) object to the **Crawler Engine**. The engine records statistics and checks for blocked responses. If the response is blocked, the engine retries the request up to `max_blocked_retries` times. Of course, the blocking detection and the retry logic for blocked requests can be customized.
|
||||
5. The **Crawler Engine** passes the [Response](../fetching/choosing.md#response-object) to the request's callback. The callback either yields a dictionary, which gets treated as a scraped item, or a follow-up request, which gets sent to the scheduler for queuing.
|
||||
6. The cycle repeats from step 2 until the scheduler is empty and no tasks are active, or the spider is paused.
|
||||
@@ -60,6 +60,10 @@ When a request comes in, the Session Manager routes it to the correct session ba
|
||||
|
||||
An optional system that, if enabled, saves the crawler's state (pending requests + seen URL fingerprints) to a pickle file on disk. Writes are atomic (temp file + rename) to prevent corruption. Checkpoints are saved periodically at a configurable interval and on graceful shutdown. Upon successful completion (not paused), checkpoint files are automatically cleaned up.
|
||||
|
||||
### Response Cache
|
||||
|
||||
An optional cache that, when development mode is enabled, stores every fetched response on disk and replays it on subsequent runs. Each response is keyed by request fingerprint and serialized as JSON (with the body base64-encoded so binary content survives). It's meant for iterating on `parse()` logic without re-hitting the target servers, not for production use.
|
||||
|
||||
### Output
|
||||
|
||||
Scraped items are collected in an `ItemList` (a list subclass with `to_json()` and `to_jsonl()` export methods). Crawl statistics are tracked in a `CrawlStats` dataclass which contains a lot of useful info.
|
||||
@@ -82,6 +86,7 @@ If you're coming from Scrapy, here's how Scrapling's spider system maps:
|
||||
| Blocked detection | Through custom middlewares | Built-in `is_blocked()` + `retry_blocked_request()` hooks |
|
||||
| Concurrency | `CONCURRENT_REQUESTS` setting | `concurrent_requests` class attribute |
|
||||
| Domain filtering | `allowed_domains` | `allowed_domains` |
|
||||
| Robots.txt | `ROBOTSTXT_OBEY` setting | `robots_txt_obey` class attribute |
|
||||
| Pause/Resume | `JOBDIR` setting | `crawldir` constructor argument |
|
||||
| Export | Feed exports | `result.items.to_json()` / `to_jsonl()` or custom through hooks |
|
||||
| Running | `scrapy crawl spider_name` | `MySpider().start()` |
|
||||
|
||||
@@ -137,3 +137,28 @@ Subdomains are matched automatically, so setting `allowed_domains = {"example.co
|
||||
|
||||
When a request is filtered out, it's counted in `stats.offsite_requests_count` so you can see how many were dropped.
|
||||
|
||||
## Robots.txt Compliance
|
||||
|
||||
Set `robots_txt_obey = True` to make the spider respect robots.txt rules before crawling any domain:
|
||||
|
||||
```python
|
||||
class PoliteSpider(Spider):
|
||||
name = "polite"
|
||||
start_urls = ["https://example.com"]
|
||||
robots_txt_obey = True
|
||||
|
||||
async def parse(self, response: Response):
|
||||
for link in response.css("a::attr(href)").getall():
|
||||
yield response.follow(link, callback=self.parse)
|
||||
```
|
||||
|
||||
When enabled, the spider will:
|
||||
|
||||
1. **Pre-fetch robots.txt** for all domains in `start_urls` before the crawl begins (concurrently).
|
||||
2. **Check every request** against the domain's robots.txt `Disallow` rules. Disallowed requests are silently dropped and counted in `stats.robots_disallowed_count`.
|
||||
3. **Respect `Crawl-delay` and `Request-rate` directives** by taking the maximum of the directive and your configured `download_delay`. This means robots.txt delays never reduce your configured delay, only increase it when needed.
|
||||
|
||||
Robots.txt files are fetched using the spider's default session and cached per domain for the entire crawl. Domains discovered mid-crawl (not in `start_urls`) have their robots.txt fetched on the first request to that domain.
|
||||
|
||||
**Note:** `robots_txt_obey` is turned off by default. It does not affect your concurrency settings -- only the delay between requests is adjusted.
|
||||
|
||||
|
||||
+27
-2
@@ -169,6 +169,26 @@ MySpider().start()
|
||||
<a href="https://proxy-seller.com/?partner=CU9CAA5TBYFFT2" target="_blank">Proxy-Seller</a> يوفر بنية تحتية موثوقة للبروكسي لكشط الويب، بما في ذلك بروكسيات IPv4 وIPv6 وISP والسكنية والمحمولة مع أداء مستقر وتغطية جغرافية واسعة وخطط مرنة لجمع البيانات على نطاق الأعمال.
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td width="200">
|
||||
<a href="http://mangoproxy.com/?utm_source=D4Vinci&utm_medium=GitHub&utm_campaign=D4Vinci" target="_blank" title="Proxies You Can Rely On: Residential, Server, and Mobile">
|
||||
<img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/MangoProxy.png">
|
||||
</a>
|
||||
</td>
|
||||
<td>
|
||||
<a href="http://mangoproxy.com/?utm_source=D4Vinci&utm_medium=GitHub&utm_campaign=D4Vinci" target="_blank">بروكسيات مستقرة</a> للكشط والأتمتة وإدارة الحسابات المتعددة. عناوين IP نظيفة، استجابة سريعة، وأداء موثوق تحت الضغط. مصممة لسير العمل القابل للتوسع.
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td width="200">
|
||||
<a href="https://www.swiftproxy.net/?ref=D4Vinci" target="_blank" title="Scalable Solutions for Web Data Access">
|
||||
<img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/SwiftProxy.png">
|
||||
</a>
|
||||
</td>
|
||||
<td>
|
||||
يوفر <a href="https://www.swiftproxy.net/?ref=D4Vinci" target="_blank">Swiftproxy</a> بروكسيات سكنية قابلة للتوسع مع أكثر من 80 مليون عنوان IP في أكثر من 195 دولة، ويقدم اتصالات سريعة وموثوقة، وتدوير تلقائي، وأداء قوي ضد الحظر. تجربة مجانية متاحة.
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
<i><sub>هل تريد عرض إعلانك هنا؟ انقر [هنا](https://github.com/sponsors/D4Vinci/sponsorships?tier_id=586646)</sub></i>
|
||||
@@ -181,7 +201,9 @@ MySpider().start()
|
||||
<a href="https://hasdata.com/?utm_source=github&utm_medium=banner&utm_campaign=D4Vinci" target="_blank" title="The web scraping service that actually beats anti-bot systems!"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/hasdata.png"></a>
|
||||
<a href="https://proxyempire.io/?ref=scrapling&utm_source=scrapling" target="_blank" title="Collect The Data Your Project Needs with the Best Residential Proxies"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/ProxyEmpire.png"></a>
|
||||
<a href="https://www.webshare.io/?referral_code=48r2m2cd5uz1" target="_blank" title="The Most Reliable Proxy with Unparalleled Performance"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/webshare.png"></a>
|
||||
<a href="https://browser.cash/?utm_source=D4Vinci&utm_medium=referral" target="_blank" title="Browser Automation & AI Browser Agent Platform"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/browserCash.png"></a>
|
||||
<a href="https://www.crawleo.dev/?utm_source=github&utm_medium=sponsor&utm_campaign=scrapling" target="_blank" title="Supercharge your AI with Real-Time Web Intelligence"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/crawleo.png"></a>
|
||||
<a href="https://www.rapidproxy.io/?ref=d4v" target="_blank" title="Affordable Access to the Proxy World – bypass CAPTCHAs blocks, and avoid additional costs."><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/rapidproxy.jpg"></a>
|
||||
|
||||
|
||||
<!-- /sponsors -->
|
||||
|
||||
@@ -198,6 +220,8 @@ MySpider().start()
|
||||
- 💾 **إيقاف واستئناف**: استمرارية الزحف القائمة على Checkpoint. اضغط Ctrl+C للإيقاف بسلاسة؛ أعد التشغيل للاستئناف من حيث توقفت.
|
||||
- 📡 **وضع Streaming**: بث العناصر المستخرجة فور وصولها عبر `async for item in spider.stream()` مع إحصائيات فورية - مثالي لواجهات المستخدم وخطوط الأنابيب وعمليات الزحف الطويلة.
|
||||
- 🛡️ **كشف الطلبات المحظورة**: كشف تلقائي وإعادة محاولة للطلبات المحظورة مع منطق قابل للتخصيص.
|
||||
- 🤖 **الامتثال لـ robots.txt**: خيار `robots_txt_obey` الاختياري الذي يحترم توجيهات `Disallow` و `Crawl-delay` و `Request-rate` مع التخزين المؤقت لكل نطاق.
|
||||
- 🧪 **وضع التطوير**: تخزين الاستجابات على القرص في التشغيل الأول وإعادة تشغيلها في التشغيلات اللاحقة - كرّر العمل على منطق `parse()` دون الحاجة لإرسال طلبات جديدة إلى الخوادم المستهدفة.
|
||||
- 📦 **تصدير مدمج**: صدّر النتائج عبر الخطافات وخط الأنابيب الخاص بك أو JSON/JSONL المدمج مع `result.items.to_json()` / `result.items.to_jsonl()` على التوالي.
|
||||
|
||||
### جلب متقدم للمواقع مع دعم الجلسات
|
||||
@@ -206,7 +230,8 @@ MySpider().start()
|
||||
- **تجاوز مكافحة الروبوتات**: قدرات تخفي متقدمة مع `StealthyFetcher` وانتحال fingerprint. يمكنه تجاوز جميع أنواع Turnstile/Interstitial من Cloudflare بسهولة بالأتمتة.
|
||||
- **إدارة الجلسات**: دعم الجلسات المستمرة مع فئات `FetcherSession` و`StealthySession` و`DynamicSession` لإدارة ملفات تعريف الارتباط والحالة عبر الطلبات.
|
||||
- **تدوير Proxy**: `ProxyRotator` مدمج مع استراتيجيات التدوير الدوري أو المخصصة عبر جميع أنواع الجلسات، بالإضافة إلى تجاوزات Proxy لكل طلب.
|
||||
- **حظر النطاقات**: حظر الطلبات إلى نطاقات محددة (ونطاقاتها الفرعية) في الجوالب المعتمدة على المتصفح.
|
||||
- **حظر النطاقات والإعلانات**: حظر الطلبات إلى نطاقات محددة (ونطاقاتها الفرعية) أو تفعيل حظر الإعلانات المدمج (~3,500 نطاق إعلانات/تتبع معروف) في الجوالب المعتمدة على المتصفح.
|
||||
- **منع تسرب DNS**: دعم اختياري لـ DNS-over-HTTPS لتوجيه استعلامات DNS عبر Cloudflare DoH، مما يمنع تسرب DNS عند استخدام Proxy.
|
||||
- **دعم Async**: دعم async كامل عبر جميع الجوالب وفئات الجلسات async المخصصة.
|
||||
|
||||
### الاستخراج التكيفي والتكامل مع الذكاء الاصطناعي
|
||||
|
||||
+27
-2
@@ -169,6 +169,26 @@ MySpider().start()
|
||||
<a href="https://proxy-seller.com/?partner=CU9CAA5TBYFFT2" target="_blank">Proxy-Seller</a> 提供可靠的网页抓取代理基础设施，包括 IPv4、IPv6、ISP、住宅和移动代理，具备稳定性能、广泛的地理覆盖和灵活的企业级数据采集方案。
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td width="200">
|
||||
<a href="http://mangoproxy.com/?utm_source=D4Vinci&utm_medium=GitHub&utm_campaign=D4Vinci" target="_blank" title="Proxies You Can Rely On: Residential, Server, and Mobile">
|
||||
<img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/MangoProxy.png">
|
||||
</a>
|
||||
</td>
|
||||
<td>
|
||||
<a href="http://mangoproxy.com/?utm_source=D4Vinci&utm_medium=GitHub&utm_campaign=D4Vinci" target="_blank">稳定的代理</a>，适用于数据抓取、自动化和多账号管理。干净的 IP、快速响应、高负载下可靠的性能。专为可扩展的工作流程而构建。
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td width="200">
|
||||
<a href="https://www.swiftproxy.net/?ref=D4Vinci" target="_blank" title="Scalable Solutions for Web Data Access">
|
||||
<img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/SwiftProxy.png">
|
||||
</a>
|
||||
</td>
|
||||
<td>
|
||||
<a href="https://www.swiftproxy.net/?ref=D4Vinci" target="_blank">Swiftproxy</a> 提供可扩展的住宅代理，覆盖 195+ 国家/地区的 8000 万+ IP，提供快速可靠的连接、自动轮换和强大的反屏蔽性能。提供免费试用。
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
<i><sub>想在这里展示您的广告吗？点击 [这里](https://github.com/sponsors/D4Vinci/sponsorships?tier_id=586646)</sub></i>
|
||||
@@ -181,7 +201,9 @@ MySpider().start()
|
||||
<a href="https://hasdata.com/?utm_source=github&utm_medium=banner&utm_campaign=D4Vinci" target="_blank" title="The web scraping service that actually beats anti-bot systems!"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/hasdata.png"></a>
|
||||
<a href="https://proxyempire.io/?ref=scrapling&utm_source=scrapling" target="_blank" title="Collect The Data Your Project Needs with the Best Residential Proxies"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/ProxyEmpire.png"></a>
|
||||
<a href="https://www.webshare.io/?referral_code=48r2m2cd5uz1" target="_blank" title="The Most Reliable Proxy with Unparalleled Performance"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/webshare.png"></a>
|
||||
<a href="https://browser.cash/?utm_source=D4Vinci&utm_medium=referral" target="_blank" title="Browser Automation & AI Browser Agent Platform"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/browserCash.png"></a>
|
||||
<a href="https://www.crawleo.dev/?utm_source=github&utm_medium=sponsor&utm_campaign=scrapling" target="_blank" title="Supercharge your AI with Real-Time Web Intelligence"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/crawleo.png"></a>
|
||||
<a href="https://www.rapidproxy.io/?ref=d4v" target="_blank" title="Affordable Access to the Proxy World – bypass CAPTCHAs blocks, and avoid additional costs."><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/rapidproxy.jpg"></a>
|
||||
|
||||
|
||||
<!-- /sponsors -->
|
||||
|
||||
@@ -198,6 +220,8 @@ MySpider().start()
|
||||
- 💾 **暂停与恢复**：基于 Checkpoint 的爬取持久化。按 Ctrl+C 优雅关闭；重启后从上次停止的地方继续。
|
||||
- 📡 **Streaming 模式**：通过 `async for item in spider.stream()` 以实时统计 Streaming 抓取的数据--非常适合 UI、管道和长时间运行的爬取。
|
||||
- 🛡️ **被阻止请求检测**：自动检测并重试被阻止的请求，支持自定义逻辑。
|
||||
- 🤖 **robots.txt 合规**：可选的 `robots_txt_obey` 标志，支持 `Disallow`、`Crawl-delay` 和 `Request-rate` 指令，并按域名缓存。
|
||||
- 🧪 **开发模式**：首次运行时将响应缓存到磁盘，后续运行时直接回放 - 在不重新请求目标服务器的情况下迭代你的 `parse()` 逻辑。
|
||||
- 📦 **内置导出**：通过钩子和您自己的管道导出结果，或使用内置的 JSON/JSONL，分别通过 `result.items.to_json()`/`result.items.to_jsonl()`。
|
||||
|
||||
### 支持 Session 的高级网站获取
|
||||
@@ -206,7 +230,8 @@ MySpider().start()
|
||||
- **反机器人绕过**：使用 `StealthyFetcher` 的高级隐秘功能和 fingerprint 伪装。可以轻松自动绕过所有类型的 Cloudflare Turnstile/Interstitial。
|
||||
- **Session 管理**：使用 `FetcherSession`、`StealthySession` 和 `DynamicSession` 类实现持久化 Session 支持，用于跨请求的 cookie 和状态管理。
|
||||
- **Proxy 轮换**：内置 `ProxyRotator`，支持轮询或自定义策略，适用于所有 Session 类型，并支持按请求覆盖 Proxy。
|
||||
- **域名屏蔽**：在基于浏览器的 Fetcher 中屏蔽对特定域名（及其子域名）的请求。
|
||||
- **域名和广告屏蔽**：在基于浏览器的 Fetcher 中屏蔽对特定域名（及其子域名）的请求，或启用内置广告屏蔽（约 3,500 个已知广告/追踪域名）。
|
||||
- **DNS 泄漏防护**：可选的 DNS-over-HTTPS 支持，通过 Cloudflare 的 DoH 路由 DNS 查询，防止使用代理时的 DNS 泄漏。
|
||||
- **Async 支持**：所有 Fetcher 和专用 async Session 类的完整 async 支持。
|
||||
|
||||
### 自适应抓取和 AI 集成
|
||||
|
||||
+27
-2
@@ -169,6 +169,26 @@ MySpider().start()
|
||||
<a href="https://proxy-seller.com/?partner=CU9CAA5TBYFFT2" target="_blank">Proxy-Seller</a> bietet zuverlässige Proxy-Infrastruktur für Web Scraping mit IPv4-, IPv6-, ISP-, Residential- und Mobile-Proxys – stabile Leistung, breite geografische Abdeckung und flexible Tarife für die Datenerfassung im Unternehmensmaßstab.
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td width="200">
|
||||
<a href="http://mangoproxy.com/?utm_source=D4Vinci&utm_medium=GitHub&utm_campaign=D4Vinci" target="_blank" title="Proxies You Can Rely On: Residential, Server, and Mobile">
|
||||
<img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/MangoProxy.png">
|
||||
</a>
|
||||
</td>
|
||||
<td>
|
||||
<a href="http://mangoproxy.com/?utm_source=D4Vinci&utm_medium=GitHub&utm_campaign=D4Vinci" target="_blank">Stabile Proxys</a> für Scraping, Automatisierung und Multi-Accounting. Saubere IPs, schnelle Reaktionszeiten und zuverlässige Leistung unter Last. Entwickelt für skalierbare Workflows.
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td width="200">
|
||||
<a href="https://www.swiftproxy.net/?ref=D4Vinci" target="_blank" title="Scalable Solutions for Web Data Access">
|
||||
<img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/SwiftProxy.png">
|
||||
</a>
|
||||
</td>
|
||||
<td>
|
||||
<a href="https://www.swiftproxy.net/?ref=D4Vinci" target="_blank">Swiftproxy</a> bietet skalierbare Residential-Proxys mit über 80 Mio. IPs in mehr als 195 Ländern und liefert schnelle, zuverlässige Verbindungen, automatische Rotation und starke Anti-Block-Leistung. Kostenlose Testversion verfügbar.
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
<i><sub>Möchten Sie Ihre Anzeige hier zeigen? Klicken Sie [hier](https://github.com/sponsors/D4Vinci/sponsorships?tier_id=586646)</sub></i>
|
||||
@@ -181,7 +201,9 @@ MySpider().start()
|
||||
<a href="https://hasdata.com/?utm_source=github&utm_medium=banner&utm_campaign=D4Vinci" target="_blank" title="The web scraping service that actually beats anti-bot systems!"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/hasdata.png"></a>
|
||||
<a href="https://proxyempire.io/?ref=scrapling&utm_source=scrapling" target="_blank" title="Collect The Data Your Project Needs with the Best Residential Proxies"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/ProxyEmpire.png"></a>
|
||||
<a href="https://www.webshare.io/?referral_code=48r2m2cd5uz1" target="_blank" title="The Most Reliable Proxy with Unparalleled Performance"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/webshare.png"></a>
|
||||
<a href="https://browser.cash/?utm_source=D4Vinci&utm_medium=referral" target="_blank" title="Browser Automation & AI Browser Agent Platform"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/browserCash.png"></a>
|
||||
<a href="https://www.crawleo.dev/?utm_source=github&utm_medium=sponsor&utm_campaign=scrapling" target="_blank" title="Supercharge your AI with Real-Time Web Intelligence"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/crawleo.png"></a>
|
||||
<a href="https://www.rapidproxy.io/?ref=d4v" target="_blank" title="Affordable Access to the Proxy World – bypass CAPTCHAs blocks, and avoid additional costs."><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/rapidproxy.jpg"></a>
|
||||
|
||||
|
||||
<!-- /sponsors -->
|
||||
|
||||
@@ -198,6 +220,8 @@ MySpider().start()
|
||||
- 💾 **Pause & Resume**: Checkpoint-basierte Crawl-Persistenz. Drücken Sie Strg+C für ein kontrolliertes Herunterfahren; starten Sie neu, um dort fortzufahren, wo Sie aufgehört haben.
|
||||
- 📡 **Streaming-Modus**: Gescrapte Elemente in Echtzeit streamen über `async for item in spider.stream()` mit Echtzeit-Statistiken -- ideal für UI, Pipelines und lang laufende Crawls.
|
||||
- 🛡️ **Erkennung blockierter Anfragen**: Automatische Erkennung und Wiederholung blockierter Anfragen mit anpassbarer Logik.
|
||||
- 🤖 **robots.txt-Konformität**: Optionales `robots_txt_obey`-Flag, das `Disallow`-, `Crawl-delay`- und `Request-rate`-Direktiven mit domainbasiertem Caching respektiert.
|
||||
- 🧪 **Entwicklungsmodus**: Antworten beim ersten Lauf auf der Festplatte zwischenspeichern und bei weiteren Läufen erneut abspielen - iterieren Sie an Ihrer `parse()`-Logik, ohne die Zielserver erneut abzufragen.
|
||||
- 📦 **Integrierter Export**: Ergebnisse über Hooks und Ihre eigene Pipeline oder den integrierten JSON/JSONL-Export mit `result.items.to_json()` / `result.items.to_jsonl()` exportieren.
|
||||
|
||||
### Erweitertes Website-Abrufen mit Session-Unterstützung
|
||||
@@ -206,7 +230,8 @@ MySpider().start()
|
||||
- **Anti-Bot-Umgehung**: Erweiterte Stealth-Fähigkeiten mit `StealthyFetcher` und Fingerprint-Spoofing. Kann alle Arten von Cloudflares Turnstile/Interstitial einfach mit Automatisierung umgehen.
|
||||
- **Session-Verwaltung**: Persistente Session-Unterstützung mit den Klassen `FetcherSession`, `StealthySession` und `DynamicSession` für Cookie- und Zustandsverwaltung über Anfragen hinweg.
|
||||
- **Proxy-Rotation**: Integrierter `ProxyRotator` mit zyklischen oder benutzerdefinierten Rotationsstrategien über alle Session-Typen hinweg, plus Proxy-Überschreibungen pro Anfrage.
|
||||
- **Domain-Blockierung**: Anfragen an bestimmte Domains (und deren Subdomains) in browserbasierten Fetchern blockieren.
|
||||
- **Domain- & Werbeblockierung**: Anfragen an bestimmte Domains (und deren Subdomains) blockieren oder die integrierte Werbeblockierung (~3.500 bekannte Werbe-/Tracker-Domains) in browserbasierten Fetchern aktivieren.
|
||||
- **DNS-Leak-Prävention**: Optionale DNS-over-HTTPS-Unterstützung zur Weiterleitung von DNS-Anfragen über Cloudflares DoH, um DNS-Leaks bei der Verwendung von Proxys zu verhindern.
|
||||
- **Async-Unterstützung**: Vollständige async-Unterstützung über alle Fetcher und dedizierte async Session-Klassen hinweg.
|
||||
|
||||
### Adaptives Scraping & KI-Integration
|
||||
|
||||
+27
-2
@@ -169,6 +169,26 @@ MySpider().start()
|
||||
<a href="https://proxy-seller.com/?partner=CU9CAA5TBYFFT2" target="_blank">Proxy-Seller</a> ofrece una infraestructura de proxy fiable para web scraping, con proxies IPv4, IPv6, ISP, residenciales y móviles con rendimiento estable, amplia cobertura geográfica y planes flexibles para la recopilación de datos a escala empresarial.
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td width="200">
|
||||
<a href="http://mangoproxy.com/?utm_source=D4Vinci&utm_medium=GitHub&utm_campaign=D4Vinci" target="_blank" title="Proxies You Can Rely On: Residential, Server, and Mobile">
|
||||
<img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/MangoProxy.png">
|
||||
</a>
|
||||
</td>
|
||||
<td>
|
||||
<a href="http://mangoproxy.com/?utm_source=D4Vinci&utm_medium=GitHub&utm_campaign=D4Vinci" target="_blank">Proxies estables</a> para scraping, automatización y multicuentas. IPs limpias, respuesta rápida y rendimiento fiable bajo carga. Diseñado para flujos de trabajo escalables.
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td width="200">
|
||||
<a href="https://www.swiftproxy.net/?ref=D4Vinci" target="_blank" title="Scalable Solutions for Web Data Access">
|
||||
<img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/SwiftProxy.png">
|
||||
</a>
|
||||
</td>
|
||||
<td>
|
||||
<a href="https://www.swiftproxy.net/?ref=D4Vinci" target="_blank">Swiftproxy</a> ofrece proxies residenciales escalables con más de 80 millones de IPs en más de 195 países, brindando conexiones rápidas y fiables, rotación automática y un sólido rendimiento anti-bloqueo. Prueba gratuita disponible.
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
<i><sub>¿Quieres mostrar tu anuncio aquí? Haz clic [aquí](https://github.com/sponsors/D4Vinci/sponsorships?tier_id=586646)</sub></i>
|
||||
@@ -181,7 +201,9 @@ MySpider().start()
|
||||
<a href="https://hasdata.com/?utm_source=github&utm_medium=banner&utm_campaign=D4Vinci" target="_blank" title="The web scraping service that actually beats anti-bot systems!"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/hasdata.png"></a>
|
||||
<a href="https://proxyempire.io/?ref=scrapling&utm_source=scrapling" target="_blank" title="Collect The Data Your Project Needs with the Best Residential Proxies"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/ProxyEmpire.png"></a>
|
||||
<a href="https://www.webshare.io/?referral_code=48r2m2cd5uz1" target="_blank" title="The Most Reliable Proxy with Unparalleled Performance"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/webshare.png"></a>
|
||||
<a href="https://browser.cash/?utm_source=D4Vinci&utm_medium=referral" target="_blank" title="Browser Automation & AI Browser Agent Platform"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/browserCash.png"></a>
|
||||
<a href="https://www.crawleo.dev/?utm_source=github&utm_medium=sponsor&utm_campaign=scrapling" target="_blank" title="Supercharge your AI with Real-Time Web Intelligence"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/crawleo.png"></a>
|
||||
<a href="https://www.rapidproxy.io/?ref=d4v" target="_blank" title="Affordable Access to the Proxy World – bypass CAPTCHAs blocks, and avoid additional costs."><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/rapidproxy.jpg"></a>
|
||||
|
||||
|
||||
<!-- /sponsors -->
|
||||
|
||||
@@ -198,6 +220,8 @@ MySpider().start()
|
||||
- 💾 **Pause & Resume**: Persistencia de rastreo basada en Checkpoint. Presiona Ctrl+C para un cierre ordenado; reinicia para continuar desde donde lo dejaste.
|
||||
- 📡 **Modo Streaming**: Transmite elementos extraídos a medida que llegan con `async for item in spider.stream()` con estadísticas en tiempo real - ideal para UI, pipelines y rastreos de larga duración.
|
||||
- 🛡️ **Detección de Solicitudes Bloqueadas**: Detección automática y reintento de solicitudes bloqueadas con lógica personalizable.
|
||||
- 🤖 **Cumplimiento de robots.txt**: Flag opcional `robots_txt_obey` que respeta las directivas `Disallow`, `Crawl-delay` y `Request-rate` con caché por dominio.
|
||||
- 🧪 **Modo de Desarrollo**: Almacena las respuestas en disco en la primera ejecución y las reproduce en ejecuciones posteriores - itera sobre tu lógica de `parse()` sin volver a consultar los servidores objetivo.
|
||||
- 📦 **Exportación Integrada**: Exporta resultados a través de hooks y tu propio pipeline o el JSON/JSONL integrado con `result.items.to_json()` / `result.items.to_jsonl()` respectivamente.
|
||||
|
||||
### Obtención Avanzada de Sitios Web con Soporte de Session
|
||||
@@ -206,7 +230,8 @@ MySpider().start()
|
||||
- **Evasión Anti-bot**: Capacidades de sigilo avanzadas con `StealthyFetcher` y falsificación de fingerprint. Puede evadir fácilmente todos los tipos de Turnstile/Interstitial de Cloudflare con automatización.
|
||||
- **Gestión de Session**: Soporte de sesión persistente con las clases `FetcherSession`, `StealthySession` y `DynamicSession` para la gestión de cookies y estado entre solicitudes.
|
||||
- **Rotación de Proxy**: `ProxyRotator` integrado con estrategias de rotación cíclica o personalizadas en todos los tipos de sesión, además de sobrescrituras de Proxy por solicitud.
|
||||
- **Bloqueo de Dominios**: Bloquea solicitudes a dominios específicos (y sus subdominios) en fetchers basados en navegador.
|
||||
- **Bloqueo de Dominios y Anuncios**: Bloquea solicitudes a dominios específicos (y sus subdominios) o activa el bloqueo de anuncios integrado (~3,500 dominios de anuncios/rastreadores conocidos) en fetchers basados en navegador.
|
||||
- **Prevención de Fugas DNS**: Soporte opcional de DNS-over-HTTPS para enrutar consultas DNS a través del DoH de Cloudflare, previniendo fugas DNS al usar proxies.
|
||||
- **Soporte Async**: Soporte async completo en todos los fetchers y clases de sesión async dedicadas.
|
||||
|
||||
### Scraping Adaptativo e Integración con IA
|
||||
|
||||
+27
-2
@@ -169,6 +169,26 @@ MySpider().start()
|
||||
<a href="https://proxy-seller.com/?partner=CU9CAA5TBYFFT2" target="_blank">Proxy-Seller</a> fournit une infrastructure proxy fiable pour le web scraping, avec des proxys IPv4, IPv6, ISP, résidentiels et mobiles offrant des performances stables, une large couverture géographique et des plans flexibles pour la collecte de données à l'échelle entreprise.
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td width="200">
|
||||
<a href="http://mangoproxy.com/?utm_source=D4Vinci&utm_medium=GitHub&utm_campaign=D4Vinci" target="_blank" title="Proxies You Can Rely On: Residential, Server, and Mobile">
|
||||
<img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/MangoProxy.png">
|
||||
</a>
|
||||
</td>
|
||||
<td>
|
||||
<a href="http://mangoproxy.com/?utm_source=D4Vinci&utm_medium=GitHub&utm_campaign=D4Vinci" target="_blank">Des proxys stables</a> pour le scraping, l'automatisation et la gestion multi-comptes. Des IPs propres, une réponse rapide et des performances fiables sous charge. Conçu pour des flux de travail évolutifs.
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td width="200">
|
||||
<a href="https://www.swiftproxy.net/?ref=D4Vinci" target="_blank" title="Scalable Solutions for Web Data Access">
|
||||
<img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/SwiftProxy.png">
|
||||
</a>
|
||||
</td>
|
||||
<td>
|
||||
<a href="https://www.swiftproxy.net/?ref=D4Vinci" target="_blank">Swiftproxy</a> propose des proxys résidentiels évolutifs avec plus de 80 millions d'IPs dans plus de 195 pays, offrant des connexions rapides et fiables, une rotation automatique et de solides performances anti-blocage. Essai gratuit disponible.
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
<i><sub>Vous souhaitez afficher votre publicité ici ? Cliquez [ici](https://github.com/sponsors/D4Vinci/sponsorships?tier_id=586646)</sub></i>
|
||||
@@ -181,7 +201,9 @@ MySpider().start()
|
||||
<a href="https://hasdata.com/?utm_source=github&utm_medium=banner&utm_campaign=D4Vinci" target="_blank" title="The web scraping service that actually beats anti-bot systems!"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/hasdata.png"></a>
|
||||
<a href="https://proxyempire.io/?ref=scrapling&utm_source=scrapling" target="_blank" title="Collect The Data Your Project Needs with the Best Residential Proxies"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/ProxyEmpire.png"></a>
|
||||
<a href="https://www.webshare.io/?referral_code=48r2m2cd5uz1" target="_blank" title="The Most Reliable Proxy with Unparalleled Performance"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/webshare.png"></a>
|
||||
<a href="https://browser.cash/?utm_source=D4Vinci&utm_medium=referral" target="_blank" title="Browser Automation & AI Browser Agent Platform"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/browserCash.png"></a>
|
||||
<a href="https://www.crawleo.dev/?utm_source=github&utm_medium=sponsor&utm_campaign=scrapling" target="_blank" title="Supercharge your AI with Real-Time Web Intelligence"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/crawleo.png"></a>
|
||||
<a href="https://www.rapidproxy.io/?ref=d4v" target="_blank" title="Affordable Access to the Proxy World – bypass CAPTCHAs blocks, and avoid additional costs."><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/rapidproxy.jpg"></a>
|
||||
|
||||
|
||||
<!-- /sponsors -->
|
||||
|
||||
@@ -198,6 +220,8 @@ MySpider().start()
|
||||
- 💾 **Pause & Reprise** : Persistance du crawl basée sur des checkpoints. Appuyez sur Ctrl+C pour un arrêt gracieux ; redémarrez pour reprendre là où vous vous étiez arrêté.
|
||||
- 📡 **Mode streaming** : Diffusez les éléments scrapés en temps réel via `async for item in spider.stream()` avec des statistiques en temps réel - idéal pour les UI, pipelines et crawls de longue durée.
|
||||
- 🛡️ **Détection des requêtes bloquées** : Détection automatique et réessai des requêtes bloquées avec une logique personnalisable.
|
||||
- 🤖 **Conformité robots.txt** : Flag optionnel `robots_txt_obey` qui respecte les directives `Disallow`, `Crawl-delay` et `Request-rate` avec mise en cache par domaine.
|
||||
- 🧪 **Mode développement** : Mettez les réponses en cache sur le disque lors de la première exécution et rejouez-les lors des exécutions suivantes - itérez sur votre logique `parse()` sans solliciter à nouveau les serveurs cibles.
|
||||
- 📦 **Export intégré** : Exportez les résultats via des hooks et votre propre pipeline ou l'export JSON/JSONL intégré avec `result.items.to_json()` / `result.items.to_jsonl()` respectivement.
|
||||
|
||||
### Récupération avancée de sites web avec support de sessions
|
||||
@@ -206,7 +230,8 @@ MySpider().start()
|
||||
- **Contournement anti-bot** : Capacités de furtivité avancées avec `StealthyFetcher` et usurpation d'empreinte. Peut facilement contourner tous les types de Turnstile/Interstitial de Cloudflare avec l'automatisation.
|
||||
- **Gestion de sessions** : Support de sessions persistantes avec les classes `FetcherSession`, `StealthySession` et `DynamicSession` pour la gestion des cookies et de l'état entre les requêtes.
|
||||
- **Rotation de proxy** : `ProxyRotator` intégré avec des stratégies de rotation cycliques ou personnalisées sur tous les types de sessions, plus des surcharges de proxy par requête.
|
||||
- **Blocage de domaines** : Bloquez les requêtes vers des domaines spécifiques (et leurs sous-domaines) dans les fetchers basés sur navigateur.
|
||||
- **Blocage de domaines et publicités** : Bloquez les requêtes vers des domaines spécifiques (et leurs sous-domaines) ou activez le blocage de publicités intégré (~3 500 domaines publicitaires/traceurs connus) dans les fetchers basés sur navigateur.
|
||||
- **Prévention des fuites DNS** : Support optionnel de DNS-over-HTTPS pour router les requêtes DNS via le DoH de Cloudflare, empêchant les fuites DNS lors de l'utilisation de proxies.
|
||||
- **Support async** : Support async complet sur tous les fetchers et classes de sessions async dédiées.
|
||||
|
||||
### Scraping adaptatif & Intégration IA
|
||||
|
||||
+27
-2
@@ -169,6 +169,26 @@ MySpider().start()
|
||||
<a href="https://proxy-seller.com/?partner=CU9CAA5TBYFFT2" target="_blank">Proxy-Seller</a> は Web スクレイピング向けの信頼性の高いプロキシインフラを提供しています。IPv4、IPv6、ISP、レジデンシャル、モバイルプロキシに対応し、安定したパフォーマンス、幅広い地理的カバレッジ、企業規模のデータ収集に柔軟なプランを備えています。
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td width="200">
|
||||
<a href="http://mangoproxy.com/?utm_source=D4Vinci&utm_medium=GitHub&utm_campaign=D4Vinci" target="_blank" title="Proxies You Can Rely On: Residential, Server, and Mobile">
|
||||
<img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/MangoProxy.png">
|
||||
</a>
|
||||
</td>
|
||||
<td>
|
||||
<a href="http://mangoproxy.com/?utm_source=D4Vinci&utm_medium=GitHub&utm_campaign=D4Vinci" target="_blank">安定したプロキシ</a>。スクレイピング、自動化、マルチアカウント管理に対応。クリーンな IP、高速レスポンス、高負荷時でも信頼性の高いパフォーマンス。スケーラブルなワークフロー向けに設計。
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td width="200">
|
||||
<a href="https://www.swiftproxy.net/?ref=D4Vinci" target="_blank" title="Scalable Solutions for Web Data Access">
|
||||
<img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/SwiftProxy.png">
|
||||
</a>
|
||||
</td>
|
||||
<td>
|
||||
<a href="https://www.swiftproxy.net/?ref=D4Vinci" target="_blank">Swiftproxy</a> は195カ国以上、8,000万以上のIPを備えたスケーラブルな住宅用プロキシを提供し、高速で信頼性の高い接続、自動ローテーション、強力なブロック回避性能を実現します。無料トライアルあり。
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
<i><sub>ここに広告を表示したいですか？[こちら](https://github.com/sponsors/D4Vinci/sponsorships?tier_id=586646)をクリック</sub></i>
|
||||
@@ -181,7 +201,9 @@ MySpider().start()
|
||||
<a href="https://hasdata.com/?utm_source=github&utm_medium=banner&utm_campaign=D4Vinci" target="_blank" title="The web scraping service that actually beats anti-bot systems!"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/hasdata.png"></a>
|
||||
<a href="https://proxyempire.io/?ref=scrapling&utm_source=scrapling" target="_blank" title="Collect The Data Your Project Needs with the Best Residential Proxies"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/ProxyEmpire.png"></a>
|
||||
<a href="https://www.webshare.io/?referral_code=48r2m2cd5uz1" target="_blank" title="The Most Reliable Proxy with Unparalleled Performance"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/webshare.png"></a>
|
||||
<a href="https://browser.cash/?utm_source=D4Vinci&utm_medium=referral" target="_blank" title="Browser Automation & AI Browser Agent Platform"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/browserCash.png"></a>
|
||||
<a href="https://www.crawleo.dev/?utm_source=github&utm_medium=sponsor&utm_campaign=scrapling" target="_blank" title="Supercharge your AI with Real-Time Web Intelligence"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/crawleo.png"></a>
|
||||
<a href="https://www.rapidproxy.io/?ref=d4v" target="_blank" title="Affordable Access to the Proxy World – bypass CAPTCHAs blocks, and avoid additional costs."><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/rapidproxy.jpg"></a>
|
||||
|
||||
|
||||
<!-- /sponsors -->
|
||||
|
||||
@@ -198,6 +220,8 @@ MySpider().start()
|
||||
- 💾 **Pause & Resume**：Checkpoint ベースのクロール永続化。Ctrl+C で正常にシャットダウン；再起動すると中断したところから再開。
|
||||
- 📡 **Streaming モード**：`async for item in spider.stream()` でリアルタイム統計とともにスクレイプされたアイテムを Streaming で受信 - UI、パイプライン、長時間実行クロールに最適。
|
||||
- 🛡️ **ブロックされたリクエストの検出**：カスタマイズ可能なロジックによるブロックされたリクエストの自動検出とリトライ。
|
||||
- 🤖 **robots.txt 準拠**：オプションの `robots_txt_obey` フラグで `Disallow`、`Crawl-delay`、`Request-rate` ディレクティブをドメインごとのキャッシュで遵守。
|
||||
- 🧪 **開発モード**：初回実行時にレスポンスをディスクにキャッシュし、以降の実行ではそれを再生 - ターゲットサーバーに再リクエストすることなく `parse()` ロジックを反復開発できます。
|
||||
- 📦 **組み込みエクスポート**：フックや独自のパイプライン、または組み込みの JSON/JSONL で結果をエクスポート。それぞれ`result.items.to_json()` / `result.items.to_jsonl()`を使用。
|
||||
|
||||
### Session サポート付き高度なウェブサイト取得
|
||||
@@ -206,7 +230,8 @@ MySpider().start()
|
||||
- **アンチボット回避**：`StealthyFetcher` と fingerprint 偽装による高度なステルス機能。自動化で Cloudflare の Turnstile/Interstitial のすべてのタイプを簡単に回避。
|
||||
- **Session 管理**：リクエスト間で Cookie と状態を管理するための `FetcherSession`、`StealthySession`、`DynamicSession` クラスによる永続的な Session サポート。
|
||||
- **Proxy 回転**：すべての Session タイプに対応したラウンドロビンまたはカスタム戦略の組み込み `ProxyRotator`、さらにリクエストごとの Proxy オーバーライド。
|
||||
- **ドメインブロック**：ブラウザベースの Fetcher で特定のドメイン（およびそのサブドメイン）へのリクエストをブロック。
|
||||
- **ドメイン＆広告ブロック**：ブラウザベースの Fetcher で特定のドメイン（およびそのサブドメイン）へのリクエストをブロック、または内蔵広告ブロック（約3,500の既知の広告/トラッカードメイン）を有効化。
|
||||
- **DNS リーク防止**：Proxy 使用時の DNS リークを防ぐため、Cloudflare の DoH 経由で DNS クエリをルーティングするオプションの DNS-over-HTTPS サポート。
|
||||
- **async サポート**：すべての Fetcher および専用 async Session クラス全体での完全な async サポート。
|
||||
|
||||
### 適応型スクレイピングと AI 統合
|
||||
|
||||
+27
-2
@@ -169,6 +169,26 @@ MySpider().start()
|
||||
<a href="https://proxy-seller.com/?partner=CU9CAA5TBYFFT2" target="_blank">Proxy-Seller</a>는 웹 스크래핑을 위한 안정적인 프록시 인프라를 제공합니다. IPv4, IPv6, ISP, 주거용 및 모바일 프록시를 지원하며, 안정적인 성능, 광범위한 지역 커버리지, 기업 규모의 데이터 수집을 위한 유연한 요금제를 갖추고 있습니다.
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td width="200">
|
||||
<a href="http://mangoproxy.com/?utm_source=D4Vinci&utm_medium=GitHub&utm_campaign=D4Vinci" target="_blank" title="Proxies You Can Rely On: Residential, Server, and Mobile">
|
||||
<img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/MangoProxy.png">
|
||||
</a>
|
||||
</td>
|
||||
<td>
|
||||
<a href="http://mangoproxy.com/?utm_source=D4Vinci&utm_medium=GitHub&utm_campaign=D4Vinci" target="_blank">안정적인 프록시</a>. 스크래핑, 자동화, 멀티 계정 관리에 적합합니다. 깨끗한 IP, 빠른 응답, 높은 부하에서도 신뢰할 수 있는 성능. 확장 가능한 워크플로우를 위해 설계되었습니다.
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td width="200">
|
||||
<a href="https://www.swiftproxy.net/?ref=D4Vinci" target="_blank" title="Scalable Solutions for Web Data Access">
|
||||
<img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/SwiftProxy.png">
|
||||
</a>
|
||||
</td>
|
||||
<td>
|
||||
<a href="https://www.swiftproxy.net/?ref=D4Vinci" target="_blank">Swiftproxy</a>는 195개국 이상에서 8천만 개 이상의 IP를 갖춘 확장 가능한 주거용 프록시를 제공하며, 빠르고 안정적인 연결, 자동 회전, 강력한 차단 방지 성능을 제공합니다. 무료 체험판 이용 가능.
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
<i><sub>여기에 광고를 게재하고 싶으신가요? [여기](https://github.com/sponsors/D4Vinci/sponsorships?tier_id=586646)를 클릭하세요</sub></i>
|
||||
@@ -181,7 +201,9 @@ MySpider().start()
|
||||
<a href="https://hasdata.com/?utm_source=github&utm_medium=banner&utm_campaign=D4Vinci" target="_blank" title="The web scraping service that actually beats anti-bot systems!"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/hasdata.png"></a>
|
||||
<a href="https://proxyempire.io/?ref=scrapling&utm_source=scrapling" target="_blank" title="Collect The Data Your Project Needs with the Best Residential Proxies"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/ProxyEmpire.png"></a>
|
||||
<a href="https://www.webshare.io/?referral_code=48r2m2cd5uz1" target="_blank" title="The Most Reliable Proxy with Unparalleled Performance"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/webshare.png"></a>
|
||||
<a href="https://browser.cash/?utm_source=D4Vinci&utm_medium=referral" target="_blank" title="Browser Automation & AI Browser Agent Platform"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/browserCash.png"></a>
|
||||
<a href="https://www.crawleo.dev/?utm_source=github&utm_medium=sponsor&utm_campaign=scrapling" target="_blank" title="Supercharge your AI with Real-Time Web Intelligence"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/crawleo.png"></a>
|
||||
<a href="https://www.rapidproxy.io/?ref=d4v" target="_blank" title="Affordable Access to the Proxy World – bypass CAPTCHAs blocks, and avoid additional costs."><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/rapidproxy.jpg"></a>
|
||||
|
||||
|
||||
<!-- /sponsors -->
|
||||
|
||||
@@ -198,6 +220,8 @@ MySpider().start()
|
||||
- 💾 **일시정지 & 재개**: 체크포인트 기반의 크롤링 영속화. Ctrl+C로 정상 종료하고, 재시작하면 중단된 지점부터 이어갑니다.
|
||||
- 📡 **스트리밍 모드**: `async for item in spider.stream()`으로 스크레이핑된 아이템을 실시간 통계와 함께 스트리밍으로 수신 - UI, 파이프라인, 장시간 크롤링에 적합합니다.
|
||||
- 🛡️ **차단된 요청 감지**: 커스텀 로직을 통한 차단된 요청의 자동 감지 및 재시도를 지원합니다.
|
||||
- 🤖 **robots.txt 준수**: 선택적 `robots_txt_obey` 플래그로 `Disallow`, `Crawl-delay`, `Request-rate` 지시문을 도메인별 캐싱과 함께 준수합니다.
|
||||
- 🧪 **개발 모드**: 첫 실행 시 응답을 디스크에 캐싱하고 이후 실행에서는 캐시된 응답을 재생합니다 - 대상 서버에 다시 요청하지 않고 `parse()` 로직을 반복 개발할 수 있습니다.
|
||||
- 📦 **내장 내보내기**: 훅이나 자체 파이프라인, 또는 내장 JSON/JSONL로 결과를 내보냅니다. 각각 `result.items.to_json()` / `result.items.to_jsonl()`을 사용합니다.
|
||||
|
||||
### 세션을 지원하는 고급 웹사이트 가져오기
|
||||
@@ -206,7 +230,8 @@ MySpider().start()
|
||||
- **안티봇 우회**: `StealthyFetcher`와 fingerprint 위장을 통한 고급 스텔스 기능. 자동화로 모든 유형의 Cloudflare Turnstile/Interstitial을 손쉽게 우회합니다.
|
||||
- **세션 관리**: `FetcherSession`, `StealthySession`, `DynamicSession` 클래스로 요청 간 쿠키와 상태를 관리하는 영속적 세션을 지원합니다.
|
||||
- **프록시 로테이션**: 모든 세션 타입에 대응하는 순환 또는 커스텀 전략의 내장 `ProxyRotator`와 요청별 프록시 오버라이드를 제공합니다.
|
||||
- **도메인 차단**: 브라우저 기반 Fetcher에서 특정 도메인(및 하위 도메인)으로의 요청을 차단합니다.
|
||||
- **도메인 및 광고 차단**: 브라우저 기반 Fetcher에서 특정 도메인(및 하위 도메인)으로의 요청을 차단하거나 내장 광고 차단(약 3,500개의 알려진 광고/트래커 도메인)을 활성화합니다.
|
||||
- **DNS 유출 방지**: 프록시 사용 시 DNS 유출을 방지하기 위해 Cloudflare DoH를 통해 DNS 쿼리를 라우팅하는 선택적 DNS-over-HTTPS 지원.
|
||||
- **비동기 지원**: 모든 Fetcher와 전용 비동기 세션 클래스에서 완전한 비동기를 지원합니다.
|
||||
|
||||
### 적응형 스크레이핑 & AI 통합
|
||||
|
||||
+27
-2
@@ -172,6 +172,26 @@ MySpider().start()
|
||||
<a href="https://proxy-seller.com/?partner=CU9CAA5TBYFFT2" target="_blank">Proxy-Seller</a> предоставляет надёжную прокси-инфраструктуру для веб-скрейпинга: IPv4, IPv6, ISP, резидентные и мобильные прокси со стабильной производительностью, широким географическим покрытием и гибкими тарифами для сбора данных в масштабах бизнеса.
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td width="200">
|
||||
<a href="http://mangoproxy.com/?utm_source=D4Vinci&utm_medium=GitHub&utm_campaign=D4Vinci" target="_blank" title="Proxies You Can Rely On: Residential, Server, and Mobile">
|
||||
<img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/MangoProxy.png">
|
||||
</a>
|
||||
</td>
|
||||
<td>
|
||||
<a href="http://mangoproxy.com/?utm_source=D4Vinci&utm_medium=GitHub&utm_campaign=D4Vinci" target="_blank">Стабильные прокси</a> для скрапинга, автоматизации и мультиаккаунтинга. Чистые IP, быстрый отклик и надёжная работа под нагрузкой. Созданы для масштабируемых рабочих процессов.
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td width="200">
|
||||
<a href="https://www.swiftproxy.net/?ref=D4Vinci" target="_blank" title="Scalable Solutions for Web Data Access">
|
||||
<img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/SwiftProxy.png">
|
||||
</a>
|
||||
</td>
|
||||
<td>
|
||||
<a href="https://www.swiftproxy.net/?ref=D4Vinci" target="_blank">Swiftproxy</a> предоставляет масштабируемые резидентные прокси с более чем 80 млн IP в 195+ странах, обеспечивая быстрые и надёжные соединения, автоматическую ротацию и высокую устойчивость к блокировкам. Доступна бесплатная пробная версия.
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
<i><sub>Хотите показать здесь свою рекламу? Нажмите [здесь](https://github.com/sponsors/D4Vinci/sponsorships?tier_id=586646)</sub></i>
|
||||
@@ -184,7 +204,9 @@ MySpider().start()
|
||||
<a href="https://hasdata.com/?utm_source=github&utm_medium=banner&utm_campaign=D4Vinci" target="_blank" title="The web scraping service that actually beats anti-bot systems!"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/hasdata.png"></a>
|
||||
<a href="https://proxyempire.io/?ref=scrapling&utm_source=scrapling" target="_blank" title="Collect The Data Your Project Needs with the Best Residential Proxies"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/ProxyEmpire.png"></a>
|
||||
<a href="https://www.webshare.io/?referral_code=48r2m2cd5uz1" target="_blank" title="The Most Reliable Proxy with Unparalleled Performance"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/webshare.png"></a>
|
||||
<a href="https://browser.cash/?utm_source=D4Vinci&utm_medium=referral" target="_blank" title="Browser Automation & AI Browser Agent Platform"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/browserCash.png"></a>
|
||||
<a href="https://www.crawleo.dev/?utm_source=github&utm_medium=sponsor&utm_campaign=scrapling" target="_blank" title="Supercharge your AI with Real-Time Web Intelligence"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/crawleo.png"></a>
|
||||
<a href="https://www.rapidproxy.io/?ref=d4v" target="_blank" title="Affordable Access to the Proxy World – bypass CAPTCHAs blocks, and avoid additional costs."><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/rapidproxy.jpg"></a>
|
||||
|
||||
|
||||
<!-- /sponsors -->
|
||||
|
||||
@@ -201,6 +223,8 @@ MySpider().start()
|
||||
- 💾 **Pause & Resume**: Persistence обхода на основе Checkpoint'ов. Нажмите Ctrl+C для мягкой остановки; перезапустите, чтобы продолжить с того места, где вы остановились.
|
||||
- 📡 **Режим Streaming**: Стримьте извлечённые элементы по мере их поступления через `async for item in spider.stream()` со статистикой в реальном времени - идеально для UI, конвейеров и длительных обходов.
|
||||
- 🛡️ **Обнаружение заблокированных запросов**: Автоматическое обнаружение и повторная отправка заблокированных запросов с настраиваемой логикой.
|
||||
- 🤖 **Соответствие robots.txt**: Опциональный флаг `robots_txt_obey`, который учитывает директивы `Disallow`, `Crawl-delay` и `Request-rate` с кэшированием по доменам.
|
||||
- 🧪 **Режим разработки**: Кэшируйте ответы на диск при первом запуске и воспроизводите их при последующих запусках - итерируйте над логикой `parse()`, не отправляя повторные запросы к целевым серверам.
|
||||
- 📦 **Встроенный экспорт**: Экспортируйте результаты через хуки и собственный конвейер или встроенный JSON/JSONL с `result.items.to_json()` / `result.items.to_jsonl()` соответственно.
|
||||
|
||||
### Продвинутая загрузка сайтов с поддержкой Session
|
||||
@@ -209,7 +233,8 @@ MySpider().start()
|
||||
- **Обход анти-ботов**: Расширенные возможности скрытности с `StealthyFetcher` и подмену fingerprint'ов. Может легко обойти все типы Cloudflare Turnstile/Interstitial с помощью автоматизации.
|
||||
- **Управление сессиями**: Поддержка постоянных сессий с классами `FetcherSession`, `StealthySession` и `DynamicSession` для управления cookie и состоянием между запросами.
|
||||
- **Ротация Proxy**: Встроенный `ProxyRotator` с циклической или пользовательскими стратегиями для всех типов сессий, а также переопределение Proxy для каждого запроса.
|
||||
- **Блокировка доменов**: Блокируйте запросы к определённым доменам (и их поддоменам) в браузерных Fetcher'ах.
|
||||
- **Блокировка доменов и рекламы**: Блокируйте запросы к определённым доменам (и их поддоменам) или включите встроенную блокировку рекламы (~3 500 известных рекламных/трекерных доменов) в браузерных Fetcher'ах.
|
||||
- **Защита от утечки DNS**: Опциональная поддержка DNS-over-HTTPS для маршрутизации DNS-запросов через Cloudflare DoH, предотвращая утечку DNS при использовании прокси.
|
||||
- **Поддержка async**: Полная async-поддержка во всех Fetcher'ах и выделенных async-классах сессий.
|
||||
|
||||
### Адаптивный скрапинг и интеграция с ИИ
|
||||
|
||||
@@ -33,6 +33,7 @@ The Scrapling MCP Server provides nine powerful tools for web scraping:
|
||||
- **Browser Impersonation**: Mimic real browsers with TLS fingerprinting, real browser headers matching that version, and more
|
||||
- **Parallel Processing**: Scrape multiple URLs concurrently for efficiency
|
||||
- **Session Persistence**: Reuse browser sessions across multiple requests for better performance
|
||||
- **Ad Blocking**: All browser-based tools automatically block requests to ~3,500 known ad and tracker domains, saving tokens and speeding up page loads
|
||||
- **Prompt Injection Protection**: Automatic sanitization of hidden content (CSS-hidden elements, aria-hidden, zero-width characters, HTML comments, template tags) that could be used for prompt injection attacks
|
||||
|
||||
#### But why use Scrapling MCP Server instead of other available tools?
|
||||
|
||||
@@ -24,7 +24,7 @@ The extract command is a set of simple terminal tools that:
|
||||
|
||||
!!! tip "AI-Targeted Mode"
|
||||
|
||||
All extract commands support an `--ai-targeted` flag. When enabled, it extracts only the main body content, strips noise tags (script, style, noscript, svg), removes hidden elements that could be used for prompt injection (CSS-hidden, aria-hidden, template tags), strips zero-width unicode characters, and removes HTML comments. This is ideal when the output is destined for an AI model.
|
||||
All extract commands support an `--ai-targeted` flag. When enabled, it extracts only the main body content, strips noise tags (script, style, noscript, svg), removes hidden elements that could be used for prompt injection (CSS-hidden, aria-hidden, template tags), strips zero-width unicode characters, and removes HTML comments. For browser commands (`fetch`/`stealthy-fetch`), it also automatically enables ad blocking. This is ideal when the output is destined for an AI model.
|
||||
|
||||
## Quick Start
|
||||
|
||||
@@ -291,6 +291,8 @@ We will go through each command in detail below.
|
||||
--real-chrome/--no-real-chrome If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch an instance of your browser and use it. (default: False)
|
||||
--proxy TEXT Proxy URL in format "http://username:password@host:port"
|
||||
-H, --extra-headers TEXT Extra headers in format "Key: Value" (can be used multiple times)
|
||||
--dns-over-https / --no-dns-over-https Route DNS through Cloudflare's DoH to prevent DNS leaks when using proxies (default: False)
|
||||
--block-ads / --no-block-ads Block requests to known ad and tracker domains (default: False)
|
||||
--ai-targeted Extract only main content and sanitize hidden elements for AI consumption (default: False)
|
||||
--help Show this message and exit.
|
||||
```
|
||||
@@ -337,6 +339,8 @@ We will go through each command in detail below.
|
||||
--hide-canvas / --show-canvas Add noise to canvas operations (default: False)
|
||||
--proxy TEXT Proxy URL in format "http://username:password@host:port"
|
||||
-H, --extra-headers TEXT Extra headers in format "Key: Value" (can be used multiple times)
|
||||
--dns-over-https / --no-dns-over-https Route DNS through Cloudflare's DoH to prevent DNS leaks when using proxies (default: False)
|
||||
--block-ads / --no-block-ads Block requests to known ad and tracker domains (default: False)
|
||||
--ai-targeted Extract only main content and sanitize hidden elements for AI consumption (default: False)
|
||||
--help Show this message and exit.
|
||||
```
|
||||
|
||||
@@ -72,7 +72,8 @@ Scrapling provides many options with this fetcher and its session classes. To ma
|
||||
| load_dom | Enabled by default, wait for all JavaScript on page(s) to fully load and execute (wait for the `domcontentloaded` state). | ✔️ |
|
||||
| timeout | The timeout (milliseconds) used in all operations and waits through the page. The default is 30,000 ms (30 seconds). | ✔️ |
|
||||
| wait | The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the `Response` object. | ✔️ |
|
||||
| page_action | Added for automation. Pass a function that takes the `page` object and does the necessary automation. | ✔️ |
|
||||
| page_action | Added for automation. Pass a function that takes the `page` object, runs after navigation, and does the necessary automation. | ✔️ |
|
||||
| page_setup | A function that takes the `page` object, runs before navigation. Use it to register event listeners or routes that must be set up before the page loads. | ✔️ |
|
||||
| wait_selector | Wait for a specific css selector to be in a specific state. | ✔️ |
|
||||
| init_script | An absolute path to a JavaScript file to be executed on page creation for all pages in this session. | ✔️ |
|
||||
| wait_selector_state | Scrapling will wait for the given state to be fulfilled for the selector given with `wait_selector`. _Default state is `attached`._ | ✔️ |
|
||||
@@ -88,13 +89,15 @@ Scrapling provides many options with this fetcher and its session classes. To ma
|
||||
| additional_args | Additional arguments to be passed to Playwright's context as additional settings, and they take higher priority than Scrapling's settings. | ✔️ |
|
||||
| selector_config | A dictionary of custom parsing arguments to be used when creating the final `Selector`/`Response` class. | ✔️ |
|
||||
| blocked_domains | A set of domain names to block requests to. Subdomains are also matched (e.g., `"example.com"` blocks `"sub.example.com"` too). | ✔️ |
|
||||
| block_ads | Block requests to ~3,500 known ad/tracking domains. Can be combined with `blocked_domains`. | ✔️ |
|
||||
| dns_over_https | Route DNS queries through Cloudflare's DNS-over-HTTPS to prevent DNS leaks when using proxies. | ✔️ |
|
||||
| proxy_rotator | A `ProxyRotator` instance for automatic proxy rotation. Cannot be combined with `proxy`. | ✔️ |
|
||||
| retries | Number of retry attempts for failed requests. Defaults to 3. | ✔️ |
|
||||
| retry_delay | Seconds to wait between retry attempts. Defaults to 1. | ✔️ |
|
||||
| capture_xhr | Pass a regex URL pattern string to capture XHR/fetch requests matching it during page load. Captured responses are available via `response.captured_xhr`. Defaults to `None` (disabled). | ✔️ |
|
||||
| executable_path | Absolute path to a custom browser executable to use instead of the bundled Chromium. Useful for non-standard installations or custom browser builds. | ✔️ |
|
||||
|
||||
In session classes, all these arguments can be set globally for the session. Still, you can configure each request individually by passing some of the arguments here that can be configured on the browser tab level like: `google_search`, `timeout`, `wait`, `page_action`, `extra_headers`, `disable_resources`, `wait_selector`, `wait_selector_state`, `network_idle`, `load_dom`, `blocked_domains`, `proxy`, and `selector_config`.
|
||||
In session classes, all these arguments can be set globally for the session. Still, you can configure each request individually by passing some of the arguments here that can be configured on the browser tab level like: `google_search`, `timeout`, `wait`, `page_action`, `page_setup`, `extra_headers`, `disable_resources`, `wait_selector`, `wait_selector_state`, `network_idle`, `load_dom`, `blocked_domains`, `proxy`, and `selector_config`.
|
||||
|
||||
!!! note "Notes:"
|
||||
|
||||
@@ -170,6 +173,29 @@ with open(file='main_cover.png', mode='wb') as f:
|
||||
|
||||
The `body` attribute of the `Response` object always returns `bytes`.
|
||||
|
||||
### Pre-Navigation Setup
|
||||
If you need to set up event listeners, routes, or scripts that must be registered before the page navigates, use `page_setup`. This function receives the `page` object and runs before `page.goto()` is called.
|
||||
|
||||
```python
|
||||
from playwright.sync_api import Page
|
||||
|
||||
def capture_websockets(page: Page):
|
||||
page.on("websocket", lambda ws: print(f"WebSocket opened: {ws.url}"))
|
||||
|
||||
page = DynamicFetcher.fetch('https://example.com', page_setup=capture_websockets)
|
||||
```
|
||||
Async version:
|
||||
```python
|
||||
from playwright.async_api import Page
|
||||
|
||||
async def capture_websockets(page: Page):
|
||||
page.on("websocket", lambda ws: print(f"WebSocket opened: {ws.url}"))
|
||||
|
||||
page = await DynamicFetcher.async_fetch('https://example.com', page_setup=capture_websockets)
|
||||
```
|
||||
|
||||
You can combine it with `page_action` -- `page_setup` runs before navigation, `page_action` runs after.
|
||||
|
||||
### Browser Automation
|
||||
This is where your knowledge about [Playwright's Page API](https://playwright.dev/python/docs/api/class-page) comes into play. The function you pass here takes the page object from Playwright's API, performs the desired action, and then the fetcher continues.
|
||||
|
||||
|
||||
@@ -21,7 +21,7 @@ All methods for making requests here share some arguments, so let's discuss them
|
||||
|
||||
- **url**: The targeted URL
|
||||
- **stealthy_headers**: If enabled (default), it creates and adds real browser headers. It also sets a Google referer header.
|
||||
- **follow_redirects**: As the name implies, tell the fetcher to follow redirections. **Enabled by default**
|
||||
- **follow_redirects**: Controls redirect behavior. **Defaults to `"safe"`**, which follows redirects but rejects those targeting internal/private IPs (SSRF protection). Pass `True` to follow all redirects without restriction, or `False` to disable redirects entirely.
|
||||
- **timeout**: The number of seconds to wait for each request to be finished. **Defaults to 30 seconds**.
|
||||
- **retries**: The number of retries that the fetcher will do for failed requests. **Defaults to three retries**.
|
||||
- **retry_delay**: Number of seconds to wait between retry attempts. **Defaults to 1 second**.
|
||||
@@ -57,7 +57,7 @@ Examples are the best way to explain this:
|
||||
>>> from scrapling.fetchers import Fetcher
|
||||
>>> # Basic GET
|
||||
>>> page = Fetcher.get('https://example.com')
|
||||
>>> page = Fetcher.get('https://scrapling.requestcatcher.com/get', stealthy_headers=True, follow_redirects=True)
|
||||
>>> page = Fetcher.get('https://scrapling.requestcatcher.com/get', stealthy_headers=True)
|
||||
>>> page = Fetcher.get('https://scrapling.requestcatcher.com/get', proxy='http://username:password@localhost:8030')
|
||||
>>> # With parameters
|
||||
>>> page = Fetcher.get('https://example.com/search', params={'q': 'query'})
|
||||
@@ -76,7 +76,7 @@ And for asynchronous requests, it's a small adjustment
|
||||
>>> from scrapling.fetchers import AsyncFetcher
|
||||
>>> # Basic GET
|
||||
>>> page = await AsyncFetcher.get('https://example.com')
|
||||
>>> page = await AsyncFetcher.get('https://scrapling.requestcatcher.com/get', stealthy_headers=True, follow_redirects=True)
|
||||
>>> page = await AsyncFetcher.get('https://scrapling.requestcatcher.com/get', stealthy_headers=True)
|
||||
>>> page = await AsyncFetcher.get('https://scrapling.requestcatcher.com/get', proxy='http://username:password@localhost:8030')
|
||||
>>> # With parameters
|
||||
>>> page = await AsyncFetcher.get('https://example.com/search', params={'q': 'query'})
|
||||
@@ -112,7 +112,7 @@ Needless to say, the `page` object in all cases is [Response](choosing.md#respon
|
||||
>>> from scrapling.fetchers import Fetcher
|
||||
>>> # Basic POST
|
||||
>>> page = Fetcher.post('https://scrapling.requestcatcher.com/post', data={'key': 'value'}, params={'q': 'query'})
|
||||
>>> page = Fetcher.post('https://scrapling.requestcatcher.com/post', data={'key': 'value'}, stealthy_headers=True, follow_redirects=True)
|
||||
>>> page = Fetcher.post('https://scrapling.requestcatcher.com/post', data={'key': 'value'}, stealthy_headers=True)
|
||||
>>> page = Fetcher.post('https://scrapling.requestcatcher.com/post', data={'key': 'value'}, proxy='http://username:password@localhost:8030', impersonate="chrome")
|
||||
>>> # Another example of form-encoded data
|
||||
>>> page = Fetcher.post('https://example.com/submit', data={'username': 'user', 'password': 'pass'}, http3=True)
|
||||
@@ -124,7 +124,7 @@ And for asynchronous requests, it's a small adjustment
|
||||
>>> from scrapling.fetchers import AsyncFetcher
|
||||
>>> # Basic POST
|
||||
>>> page = await AsyncFetcher.post('https://scrapling.requestcatcher.com/post', data={'key': 'value'})
|
||||
>>> page = await AsyncFetcher.post('https://scrapling.requestcatcher.com/post', data={'key': 'value'}, stealthy_headers=True, follow_redirects=True)
|
||||
>>> page = await AsyncFetcher.post('https://scrapling.requestcatcher.com/post', data={'key': 'value'}, stealthy_headers=True)
|
||||
>>> page = await AsyncFetcher.post('https://scrapling.requestcatcher.com/post', data={'key': 'value'}, proxy='http://username:password@localhost:8030', impersonate="chrome")
|
||||
>>> # Another example of form-encoded data
|
||||
>>> page = await AsyncFetcher.post('https://example.com/submit', data={'username': 'user', 'password': 'pass'}, http3=True)
|
||||
@@ -136,7 +136,7 @@ And for asynchronous requests, it's a small adjustment
|
||||
>>> from scrapling.fetchers import Fetcher
|
||||
>>> # Basic PUT
|
||||
>>> page = Fetcher.put('https://example.com/update', data={'status': 'updated'})
|
||||
>>> page = Fetcher.put('https://example.com/update', data={'status': 'updated'}, stealthy_headers=True, follow_redirects=True, impersonate="chrome")
|
||||
>>> page = Fetcher.put('https://example.com/update', data={'status': 'updated'}, stealthy_headers=True, impersonate="chrome")
|
||||
>>> page = Fetcher.put('https://example.com/update', data={'status': 'updated'}, proxy='http://username:password@localhost:8030')
|
||||
>>> # Another example of form-encoded data
|
||||
>>> page = Fetcher.put("https://scrapling.requestcatcher.com/put", data={'key': ['value1', 'value2']})
|
||||
@@ -146,7 +146,7 @@ And for asynchronous requests, it's a small adjustment
|
||||
>>> from scrapling.fetchers import AsyncFetcher
|
||||
>>> # Basic PUT
|
||||
>>> page = await AsyncFetcher.put('https://example.com/update', data={'status': 'updated'})
|
||||
>>> page = await AsyncFetcher.put('https://example.com/update', data={'status': 'updated'}, stealthy_headers=True, follow_redirects=True, impersonate="chrome")
|
||||
>>> page = await AsyncFetcher.put('https://example.com/update', data={'status': 'updated'}, stealthy_headers=True, impersonate="chrome")
|
||||
>>> page = await AsyncFetcher.put('https://example.com/update', data={'status': 'updated'}, proxy='http://username:password@localhost:8030')
|
||||
>>> # Another example of form-encoded data
|
||||
>>> page = await AsyncFetcher.put("https://scrapling.requestcatcher.com/put", data={'key': ['value1', 'value2']})
|
||||
@@ -156,14 +156,14 @@ And for asynchronous requests, it's a small adjustment
|
||||
```python
|
||||
>>> from scrapling.fetchers import Fetcher
|
||||
>>> page = Fetcher.delete('https://example.com/resource/123')
|
||||
>>> page = Fetcher.delete('https://example.com/resource/123', stealthy_headers=True, follow_redirects=True, impersonate="chrome")
|
||||
>>> page = Fetcher.delete('https://example.com/resource/123', stealthy_headers=True, impersonate="chrome")
|
||||
>>> page = Fetcher.delete('https://example.com/resource/123', proxy='http://username:password@localhost:8030')
|
||||
```
|
||||
And for asynchronous requests, it's a small adjustment
|
||||
```python
|
||||
>>> from scrapling.fetchers import AsyncFetcher
|
||||
>>> page = await AsyncFetcher.delete('https://example.com/resource/123')
|
||||
>>> page = await AsyncFetcher.delete('https://example.com/resource/123', stealthy_headers=True, follow_redirects=True, impersonate="chrome")
|
||||
>>> page = await AsyncFetcher.delete('https://example.com/resource/123', stealthy_headers=True, impersonate="chrome")
|
||||
>>> page = await AsyncFetcher.delete('https://example.com/resource/123', proxy='http://username:password@localhost:8030')
|
||||
```
|
||||
|
||||
|
||||
@@ -49,7 +49,8 @@ Scrapling provides many options with this fetcher and its session classes. Befor
|
||||
| load_dom | Enabled by default, wait for all JavaScript on page(s) to fully load and execute (wait for the `domcontentloaded` state). | ✔️ |
|
||||
| timeout | The timeout (milliseconds) used in all operations and waits through the page. The default is 30,000 ms (30 seconds). | ✔️ |
|
||||
| wait | The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the `Response` object. | ✔️ |
|
||||
| page_action | Added for automation. Pass a function that takes the `page` object and does the necessary automation. | ✔️ |
|
||||
| page_action | Added for automation. Pass a function that takes the `page` object, runs after navigation, and does the necessary automation. | ✔️ |
|
||||
| page_setup | A function that takes the `page` object, runs before navigation. Use it to register event listeners or routes that must be set up before the page loads. | ✔️ |
|
||||
| wait_selector | Wait for a specific css selector to be in a specific state. | ✔️ |
|
||||
| init_script | An absolute path to a JavaScript file to be executed on page creation for all pages in this session. | ✔️ |
|
||||
| wait_selector_state | Scrapling will wait for the given state to be fulfilled for the selector given with `wait_selector`. _Default state is `attached`._ | ✔️ |
|
||||
@@ -69,13 +70,15 @@ Scrapling provides many options with this fetcher and its session classes. Befor
|
||||
| additional_args | Additional arguments to be passed to Playwright's context as additional settings, and they take higher priority than Scrapling's settings. | ✔️ |
|
||||
| selector_config | A dictionary of custom parsing arguments to be used when creating the final `Selector`/`Response` class. | ✔️ |
|
||||
| blocked_domains | A set of domain names to block requests to. Subdomains are also matched (e.g., `"example.com"` blocks `"sub.example.com"` too). | ✔️ |
|
||||
| block_ads | Block requests to ~3,500 known ad/tracking domains. Can be combined with `blocked_domains`. | ✔️ |
|
||||
| dns_over_https | Route DNS queries through Cloudflare's DNS-over-HTTPS to prevent DNS leaks when using proxies. | ✔️ |
|
||||
| proxy_rotator | A `ProxyRotator` instance for automatic proxy rotation. Cannot be combined with `proxy`. | ✔️ |
|
||||
| retries | Number of retry attempts for failed requests. Defaults to 3. | ✔️ |
|
||||
| retry_delay | Seconds to wait between retry attempts. Defaults to 1. | ✔️ |
|
||||
| capture_xhr | Pass a regex URL pattern string to capture XHR/fetch requests matching it during page load. Captured responses are available via `response.captured_xhr`. Defaults to `None` (disabled). | ✔️ |
|
||||
| executable_path | Absolute path to a custom browser executable to use instead of the bundled Chromium. Useful for non-standard installations or custom browser builds. | ✔️ |
|
||||
|
||||
In session classes, all these arguments can be set globally for the session. Still, you can configure each request individually by passing some of the arguments here that can be configured on the browser tab level like: `google_search`, `timeout`, `wait`, `page_action`, `extra_headers`, `disable_resources`, `wait_selector`, `wait_selector_state`, `network_idle`, `load_dom`, `solve_cloudflare`, `blocked_domains`, `proxy`, and `selector_config`.
|
||||
In session classes, all these arguments can be set globally for the session. Still, you can configure each request individually by passing some of the arguments here that can be configured on the browser tab level like: `google_search`, `timeout`, `wait`, `page_action`, `page_setup`, `extra_headers`, `disable_resources`, `wait_selector`, `wait_selector_state`, `network_idle`, `load_dom`, `solve_cloudflare`, `blocked_domains`, `proxy`, and `selector_config`.
|
||||
|
||||
!!! note "Notes:"
|
||||
|
||||
|
||||
+10
-1
@@ -80,6 +80,12 @@ MySpider().start()
|
||||
<a href="https://proxy-seller.com/?partner=CU9CAA5TBYFFT2" target="_blank" title="Proxy-Seller provides reliable proxy infrastructure for Web Scraping">
|
||||
<img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/ProxySeller.png" class="ad">
|
||||
</a>
|
||||
<a href="http://mangoproxy.com/?utm_source=D4Vinci&utm_medium=GitHub&utm_campaign=D4Vinci" target="_blank" title="Proxies You Can Rely On: Residential, Server, and Mobile">
|
||||
<img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/MangoProxy.png" class="ad">
|
||||
</a>
|
||||
<a href="https://www.swiftproxy.net/?ref=D4Vinci" target="_blank" title="Scalable Solutions for Web Data Access">
|
||||
<img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/SwiftProxy.png" class="ad">
|
||||
</a>
|
||||
<br />
|
||||
<br />
|
||||
</div>
|
||||
@@ -96,6 +102,8 @@ MySpider().start()
|
||||
- 💾 **Pause & Resume**: Checkpoint-based crawl persistence. Press Ctrl+C for a graceful shutdown; restart to resume from where you left off.
|
||||
- 📡 **Streaming Mode**: Stream scraped items as they arrive via `async for item in spider.stream()` with real-time stats - ideal for UI, pipelines, and long-running crawls.
|
||||
- 🛡️ **Blocked Request Detection**: Automatic detection and retry of blocked requests with customizable logic.
|
||||
- 🤖 **Robots.txt Compliance**: Optional `robots_txt_obey` flag that respects `Disallow`, `Crawl-delay`, and `Request-rate` directives with per-domain caching.
|
||||
- 🧪 **Development Mode**: Cache responses to disk on the first run and replay them on subsequent runs - iterate on your `parse()` logic without re-hitting the target servers.
|
||||
- 📦 **Built-in Export**: Export results through hooks and your own pipeline or the built-in JSON/JSONL with `result.items.to_json()` / `result.items.to_jsonl()` respectively.
|
||||
|
||||
### Advanced Websites Fetching with Session Support
|
||||
@@ -104,7 +112,8 @@ MySpider().start()
|
||||
- **Anti-bot Bypass**: Advanced stealth capabilities with `StealthyFetcher` and fingerprint spoofing. Can easily bypass all types of Cloudflare's Turnstile/Interstitial with automation.
|
||||
- **Session Management**: Persistent session support with `FetcherSession`, `StealthySession`, and `DynamicSession` classes for cookie and state management across requests.
|
||||
- **Proxy Rotation**: Built-in `ProxyRotator` with cyclic or custom rotation strategies across all session types, plus per-request proxy overrides.
|
||||
- **Domain Blocking**: Block requests to specific domains (and their subdomains) in browser-based fetchers.
|
||||
- **Domain & Ad Blocking**: Block requests to specific domains (and their subdomains) or enable built-in ad blocking (~3,500 known ad/tracker domains) in browser-based fetchers.
|
||||
- **DNS Leak Prevention**: Optional DNS-over-HTTPS support to route DNS queries through Cloudflare's DoH, preventing DNS leaks when using proxies.
|
||||
- **Async Support**: Complete async support across all fetchers and dedicated async session classes.
|
||||
|
||||
### Adaptive Scraping & AI Integration
|
||||
|
||||
+2
-2
@@ -264,7 +264,7 @@ page = Fetcher.get('https://scrapling.requestcatcher.com/get', impersonate="chro
|
||||
With that out of the way, here's how to do all HTTP methods:
|
||||
```python
|
||||
>>> from scrapling.fetchers import Fetcher
|
||||
>>> page = Fetcher.get('https://scrapling.requestcatcher.com/get', stealthy_headers=True, follow_redirects=True)
|
||||
>>> page = Fetcher.get('https://scrapling.requestcatcher.com/get', stealthy_headers=True)
|
||||
>>> page = Fetcher.post('https://scrapling.requestcatcher.com/post', data={'key': 'value'}, proxy='http://username:password@localhost:8030')
|
||||
>>> page = Fetcher.put('https://scrapling.requestcatcher.com/put', data={'key': 'value'})
|
||||
>>> page = Fetcher.delete('https://scrapling.requestcatcher.com/delete')
|
||||
@@ -272,7 +272,7 @@ With that out of the way, here's how to do all HTTP methods:
|
||||
For Async requests, you will replace the import like below:
|
||||
```python
|
||||
>>> from scrapling.fetchers import AsyncFetcher
|
||||
>>> page = await AsyncFetcher.get('https://scrapling.requestcatcher.com/get', stealthy_headers=True, follow_redirects=True)
|
||||
>>> page = await AsyncFetcher.get('https://scrapling.requestcatcher.com/get', stealthy_headers=True)
|
||||
>>> page = await AsyncFetcher.post('https://scrapling.requestcatcher.com/post', data={'key': 'value'}, proxy='http://username:password@localhost:8030')
|
||||
>>> page = await AsyncFetcher.put('https://scrapling.requestcatcher.com/put', data={'key': 'value'})
|
||||
>>> page = await AsyncFetcher.delete('https://scrapling.requestcatcher.com/delete')
|
||||
|
||||
@@ -17,6 +17,7 @@ The spider system uses three class attributes to control how aggressively it cra
|
||||
| `concurrent_requests` | `4` | Maximum number of requests being processed at the same time |
|
||||
| `concurrent_requests_per_domain` | `0` | Maximum concurrent requests per domain (0 = no per-domain limit) |
|
||||
| `download_delay` | `0.0` | Seconds to wait before each request |
|
||||
| `robots_txt_obey` | `False` | Respect robots.txt rules (Disallow, Crawl-delay, Request-rate) |
|
||||
|
||||
```python
|
||||
class PoliteSpider(Spider):
|
||||
@@ -96,6 +97,51 @@ async def on_start(self, resuming: bool = False):
|
||||
self.logger.info("Starting fresh crawl")
|
||||
```
|
||||
|
||||
## Development Mode
|
||||
|
||||
When you're iterating on a spider's `parse()` logic, re-hitting the target servers on every run is slow and noisy. Development mode caches every response to disk on the first run and replays them from disk on subsequent runs, so you can tweak your selectors and re-run the spider as many times as you want without making a single network request.
|
||||
|
||||
Enable it by setting `development_mode = True` on your spider:
|
||||
|
||||
```python
|
||||
class MySpider(Spider):
|
||||
name = "my_spider"
|
||||
start_urls = ["https://example.com"]
|
||||
development_mode = True
|
||||
|
||||
async def parse(self, response: Response):
|
||||
yield {"title": response.css("title::text").get("")}
|
||||
```
|
||||
|
||||
The first run fetches normally and stores each response on disk. Every subsequent run serves the same requests from the cache, skipping the network entirely.
|
||||
|
||||
### Cache Location
|
||||
|
||||
By default, responses are cached in `.scrapling_cache/{spider.name}/` relative to the current working directory (where you ran the spider from, **not** where the spider script lives). You can override the location with `development_cache_dir`:
|
||||
|
||||
```python
|
||||
class MySpider(Spider):
|
||||
name = "my_spider"
|
||||
start_urls = ["https://example.com"]
|
||||
development_mode = True
|
||||
development_cache_dir = "/tmp/my_spider_cache"
|
||||
```
|
||||
|
||||
### How It Works
|
||||
|
||||
1. **Cache key**: Each response is keyed by the request's fingerprint, so any change to fingerprint-affecting attributes (`fp_include_kwargs`, `fp_include_headers`, `fp_keep_fragments`) will produce a fresh fetch.
|
||||
2. **Storage format**: One JSON file per response, named `{fingerprint_hex}.json`. The body is base64-encoded so binary content is preserved exactly. Writes are atomic (temp file + rename).
|
||||
3. **Replay**: On a cache hit, the engine skips the network entirely, including `download_delay`, rate limiting, and the `is_blocked()` retry path. The cached response goes straight to your callback.
|
||||
4. **Stats**: Cached requests still count toward `requests_count`, `response_bytes`, and the per-status counters, so your stat output looks the same as a normal crawl. Two extra counters, `cache_hits` and `cache_misses`, let you see how the cache performed.
|
||||
|
||||
### Clearing the Cache
|
||||
|
||||
There's no automatic expiration. To force a fresh crawl, delete the cache directory or call the manager's `clear()` method directly.
|
||||
|
||||
!!! warning
|
||||
|
||||
Development mode is meant for development, not production. Cached responses never expire, and replay bypasses rate limiting and blocked-request retries. Don't ship a spider with `development_mode = True`.
|
||||
|
||||
## Streaming
|
||||
|
||||
For long-running spiders or applications that need real-time access to scraped items, use the `stream()` method instead of `start()`:
|
||||
@@ -234,6 +280,9 @@ print(f"Requests: {stats.requests_count}")
|
||||
print(f"Failed: {stats.failed_requests_count}")
|
||||
print(f"Blocked: {stats.blocked_requests_count}")
|
||||
print(f"Offsite filtered: {stats.offsite_requests_count}")
|
||||
print(f"Robots.txt disallowed: {stats.robots_disallowed_count}")
|
||||
print(f"Cache hits: {stats.cache_hits}")
|
||||
print(f"Cache misses: {stats.cache_misses}")
|
||||
print(f"Items scraped: {stats.items_scraped}")
|
||||
print(f"Items dropped: {stats.items_dropped}")
|
||||
print(f"Response bytes: {stats.response_bytes}")
|
||||
|
||||
@@ -19,7 +19,7 @@ Here's what happens step by step when you run a spider without many details:
|
||||
|
||||
1. The **Spider** produces the first batch of `Request` objects. By default, it creates one request for each URL in `start_urls`, but you can override `start_requests()` for custom logic.
|
||||
2. The **Scheduler** receives requests and places them in a priority queue, and creates fingerprints for them. Higher-priority requests are dequeued first.
|
||||
3. The **Crawler Engine** asks the **Scheduler** to dequeue the next request, respecting concurrency limits (global and per-domain) and download delays. Once the **Crawler Engine** receives the request, it passes it to the **Session Manager**, which routes it to the correct session based on the request's `sid` (session ID).
|
||||
3. The **Crawler Engine** asks the **Scheduler** to dequeue the next request, respecting concurrency limits (global and per-domain) and download delays. If `robots_txt_obey` is enabled, the engine checks the domain's robots.txt rules before proceeding -- disallowed requests are dropped silently. Once the **Crawler Engine** receives the request, it passes it to the **Session Manager**, which routes it to the correct session based on the request's `sid` (session ID).
|
||||
4. The **session** fetches the page and returns a [Response](../fetching/choosing.md#response-object) object to the **Crawler Engine**. The engine records statistics and checks for blocked responses. If the response is blocked, the engine retries the request up to `max_blocked_retries` times. Of course, the blocking detection and the retry logic for blocked requests can be customized.
|
||||
5. The **Crawler Engine** passes the [Response](../fetching/choosing.md#response-object) to the request's callback. The callback either yields a dictionary, which gets treated as a scraped item, or a follow-up request, which gets sent to the scheduler for queuing.
|
||||
6. The cycle repeats from step 2 until the scheduler is empty and no tasks are active, or the spider is paused.
|
||||
@@ -69,6 +69,10 @@ When a request comes in, the Session Manager routes it to the correct session ba
|
||||
|
||||
An optional system that, if enabled, saves the crawler's state (pending requests + seen URL fingerprints) to a pickle file on disk. Writes are atomic (temp file + rename) to prevent corruption. Checkpoints are saved periodically at a configurable interval and on graceful shutdown. Upon successful completion (not paused), checkpoint files are automatically cleaned up.
|
||||
|
||||
### Response Cache
|
||||
|
||||
An optional cache that, when development mode is enabled, stores every fetched response on disk and replays it on subsequent runs. Each response is keyed by request fingerprint and serialized as JSON (with the body base64-encoded so binary content survives). It's meant for iterating on `parse()` logic without re-hitting the target servers, not for production use.
|
||||
|
||||
### Output
|
||||
|
||||
Scraped items are collected in an `ItemList` (a list subclass with `to_json()` and `to_jsonl()` export methods). Crawl statistics are tracked in a `CrawlStats` dataclass which contains a lot of useful info.
|
||||
@@ -91,6 +95,7 @@ If you're coming from Scrapy, here's how Scrapling's spider system maps:
|
||||
| Blocked detection | Through custom middlewares | Built-in `is_blocked()` + `retry_blocked_request()` hooks |
|
||||
| Concurrency | `CONCURRENT_REQUESTS` setting | `concurrent_requests` class attribute |
|
||||
| Domain filtering | `allowed_domains` | `allowed_domains` |
|
||||
| Robots.txt | `ROBOTSTXT_OBEY` setting | `robots_txt_obey` class attribute |
|
||||
| Pause/Resume | `JOBDIR` setting | `crawldir` constructor argument |
|
||||
| Export | Feed exports | `result.items.to_json()` / `to_jsonl()` or custom through hooks |
|
||||
| Running | `scrapy crawl spider_name` | `MySpider().start()` |
|
||||
|
||||
@@ -149,6 +149,31 @@ Subdomains are matched automatically, so setting `allowed_domains = {"example.co
|
||||
|
||||
When a request is filtered out, it's counted in `stats.offsite_requests_count` so you can see how many were dropped.
|
||||
|
||||
## Robots.txt Compliance
|
||||
|
||||
Set `robots_txt_obey = True` to make the spider respect robots.txt rules before crawling any domain:
|
||||
|
||||
```python
|
||||
class PoliteSpider(Spider):
|
||||
name = "polite"
|
||||
start_urls = ["https://example.com"]
|
||||
robots_txt_obey = True
|
||||
|
||||
async def parse(self, response: Response):
|
||||
for link in response.css("a::attr(href)").getall():
|
||||
yield response.follow(link, callback=self.parse)
|
||||
```
|
||||
|
||||
When enabled, the spider will:
|
||||
|
||||
1. **Pre-fetch robots.txt** for all domains in `start_urls` before the crawl begins (concurrently).
|
||||
2. **Check every request** against the domain's robots.txt `Disallow` rules. Disallowed requests are silently dropped and counted in `stats.robots_disallowed_count`.
|
||||
3. **Respect `Crawl-delay` and `Request-rate` directives** by taking the maximum of the directive and your configured `download_delay`. This means robots.txt delays never reduce your configured delay, only increase it when needed.
|
||||
|
||||
Robots.txt files are fetched using the spider's default session and cached per domain for the entire crawl. Domains discovered mid-crawl (not in `start_urls`) have their robots.txt fetched on the first request to that domain.
|
||||
|
||||
**Note:** `robots_txt_obey` is turned off by default to avoid surprising behavior. If you enable it, it does not affect your concurrency settings (`concurrent_requests`, `concurrent_requests_per_domain`) -- only the delay between requests is adjusted.
|
||||
|
||||
## What's Next
|
||||
|
||||
Now that you have the basics, you can explore:
|
||||
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 15 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 7.6 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 70 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 27 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 4.5 KiB |
+10
-9
@@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta"
|
||||
[project]
|
||||
name = "scrapling"
|
||||
# Static version instead of a dynamic version so we can get better layer caching while building docker, check the docker file to understand
|
||||
version = "0.4.4"
|
||||
version = "0.4.7"
|
||||
description = "Scrapling is an undetectable, powerful, flexible, high-performance Python library that makes Web Scraping easy and effortless as it should be!"
|
||||
readme = {file = "README.md", content-type = "text/markdown"}
|
||||
license = {file = "LICENSE"}
|
||||
@@ -61,27 +61,28 @@ classifiers = [
|
||||
"Typing :: Typed",
|
||||
]
|
||||
dependencies = [
|
||||
"lxml>=6.0.2",
|
||||
"lxml>=6.0.3",
|
||||
"cssselect>=1.4.0",
|
||||
"orjson>=3.11.7",
|
||||
"orjson>=3.11.8",
|
||||
"tld>=0.13.2",
|
||||
"w3lib>=2.4.1",
|
||||
"typing_extensions",
|
||||
"typing_extensions"
|
||||
]
|
||||
|
||||
[project.optional-dependencies]
|
||||
fetchers = [
|
||||
"click>=8.3.0",
|
||||
"curl_cffi>=0.14.0",
|
||||
"curl_cffi>=0.15.0",
|
||||
"playwright==1.58.0",
|
||||
"patchright==1.58.2",
|
||||
"browserforge>=1.2.4",
|
||||
"apify-fingerprint-datapoints>=0.11.0",
|
||||
"msgspec>=0.20.0",
|
||||
"anyio>=4.12.1"
|
||||
"apify-fingerprint-datapoints>=0.12.0",
|
||||
"msgspec>=0.21.1",
|
||||
"anyio>=4.13.0",
|
||||
"protego>=0.6.0",
|
||||
]
|
||||
ai = [
|
||||
"mcp>=1.26.0",
|
||||
"mcp>=1.27.0",
|
||||
"markdownify>=1.2.0",
|
||||
"scrapling[fetchers]",
|
||||
]
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
__author__ = "Karim Shoair (karim.shoair@pm.me)"
|
||||
__version__ = "0.4.4"
|
||||
__version__ = "0.4.7"
|
||||
__copyright__ = "Copyright (c) 2024 Karim Shoair"
|
||||
|
||||
from typing import Any, TYPE_CHECKING
|
||||
|
||||
@@ -53,6 +53,8 @@ def __Request_and_Save(
|
||||
if not output_path.is_absolute():
|
||||
output_path = Path.cwd() / output_file
|
||||
|
||||
if ai_targeted:
|
||||
kwargs.setdefault("block_ads", True)
|
||||
response = fetcher_func(url, **kwargs)
|
||||
Convertor.write_content_to_file(response, str(output_path), css_selector, main_content_only=ai_targeted)
|
||||
log.info(f"Content successfully saved to '{output_path}'")
|
||||
@@ -309,6 +311,16 @@ def _common_browser_options(f):
|
||||
default=True,
|
||||
help="Run browser in headless mode (default: True)",
|
||||
),
|
||||
option(
|
||||
"--dns-over-https/--no-dns-over-https",
|
||||
default=False,
|
||||
help="Route DNS through Cloudflare's DoH to prevent DNS leaks when using proxies (default: False)",
|
||||
),
|
||||
option(
|
||||
"--block-ads/--no-block-ads",
|
||||
default=False,
|
||||
help="Block requests to known ad and tracker domains (default: False)",
|
||||
),
|
||||
]
|
||||
for decorator in decorators:
|
||||
f = decorator(f)
|
||||
@@ -498,6 +510,8 @@ def __build_browser_kwargs(
|
||||
real_chrome,
|
||||
proxy,
|
||||
parsed_headers,
|
||||
dns_over_https,
|
||||
block_ads,
|
||||
) -> Dict[str, Any]:
|
||||
"""Build shared kwargs dict for browser-based commands."""
|
||||
kwargs: Dict[str, Any] = {
|
||||
@@ -507,6 +521,8 @@ def __build_browser_kwargs(
|
||||
"timeout": timeout,
|
||||
"locale": locale,
|
||||
"real_chrome": real_chrome,
|
||||
"dns_over_https": dns_over_https,
|
||||
"block_ads": block_ads,
|
||||
}
|
||||
if wait > 0:
|
||||
kwargs["wait"] = wait
|
||||
@@ -538,6 +554,8 @@ def fetch(
|
||||
proxy,
|
||||
extra_headers,
|
||||
ai_targeted,
|
||||
dns_over_https,
|
||||
block_ads,
|
||||
):
|
||||
"""Opens up a browser and fetch content using DynamicFetcher."""
|
||||
parsed_headers, _ = _ParseHeaders(extra_headers, False)
|
||||
@@ -552,6 +570,8 @@ def fetch(
|
||||
real_chrome,
|
||||
proxy,
|
||||
parsed_headers,
|
||||
dns_over_https,
|
||||
block_ads,
|
||||
)
|
||||
from scrapling.fetchers import DynamicFetcher
|
||||
|
||||
@@ -597,6 +617,8 @@ def stealthy_fetch(
|
||||
allow_webgl,
|
||||
hide_canvas,
|
||||
ai_targeted,
|
||||
dns_over_https,
|
||||
block_ads,
|
||||
):
|
||||
"""Opens up a browser with advanced stealth features and fetch content using StealthyFetcher."""
|
||||
parsed_headers, _ = _ParseHeaders(extra_headers, False)
|
||||
@@ -611,6 +633,8 @@ def stealthy_fetch(
|
||||
real_chrome,
|
||||
proxy,
|
||||
parsed_headers,
|
||||
dns_over_https,
|
||||
block_ads,
|
||||
)
|
||||
kwargs.update(
|
||||
{
|
||||
|
||||
@@ -2,12 +2,14 @@ from scrapling.core._types import (
|
||||
Any,
|
||||
Dict,
|
||||
List,
|
||||
Set,
|
||||
Tuple,
|
||||
Sequence,
|
||||
Callable,
|
||||
Optional,
|
||||
SetCookieParam,
|
||||
SelectorWaitStates,
|
||||
FollowRedirects,
|
||||
)
|
||||
|
||||
# Parameter definitions for shell function signatures (defined once at module level)
|
||||
@@ -26,7 +28,7 @@ _REQUESTS_PARAMS = {
|
||||
"headers": Any,
|
||||
"retries": Optional[int],
|
||||
"retry_delay": Optional[int],
|
||||
"follow_redirects": Optional[bool],
|
||||
"follow_redirects": Optional[FollowRedirects],
|
||||
"max_redirects": Optional[int],
|
||||
"verify": Optional[bool],
|
||||
"cert": Optional[str | Tuple[str, str]],
|
||||
@@ -45,6 +47,7 @@ _FETCH_PARAMS = {
|
||||
"wait": int | float,
|
||||
"timezone_id": str | None,
|
||||
"page_action": Optional[Callable],
|
||||
"page_setup": Optional[Callable],
|
||||
"proxy": Optional[str | Dict[str, str] | Tuple],
|
||||
"extra_headers": Optional[Dict[str, str]],
|
||||
"timeout": int | float,
|
||||
@@ -57,6 +60,13 @@ _FETCH_PARAMS = {
|
||||
"cdp_url": Optional[str],
|
||||
"useragent": Optional[str],
|
||||
"extra_flags": Optional[List[str]],
|
||||
"blocked_domains": Optional[Set[str]],
|
||||
"block_ads": bool,
|
||||
"retries": int,
|
||||
"retry_delay": int | float,
|
||||
"capture_xhr": str | None,
|
||||
"executable_path": Optional[str],
|
||||
"dns_over_https": bool,
|
||||
}
|
||||
|
||||
_STEALTHY_FETCH_PARAMS = {
|
||||
@@ -71,6 +81,7 @@ _STEALTHY_FETCH_PARAMS = {
|
||||
"wait": int | float,
|
||||
"timezone_id": str | None,
|
||||
"page_action": Optional[Callable],
|
||||
"page_setup": Optional[Callable],
|
||||
"proxy": Optional[str | Dict[str, str] | Tuple],
|
||||
"extra_headers": Optional[Dict[str, str]],
|
||||
"timeout": int | float,
|
||||
@@ -83,6 +94,13 @@ _STEALTHY_FETCH_PARAMS = {
|
||||
"cdp_url": Optional[str],
|
||||
"useragent": Optional[str],
|
||||
"extra_flags": Optional[List[str]],
|
||||
"blocked_domains": Optional[Set[str]],
|
||||
"block_ads": bool,
|
||||
"retries": int,
|
||||
"retry_delay": int | float,
|
||||
"capture_xhr": str | None,
|
||||
"executable_path": Optional[str],
|
||||
"dns_over_https": bool,
|
||||
"allow_webgl": bool,
|
||||
"hide_canvas": bool,
|
||||
"block_webrtc": bool,
|
||||
|
||||
@@ -40,6 +40,7 @@ SelectorWaitStates = Literal["attached", "detached", "hidden", "visible"]
|
||||
PageLoadStates = Literal["commit", "domcontentloaded", "load", "networkidle"]
|
||||
extraction_types = Literal["text", "html", "markdown"]
|
||||
StrOrBytes = Union[str, bytes]
|
||||
FollowRedirects = Union[bool, Literal["safe", "all", "obeycode", "firstonly"]]
|
||||
|
||||
|
||||
# Copied from `playwright._impl._api_structures.SetCookieParam`
|
||||
|
||||
+16
-5
@@ -27,6 +27,7 @@ from scrapling.core._types import (
|
||||
SetCookieParam,
|
||||
extraction_types,
|
||||
SelectorWaitStates,
|
||||
FollowRedirects,
|
||||
)
|
||||
|
||||
SessionType = Literal["dynamic", "stealthy"]
|
||||
@@ -122,6 +123,7 @@ class ScraplingMCPServer:
|
||||
async def open_session(
|
||||
self,
|
||||
session_type: SessionType,
|
||||
session_id: Optional[str] = None,
|
||||
headless: bool = True,
|
||||
google_search: bool = True,
|
||||
real_chrome: bool = False,
|
||||
@@ -151,6 +153,7 @@ class ScraplingMCPServer:
|
||||
Use close_session to close the session when done, and list_sessions to see all active sessions.
|
||||
|
||||
:param session_type: The type of session to open. Use "dynamic" for standard Playwright browser, or "stealthy" for anti-bot bypass with fingerprint spoofing.
|
||||
:param session_id: Optional custom session ID. If not provided, a random 12-character hex ID will be generated. Useful for naming sessions for easier management.
|
||||
:param headless: Run the browser in headless/hidden (default), or headful/visible mode.
|
||||
:param google_search: Enabled by default, Scrapling will set a Google referer header.
|
||||
:param real_chrome: If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch an instance of your browser and use it.
|
||||
@@ -174,6 +177,12 @@ class ScraplingMCPServer:
|
||||
:param solve_cloudflare: (Stealthy only) Solves all types of the Cloudflare's Turnstile/Interstitial challenges.
|
||||
:param additional_args: (Stealthy only) Additional arguments to be passed to Playwright's context as additional settings.
|
||||
"""
|
||||
session_id = session_id or uuid4().hex[:12]
|
||||
if session_id in self._sessions:
|
||||
raise ValueError(
|
||||
f"Session '{session_id}' already exists. Use a different ID or close the existing session first."
|
||||
)
|
||||
|
||||
common_kwargs: Dict[str, Any] = dict(
|
||||
wait=wait,
|
||||
proxy=proxy,
|
||||
@@ -182,6 +191,7 @@ class ScraplingMCPServer:
|
||||
cookies=cookies,
|
||||
cdp_url=cdp_url,
|
||||
headless=headless,
|
||||
block_ads=True,
|
||||
max_pages=max_pages,
|
||||
useragent=useragent,
|
||||
timezone_id=timezone_id,
|
||||
@@ -209,7 +219,6 @@ class ScraplingMCPServer:
|
||||
|
||||
await session.start()
|
||||
|
||||
session_id = uuid4().hex[:12]
|
||||
entry = _SessionEntry(session=session, session_type=session_type)
|
||||
self._sessions[session_id] = entry
|
||||
|
||||
@@ -262,7 +271,7 @@ class ScraplingMCPServer:
|
||||
headers: Optional[Mapping[str, Optional[str]]] = None,
|
||||
cookies: Optional[Dict[str, str]] = None,
|
||||
timeout: Optional[int | float] = 30,
|
||||
follow_redirects: bool = True,
|
||||
follow_redirects: FollowRedirects = "safe",
|
||||
max_redirects: int = 30,
|
||||
retries: Optional[int] = 3,
|
||||
retry_delay: Optional[int] = 1,
|
||||
@@ -289,7 +298,7 @@ class ScraplingMCPServer:
|
||||
:param headers: Headers to include in the request.
|
||||
:param cookies: Cookies to use in the request.
|
||||
:param timeout: Number of seconds to wait before timing out.
|
||||
:param follow_redirects: Whether to follow redirects. Defaults to True.
|
||||
:param follow_redirects: Whether to follow redirects. Defaults to "safe", which follows redirects but rejects those targeting internal/private IPs (SSRF protection). Pass True to follow all redirects without restriction.
|
||||
:param max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited.
|
||||
:param retries: Number of retry attempts. Defaults to 3.
|
||||
:param retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second.
|
||||
@@ -335,7 +344,7 @@ class ScraplingMCPServer:
|
||||
headers: Optional[Mapping[str, Optional[str]]] = None,
|
||||
cookies: Optional[Dict[str, str]] = None,
|
||||
timeout: Optional[int | float] = 30,
|
||||
follow_redirects: bool = True,
|
||||
follow_redirects: FollowRedirects = "safe",
|
||||
max_redirects: int = 30,
|
||||
retries: Optional[int] = 3,
|
||||
retry_delay: Optional[int] = 1,
|
||||
@@ -362,7 +371,7 @@ class ScraplingMCPServer:
|
||||
:param headers: Headers to include in the request.
|
||||
:param cookies: Cookies to use in the request.
|
||||
:param timeout: Number of seconds to wait before timing out.
|
||||
:param follow_redirects: Whether to follow redirects. Defaults to True.
|
||||
:param follow_redirects: Whether to follow redirects. Defaults to "safe", which follows redirects but rejects those targeting internal/private IPs (SSRF protection). Pass True to follow all redirects without restriction.
|
||||
:param max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited.
|
||||
:param retries: Number of retry attempts. Defaults to 3.
|
||||
:param retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second.
|
||||
@@ -568,6 +577,7 @@ class ScraplingMCPServer:
|
||||
cookies=cookies,
|
||||
cdp_url=cdp_url,
|
||||
headless=headless,
|
||||
block_ads=True,
|
||||
max_pages=len(urls),
|
||||
useragent=useragent,
|
||||
timezone_id=timezone_id,
|
||||
@@ -776,6 +786,7 @@ class ScraplingMCPServer:
|
||||
timeout=timeout,
|
||||
cookies=cookies,
|
||||
headless=headless,
|
||||
block_ads=True,
|
||||
useragent=useragent,
|
||||
timezone_id=timezone_id,
|
||||
real_chrome=real_chrome,
|
||||
|
||||
@@ -26,7 +26,12 @@ class SelectorsGeneration:
|
||||
if target.parent:
|
||||
if target.attrib.get("id"):
|
||||
# id is enough
|
||||
part = f"#{target.attrib['id']}" if css else f"[@id='{target.attrib['id']}']"
|
||||
if css:
|
||||
part = f"#{target.attrib['id']}"
|
||||
elif full_path:
|
||||
part = f"*[@id='{target.attrib['id']}']"
|
||||
else:
|
||||
part = f"[@id='{target.attrib['id']}']"
|
||||
selectorPath.append(part)
|
||||
if not full_path:
|
||||
return " > ".join(reversed(selectorPath)) if css else "//*" + "/".join(reversed(selectorPath))
|
||||
@@ -47,7 +52,7 @@ class SelectorsGeneration:
|
||||
if counter[target.tag] > 1:
|
||||
part += f":nth-of-type({counter[target.tag]})" if css else f"[{counter[target.tag]}]"
|
||||
|
||||
selectorPath.append(part)
|
||||
selectorPath.append(part)
|
||||
target = target.parent
|
||||
if target is None or target.tag == "html":
|
||||
return " > ".join(reversed(selectorPath)) if css else "//" + "/".join(reversed(selectorPath))
|
||||
|
||||
@@ -294,7 +294,7 @@ class CurlParser:
|
||||
headers=headers,
|
||||
cookies=cookies,
|
||||
proxy=proxies,
|
||||
follow_redirects=True, # Scrapling default is True
|
||||
follow_redirects="safe", # Follows redirects but rejects those to internal/private IPs
|
||||
)
|
||||
|
||||
def convert2fetcher(self, curl_command: Request | str) -> Optional[Response]:
|
||||
|
||||
@@ -196,11 +196,14 @@ class SyncSession:
|
||||
context_options = self._build_context_with_proxy(proxy)
|
||||
context: BrowserContext = self.browser.new_context(**context_options)
|
||||
|
||||
page_info = None
|
||||
try:
|
||||
context = self._initialize_context(self._config, context)
|
||||
page_info = self._get_page(timeout, extra_headers, disable_resources, blocked_domains, context=context)
|
||||
yield page_info
|
||||
finally:
|
||||
if page_info is not None and page_info in self.page_pool.pages:
|
||||
self.page_pool.pages.remove(page_info)
|
||||
context.close()
|
||||
else:
|
||||
# Standard mode: use PagePool with persistent context
|
||||
@@ -380,6 +383,7 @@ class AsyncSession:
|
||||
context_options = self._build_context_with_proxy(proxy)
|
||||
context: AsyncBrowserContext = await self.browser.new_context(**context_options)
|
||||
|
||||
page_info = None
|
||||
try:
|
||||
context = await self._initialize_context(self._config, context)
|
||||
page_info = await self._get_page(
|
||||
@@ -387,6 +391,8 @@ class AsyncSession:
|
||||
)
|
||||
yield page_info
|
||||
finally:
|
||||
if page_info is not None and page_info in self.page_pool.pages:
|
||||
self.page_pool.pages.remove(page_info)
|
||||
await context.close()
|
||||
else:
|
||||
# Standard mode: use PagePool with persistent context
|
||||
@@ -449,6 +455,13 @@ class BaseSessionMixin:
|
||||
if config.extra_flags or extra_flags:
|
||||
flags = list(set(tuple(flags) + tuple(config.extra_flags or extra_flags or ())))
|
||||
|
||||
if config.dns_over_https:
|
||||
doh_flag = "--dns-over-https-templates=https://cloudflare-dns.com/dns-query"
|
||||
if isinstance(flags, list):
|
||||
flags.append(doh_flag)
|
||||
else:
|
||||
flags = list(flags) + [doh_flag]
|
||||
|
||||
self._browser_options.update(
|
||||
{
|
||||
"args": flags,
|
||||
|
||||
@@ -47,7 +47,8 @@ class DynamicSession(SyncSession, DynamicSessionMixin):
|
||||
:param network_idle: Wait for the page until there are no network connections for at least 500 ms.
|
||||
:param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
|
||||
:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
|
||||
:param page_action: Added for automation. A function that takes the `page` object and does the automation you need.
|
||||
:param page_action: Added for automation. A function that takes the `page` object, runs after navigation, and does the automation you need.
|
||||
:param page_setup: A function that takes the `page` object, runs before navigation. Use it to register event listeners or routes that must be set up before the page loads.
|
||||
:param wait_selector: Wait for a specific CSS selector to be in a specific state.
|
||||
:param init_script: An absolute path to a JavaScript file to be executed on page creation for all pages in this session.
|
||||
:param locale: Specify user locale, for example, `en-GB`, `de-DE`, etc. Locale will affect navigator.language value, Accept-Language request header value as well as number and date formatting
|
||||
@@ -105,7 +106,8 @@ class DynamicSession(SyncSession, DynamicSessionMixin):
|
||||
:param google_search: Enabled by default, Scrapling will set a Google referer header.
|
||||
:param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
|
||||
:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
|
||||
:param page_action: Added for automation. A function that takes the `page` object and does the automation you need.
|
||||
:param page_action: Added for automation. A function that takes the `page` object, runs after navigation, and does the automation you need.
|
||||
:param page_setup: A function that takes the `page` object, runs before navigation. Use it to register event listeners or routes that must be set up before the page loads.
|
||||
:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by `google_search` takes priority over the referer set here if used together._
|
||||
:param disable_resources: Drop requests for unnecessary resources for a speed boost.
|
||||
Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
|
||||
@@ -152,6 +154,12 @@ class DynamicSession(SyncSession, DynamicSessionMixin):
|
||||
),
|
||||
)
|
||||
|
||||
if params.page_setup:
|
||||
try:
|
||||
params.page_setup(page)
|
||||
except Exception as e: # pragma: no cover
|
||||
log.error(f"Error executing page_setup: {e}")
|
||||
|
||||
try:
|
||||
first_response = page.goto(url, referer=referer)
|
||||
self._wait_for_page_stability(page, params.load_dom, params.network_idle)
|
||||
@@ -228,7 +236,8 @@ class AsyncDynamicSession(AsyncSession, DynamicSessionMixin):
|
||||
:param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute.
|
||||
:param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
|
||||
:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
|
||||
:param page_action: Added for automation. A function that takes the `page` object and does the automation you need.
|
||||
:param page_action: Added for automation. A function that takes the `page` object, runs after navigation, and does the automation you need.
|
||||
:param page_setup: A function that takes the `page` object, runs before navigation. Use it to register event listeners or routes that must be set up before the page loads.
|
||||
:param wait_selector: Wait for a specific CSS selector to be in a specific state.
|
||||
:param init_script: An absolute path to a JavaScript file to be executed on page creation for all pages in this session.
|
||||
:param locale: Specify user locale, for example, `en-GB`, `de-DE`, etc. Locale will affect navigator.language value, Accept-Language request header value as well as number and date formatting
|
||||
@@ -285,7 +294,8 @@ class AsyncDynamicSession(AsyncSession, DynamicSessionMixin):
|
||||
:param google_search: Enabled by default, Scrapling will set a Google referer header.
|
||||
:param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
|
||||
:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
|
||||
:param page_action: Added for automation. A function that takes the `page` object and does the automation you need.
|
||||
:param page_action: Added for automation. A function that takes the `page` object, runs after navigation, and does the automation you need.
|
||||
:param page_setup: A function that takes the `page` object, runs before navigation. Use it to register event listeners or routes that must be set up before the page loads.
|
||||
:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by `google_search` takes priority over the referer set here if used together._
|
||||
:param disable_resources: Drop requests for unnecessary resources for a speed boost.
|
||||
Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
|
||||
@@ -333,6 +343,12 @@ class AsyncDynamicSession(AsyncSession, DynamicSessionMixin):
|
||||
),
|
||||
)
|
||||
|
||||
if params.page_setup:
|
||||
try:
|
||||
await params.page_setup(page)
|
||||
except Exception as e: # pragma: no cover
|
||||
log.error(f"Error executing page_setup: {e}")
|
||||
|
||||
try:
|
||||
first_response = await page.goto(url, referer=referer)
|
||||
await self._wait_for_page_stability(page, params.load_dom, params.network_idle)
|
||||
|
||||
@@ -47,7 +47,8 @@ class StealthySession(SyncSession, StealthySessionMixin):
|
||||
:param network_idle: Wait for the page until there are no network connections for at least 500 ms.
|
||||
:param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
|
||||
:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
|
||||
:param page_action: Added for automation. A function that takes the `page` object and does the automation you need.
|
||||
:param page_action: Added for automation. A function that takes the `page` object, runs after navigation, and does the automation you need.
|
||||
:param page_setup: A function that takes the `page` object, runs before navigation. Use it to register event listeners or routes that must be set up before the page loads.
|
||||
:param wait_selector: Wait for a specific CSS selector to be in a specific state.
|
||||
:param init_script: An absolute path to a JavaScript file to be executed on page creation for all pages in this session.
|
||||
:param locale: Specify user locale, for example, `en-GB`, `de-DE`, etc. Locale will affect navigator.language value, Accept-Language request header value as well as number and date formatting
|
||||
@@ -187,7 +188,8 @@ class StealthySession(SyncSession, StealthySessionMixin):
|
||||
:param google_search: Enabled by default, Scrapling will set a Google referer header.
|
||||
:param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
|
||||
:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
|
||||
:param page_action: Added for automation. A function that takes the `page` object and does the automation you need.
|
||||
:param page_action: Added for automation. A function that takes the `page` object, runs after navigation, and does the automation you need.
|
||||
:param page_setup: A function that takes the `page` object, runs before navigation. Use it to register event listeners or routes that must be set up before the page loads.
|
||||
:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by `google_search` takes priority over the referer set here if used together._
|
||||
:param disable_resources: Drop requests for unnecessary resources for a speed boost.
|
||||
Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
|
||||
@@ -235,6 +237,12 @@ class StealthySession(SyncSession, StealthySessionMixin):
|
||||
),
|
||||
)
|
||||
|
||||
if params.page_setup:
|
||||
try:
|
||||
params.page_setup(page)
|
||||
except Exception as e: # pragma: no cover
|
||||
log.error(f"Error executing page_setup: {e}")
|
||||
|
||||
try:
|
||||
first_response = page.goto(url, referer=referer)
|
||||
self._wait_for_page_stability(page, params.load_dom, params.network_idle)
|
||||
@@ -315,7 +323,8 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin):
|
||||
:param network_idle: Wait for the page until there are no network connections for at least 500 ms.
|
||||
:param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
|
||||
:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
|
||||
:param page_action: Added for automation. A function that takes the `page` object and does the automation you need.
|
||||
:param page_action: Added for automation. A function that takes the `page` object, runs after navigation, and does the automation you need.
|
||||
:param page_setup: A function that takes the `page` object, runs before navigation. Use it to register event listeners or routes that must be set up before the page loads.
|
||||
:param wait_selector: Wait for a specific CSS selector to be in a specific state.
|
||||
:param init_script: An absolute path to a JavaScript file to be executed on page creation for all pages in this session.
|
||||
:param locale: Specify user locale, for example, `en-GB`, `de-DE`, etc. Locale will affect navigator.language value, Accept-Language request header value as well as number and date formatting
|
||||
@@ -454,7 +463,8 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin):
|
||||
:param google_search: Enabled by default, Scrapling will set a Google referer header.
|
||||
:param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
|
||||
:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
|
||||
:param page_action: Added for automation. A function that takes the `page` object and does the automation you need.
|
||||
:param page_action: Added for automation. A function that takes the `page` object, runs after navigation, and does the automation you need.
|
||||
:param page_setup: A function that takes the `page` object, runs before navigation. Use it to register event listeners or routes that must be set up before the page loads.
|
||||
:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by `google_search` takes priority over the referer set here if used together._
|
||||
:param disable_resources: Drop requests for unnecessary resources for a speed boost.
|
||||
Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
|
||||
@@ -503,6 +513,12 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin):
|
||||
),
|
||||
)
|
||||
|
||||
if params.page_setup:
|
||||
try:
|
||||
await params.page_setup(page)
|
||||
except Exception as e: # pragma: no cover
|
||||
log.error(f"Error executing page_setup: {e}")
|
||||
|
||||
try:
|
||||
first_response = await page.goto(url, referer=referer)
|
||||
await self._wait_for_page_stability(page, params.load_dom, params.network_idle)
|
||||
|
||||
@@ -19,6 +19,7 @@ from scrapling.core._types import (
|
||||
TypeAlias,
|
||||
SetCookieParam,
|
||||
SelectorWaitStates,
|
||||
FollowRedirects,
|
||||
)
|
||||
from scrapling.engines.toolbelt.proxy_rotation import ProxyRotator
|
||||
|
||||
@@ -39,7 +40,7 @@ class RequestsSession(TypedDict, total=False):
|
||||
headers: Optional[Mapping[str, Optional[str]]]
|
||||
retries: Optional[int]
|
||||
retry_delay: Optional[int]
|
||||
follow_redirects: Optional[bool]
|
||||
follow_redirects: Optional[FollowRedirects]
|
||||
max_redirects: Optional[int]
|
||||
verify: Optional[bool]
|
||||
cert: Optional[str | Tuple[str, str]]
|
||||
@@ -73,6 +74,7 @@ class PlaywrightSession(TypedDict, total=False):
|
||||
wait: int | float
|
||||
timezone_id: str | None
|
||||
page_action: Optional[Callable]
|
||||
page_setup: Optional[Callable]
|
||||
proxy: Optional[str | Dict[str, str] | Tuple]
|
||||
proxy_rotator: Optional[ProxyRotator]
|
||||
extra_headers: Optional[Dict[str, str]]
|
||||
@@ -87,10 +89,12 @@ class PlaywrightSession(TypedDict, total=False):
|
||||
useragent: Optional[str]
|
||||
extra_flags: Optional[List[str]]
|
||||
blocked_domains: Optional[Set[str]]
|
||||
block_ads: bool
|
||||
retries: int
|
||||
retry_delay: int | float
|
||||
capture_xhr: str | None
|
||||
executable_path: Optional[str]
|
||||
dns_over_https: bool
|
||||
|
||||
|
||||
class PlaywrightFetchParams(TypedDict, total=False):
|
||||
@@ -102,6 +106,7 @@ class PlaywrightFetchParams(TypedDict, total=False):
|
||||
disable_resources: bool
|
||||
wait_selector: Optional[str]
|
||||
page_action: Optional[Callable]
|
||||
page_setup: Optional[Callable]
|
||||
selector_config: Optional[Dict]
|
||||
extra_headers: Optional[Dict[str, str]]
|
||||
wait_selector_state: SelectorWaitStates
|
||||
|
||||
@@ -53,7 +53,7 @@ def _is_invalid_cdp_url(cdp_url: str) -> bool | str:
|
||||
# Type aliases for cleaner annotations
|
||||
PagesCount = Annotated[int, Meta(ge=1, le=50)]
|
||||
RetriesCount = Annotated[int, Meta(ge=1, le=10)]
|
||||
Seconds = Annotated[int, float, Meta(ge=0)]
|
||||
Seconds = Annotated[float, Meta(ge=0)]
|
||||
|
||||
|
||||
class PlaywrightConfig(Struct, kw_only=True, frozen=False, weakref=True):
|
||||
@@ -71,6 +71,7 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False, weakref=True):
|
||||
wait: Seconds = 0
|
||||
timezone_id: str | None = ""
|
||||
page_action: Optional[Callable] = None
|
||||
page_setup: Optional[Callable] = None
|
||||
proxy: Optional[str | Dict[str, str] | Tuple] = None # The default value for proxy in Playwright's source is `None`
|
||||
proxy_rotator: Optional[ProxyRotator] = None
|
||||
extra_headers: Optional[Dict[str, str]] = None
|
||||
@@ -85,15 +86,19 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False, weakref=True):
|
||||
useragent: Optional[str] = None
|
||||
extra_flags: Optional[List[str]] = None
|
||||
blocked_domains: Optional[Set[str]] = None
|
||||
block_ads: bool = False
|
||||
retries: RetriesCount = 3
|
||||
retry_delay: Seconds = 1
|
||||
capture_xhr: str | None = None
|
||||
executable_path: Optional[str] = None
|
||||
dns_over_https: bool = False
|
||||
|
||||
def __post_init__(self): # pragma: no cover
|
||||
"""Custom validation after msgspec validation"""
|
||||
if self.page_action and not callable(self.page_action):
|
||||
raise TypeError(f"page_action must be callable, got {type(self.page_action).__name__}")
|
||||
if self.page_setup and not callable(self.page_setup):
|
||||
raise TypeError(f"page_setup must be callable, got {type(self.page_setup).__name__}")
|
||||
if self.proxy and self.proxy_rotator:
|
||||
raise ValueError(
|
||||
"Cannot use 'proxy_rotator' together with 'proxy'. "
|
||||
@@ -127,6 +132,14 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False, weakref=True):
|
||||
if validation_msg:
|
||||
raise ValueError(validation_msg)
|
||||
|
||||
if self.block_ads:
|
||||
from scrapling.engines.toolbelt.ad_domains import AD_DOMAINS
|
||||
|
||||
if self.blocked_domains:
|
||||
self.blocked_domains = self.blocked_domains | set(AD_DOMAINS)
|
||||
else:
|
||||
self.blocked_domains = set(AD_DOMAINS)
|
||||
|
||||
|
||||
class StealthConfig(PlaywrightConfig, kw_only=True, frozen=False, weakref=True):
|
||||
allow_webgl: bool = True
|
||||
@@ -150,6 +163,7 @@ class _fetch_params:
|
||||
timeout: Seconds
|
||||
wait: Seconds
|
||||
page_action: Optional[Callable]
|
||||
page_setup: Optional[Callable]
|
||||
extra_headers: Optional[Dict[str, str]]
|
||||
disable_resources: bool
|
||||
wait_selector: Optional[str]
|
||||
|
||||
+13
-11
@@ -20,6 +20,7 @@ from scrapling.core._types import (
|
||||
Optional,
|
||||
Awaitable,
|
||||
SUPPORTED_HTTP_METHODS,
|
||||
FollowRedirects,
|
||||
)
|
||||
|
||||
from .toolbelt.custom import Response
|
||||
@@ -77,7 +78,7 @@ class _ConfigurationLogic(ABC):
|
||||
self._default_headers = kwargs.get("headers") or {}
|
||||
self._default_retries = kwargs.get("retries", 3)
|
||||
self._default_retry_delay = kwargs.get("retry_delay", 1)
|
||||
self._default_follow_redirects = kwargs.get("follow_redirects", True)
|
||||
self._default_follow_redirects = kwargs.get("follow_redirects", "safe")
|
||||
self._default_max_redirects = kwargs.get("max_redirects", 30)
|
||||
self._default_verify = kwargs.get("verify", True)
|
||||
self._default_cert = kwargs.get("cert") or None
|
||||
@@ -250,6 +251,7 @@ class _SyncSessionLogic(_ConfigurationLogic):
|
||||
request_args = self._merge_request_args(stealth=stealth, proxy=proxy, **kwargs)
|
||||
try:
|
||||
response = session.request(method, **request_args)
|
||||
assert response is not None
|
||||
result = ResponseFactory.from_http_request(response, selector_config, meta={"proxy": proxy})
|
||||
return result
|
||||
except CurlError as e: # pragma: no cover
|
||||
@@ -284,7 +286,7 @@ class _SyncSessionLogic(_ConfigurationLogic):
|
||||
- headers: Headers to include in the request.
|
||||
- cookies: Cookies to use in the request.
|
||||
- timeout: Number of seconds to wait before timing out.
|
||||
- follow_redirects: Whether to follow redirects. Defaults to True.
|
||||
- follow_redirects: Whether to follow redirects. Defaults to "safe" (rejects redirects to internal/private IPs).
|
||||
- max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited.
|
||||
- retries: Number of retry attempts. Defaults to 3.
|
||||
- retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second.
|
||||
@@ -316,7 +318,7 @@ class _SyncSessionLogic(_ConfigurationLogic):
|
||||
- headers: Headers to include in the request.
|
||||
- cookies: Cookies to use in the request.
|
||||
- timeout: Number of seconds to wait before timing out.
|
||||
- follow_redirects: Whether to follow redirects. Defaults to True.
|
||||
- follow_redirects: Whether to follow redirects. Defaults to "safe" (rejects redirects to internal/private IPs).
|
||||
- max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited.
|
||||
- retries: Number of retry attempts. Defaults to 3.
|
||||
- retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second.
|
||||
@@ -348,7 +350,7 @@ class _SyncSessionLogic(_ConfigurationLogic):
|
||||
- headers: Headers to include in the request.
|
||||
- cookies: Cookies to use in the request.
|
||||
- timeout: Number of seconds to wait before timing out.
|
||||
- follow_redirects: Whether to follow redirects. Defaults to True.
|
||||
- follow_redirects: Whether to follow redirects. Defaults to "safe" (rejects redirects to internal/private IPs).
|
||||
- max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited.
|
||||
- retries: Number of retry attempts. Defaults to 3.
|
||||
- retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second.
|
||||
@@ -380,7 +382,7 @@ class _SyncSessionLogic(_ConfigurationLogic):
|
||||
- headers: Headers to include in the request.
|
||||
- cookies: Cookies to use in the request.
|
||||
- timeout: Number of seconds to wait before timing out.
|
||||
- follow_redirects: Whether to follow redirects. Defaults to True.
|
||||
- follow_redirects: Whether to follow redirects. Defaults to "safe" (rejects redirects to internal/private IPs).
|
||||
- max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited.
|
||||
- retries: Number of retry attempts. Defaults to 3.
|
||||
- retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second.
|
||||
@@ -501,7 +503,7 @@ class _ASyncSessionLogic(_ConfigurationLogic):
|
||||
- headers: Headers to include in the request.
|
||||
- cookies: Cookies to use in the request.
|
||||
- timeout: Number of seconds to wait before timing out.
|
||||
- follow_redirects: Whether to follow redirects. Defaults to True.
|
||||
- follow_redirects: Whether to follow redirects. Defaults to "safe" (rejects redirects to internal/private IPs).
|
||||
- max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited.
|
||||
- retries: Number of retry attempts. Defaults to 3.
|
||||
- retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second.
|
||||
@@ -533,7 +535,7 @@ class _ASyncSessionLogic(_ConfigurationLogic):
|
||||
- headers: Headers to include in the request.
|
||||
- cookies: Cookies to use in the request.
|
||||
- timeout: Number of seconds to wait before timing out.
|
||||
- follow_redirects: Whether to follow redirects. Defaults to True.
|
||||
- follow_redirects: Whether to follow redirects. Defaults to "safe" (rejects redirects to internal/private IPs).
|
||||
- max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited.
|
||||
- retries: Number of retry attempts. Defaults to 3.
|
||||
- retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second.
|
||||
@@ -565,7 +567,7 @@ class _ASyncSessionLogic(_ConfigurationLogic):
|
||||
- headers: Headers to include in the request.
|
||||
- cookies: Cookies to use in the request.
|
||||
- timeout: Number of seconds to wait before timing out.
|
||||
- follow_redirects: Whether to follow redirects. Defaults to True.
|
||||
- follow_redirects: Whether to follow redirects. Defaults to "safe" (rejects redirects to internal/private IPs).
|
||||
- max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited.
|
||||
- retries: Number of retry attempts. Defaults to 3.
|
||||
- retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second.
|
||||
@@ -597,7 +599,7 @@ class _ASyncSessionLogic(_ConfigurationLogic):
|
||||
- headers: Headers to include in the request.
|
||||
- cookies: Cookies to use in the request.
|
||||
- timeout: Number of seconds to wait before timing out.
|
||||
- follow_redirects: Whether to follow redirects. Defaults to True.
|
||||
- follow_redirects: Whether to follow redirects. Defaults to "safe" (rejects redirects to internal/private IPs).
|
||||
- max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited.
|
||||
- retries: Number of retry attempts. Defaults to 3.
|
||||
- retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second.
|
||||
@@ -662,7 +664,7 @@ class FetcherSession:
|
||||
headers: Optional[Dict[str, str]] = None,
|
||||
retries: Optional[int] = 3,
|
||||
retry_delay: Optional[int] = 1,
|
||||
follow_redirects: bool = True,
|
||||
follow_redirects: FollowRedirects = "safe",
|
||||
max_redirects: int = 30,
|
||||
verify: bool = True,
|
||||
cert: Optional[str | Tuple[str, str]] = None,
|
||||
@@ -681,7 +683,7 @@ class FetcherSession:
|
||||
:param headers: Headers to include in the session with every request.
|
||||
:param retries: Number of retry attempts. Defaults to 3.
|
||||
:param retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second.
|
||||
:param follow_redirects: Whether to follow redirects. Defaults to True.
|
||||
:param follow_redirects: Whether to follow redirects. Defaults to "safe", which follows redirects but rejects those targeting internal/private IPs (SSRF protection). Pass True to follow all redirects without restriction.
|
||||
:param max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited.
|
||||
:param verify: Whether to verify HTTPS certificates. Defaults to True.
|
||||
:param cert: Tuple of (cert, key) filenames for the client certificate.
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -19,6 +19,27 @@ class ProxyDict(Struct):
|
||||
password: str = ""
|
||||
|
||||
|
||||
def _is_domain_blocked(hostname: str, domains: frozenset) -> bool:
|
||||
"""Check if a hostname matches any blocked domain using O(1) frozenset lookups.
|
||||
|
||||
Walks up the hostname's suffix chain: for "tracker.ads.doubleclick.net",
|
||||
checks "tracker.ads.doubleclick.net", "ads.doubleclick.net", "doubleclick.net".
|
||||
|
||||
:param hostname: The hostname to check.
|
||||
:param domains: A frozenset of blocked domain names.
|
||||
:return: True if the hostname or any of its parent domains is in the blocked set.
|
||||
"""
|
||||
if hostname in domains:
|
||||
return True
|
||||
idx = hostname.find(".")
|
||||
while idx != -1:
|
||||
suffix = hostname[idx + 1 :]
|
||||
if "." in suffix and suffix in domains:
|
||||
return True
|
||||
idx = hostname.find(".", idx + 1)
|
||||
return False
|
||||
|
||||
|
||||
def create_intercept_handler(disable_resources: bool, blocked_domains: Optional[Set[str]] = None) -> Callable:
|
||||
"""Create a route handler that blocks both resource types and specific domains.
|
||||
|
||||
@@ -27,7 +48,7 @@ def create_intercept_handler(disable_resources: bool, blocked_domains: Optional[
|
||||
:return: A sync route handler function.
|
||||
"""
|
||||
disabled_resources = EXTRA_RESOURCES if disable_resources else set()
|
||||
domains = blocked_domains or set()
|
||||
domains = frozenset(blocked_domains) if blocked_domains else frozenset()
|
||||
|
||||
def handler(route: Route):
|
||||
if route.request.resource_type in disabled_resources:
|
||||
@@ -35,7 +56,7 @@ def create_intercept_handler(disable_resources: bool, blocked_domains: Optional[
|
||||
route.abort()
|
||||
elif domains:
|
||||
hostname = urlparse(route.request.url).hostname or ""
|
||||
if any(hostname == d or hostname.endswith("." + d) for d in domains):
|
||||
if _is_domain_blocked(hostname, domains):
|
||||
log.debug(f'Blocking request to blocked domain "{hostname}" ({route.request.url})')
|
||||
route.abort()
|
||||
else:
|
||||
@@ -54,7 +75,7 @@ def create_async_intercept_handler(disable_resources: bool, blocked_domains: Opt
|
||||
:return: An async route handler function.
|
||||
"""
|
||||
disabled_resources = EXTRA_RESOURCES if disable_resources else set()
|
||||
domains = blocked_domains or set()
|
||||
domains = frozenset(blocked_domains) if blocked_domains else frozenset()
|
||||
|
||||
async def handler(route: async_Route):
|
||||
if route.request.resource_type in disabled_resources:
|
||||
@@ -62,7 +83,7 @@ def create_async_intercept_handler(disable_resources: bool, blocked_domains: Opt
|
||||
await route.abort()
|
||||
elif domains:
|
||||
hostname = urlparse(route.request.url).hostname or ""
|
||||
if any(hostname == d or hostname.endswith("." + d) for d in domains):
|
||||
if _is_domain_blocked(hostname, domains):
|
||||
log.debug(f'Blocking request to blocked domain "{hostname}" ({route.request.url})')
|
||||
await route.abort()
|
||||
else:
|
||||
|
||||
@@ -15,13 +15,16 @@ class DynamicFetcher(BaseFetcher):
|
||||
:param headless: Run the browser in headless/hidden (default), or headful/visible mode.
|
||||
:param disable_resources: Drop requests for unnecessary resources for a speed boost.
|
||||
:param blocked_domains: A set of domain names to block requests to. Subdomains are also matched (e.g., ``"example.com"`` blocks ``"sub.example.com"`` too).
|
||||
:param block_ads: Block requests to ~3,500 known ad/tracking domains. Can be combined with ``blocked_domains``.
|
||||
:param dns_over_https: Route DNS queries through Cloudflare's DNS-over-HTTPS to prevent DNS leaks when using proxies.
|
||||
:param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it.
|
||||
:param cookies: Set cookies for the next request.
|
||||
:param network_idle: Wait for the page until there are no network connections for at least 500 ms.
|
||||
:param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute.
|
||||
:param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
|
||||
:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the Response object.
|
||||
:param page_action: Added for automation. A function that takes the `page` object and does the automation you need.
|
||||
:param page_action: Added for automation. A function that takes the `page` object, runs after navigation, and does the automation you need.
|
||||
:param page_setup: A function that takes the `page` object, runs before navigation. Use it to register event listeners or routes that must be set up before the page loads.
|
||||
:param wait_selector: Wait for a specific CSS selector to be in a specific state.
|
||||
:param init_script: An absolute path to a JavaScript file to be executed on page creation with this request.
|
||||
:param locale: Set the locale for the browser if wanted. Defaults to the system default locale.
|
||||
@@ -55,13 +58,16 @@ class DynamicFetcher(BaseFetcher):
|
||||
:param headless: Run the browser in headless/hidden (default), or headful/visible mode.
|
||||
:param disable_resources: Drop requests for unnecessary resources for a speed boost.
|
||||
:param blocked_domains: A set of domain names to block requests to. Subdomains are also matched (e.g., ``"example.com"`` blocks ``"sub.example.com"`` too).
|
||||
:param block_ads: Block requests to ~3,500 known ad/tracking domains. Can be combined with ``blocked_domains``.
|
||||
:param dns_over_https: Route DNS queries through Cloudflare's DNS-over-HTTPS to prevent DNS leaks when using proxies.
|
||||
:param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it.
|
||||
:param cookies: Set cookies for the next request.
|
||||
:param network_idle: Wait for the page until there are no network connections for at least 500 ms.
|
||||
:param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute.
|
||||
:param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
|
||||
:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the Response object.
|
||||
:param page_action: Added for automation. A function that takes the `page` object and does the automation you need.
|
||||
:param page_action: Added for automation. A function that takes the `page` object, runs after navigation, and does the automation you need.
|
||||
:param page_setup: A function that takes the `page` object, runs before navigation. Use it to register event listeners or routes that must be set up before the page loads.
|
||||
:param wait_selector: Wait for a specific CSS selector to be in a specific state.
|
||||
:param init_script: An absolute path to a JavaScript file to be executed on page creation with this request.
|
||||
:param locale: Set the locale for the browser if wanted. Defaults to the system default locale.
|
||||
|
||||
@@ -20,12 +20,15 @@ class StealthyFetcher(BaseFetcher):
|
||||
:param disable_resources: Drop requests for unnecessary resources for a speed boost.
|
||||
Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
|
||||
:param blocked_domains: A set of domain names to block requests to. Subdomains are also matched (e.g., ``"example.com"`` blocks ``"sub.example.com"`` too).
|
||||
:param block_ads: Block requests to ~3,500 known ad/tracking domains. Can be combined with ``blocked_domains``.
|
||||
:param dns_over_https: Route DNS queries through Cloudflare's DNS-over-HTTPS to prevent DNS leaks when using proxies.
|
||||
:param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it.
|
||||
:param cookies: Set cookies for the next request.
|
||||
:param network_idle: Wait for the page until there are no network connections for at least 500 ms.
|
||||
:param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
|
||||
:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
|
||||
:param page_action: Added for automation. A function that takes the `page` object and does the automation you need.
|
||||
:param page_action: Added for automation. A function that takes the `page` object, runs after navigation, and does the automation you need.
|
||||
:param page_setup: A function that takes the `page` object, runs before navigation. Use it to register event listeners or routes that must be set up before the page loads.
|
||||
:param wait_selector: Wait for a specific CSS selector to be in a specific state.
|
||||
:param init_script: An absolute path to a JavaScript file to be executed on page creation for all pages in this session.
|
||||
:param locale: Specify user locale, for example, `en-GB`, `de-DE`, etc. Locale will affect navigator.language value, Accept-Language request header value as well as number and date formatting
|
||||
@@ -69,12 +72,15 @@ class StealthyFetcher(BaseFetcher):
|
||||
:param disable_resources: Drop requests for unnecessary resources for a speed boost.
|
||||
Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
|
||||
:param blocked_domains: A set of domain names to block requests to. Subdomains are also matched (e.g., ``"example.com"`` blocks ``"sub.example.com"`` too).
|
||||
:param block_ads: Block requests to ~3,500 known ad/tracking domains. Can be combined with ``blocked_domains``.
|
||||
:param dns_over_https: Route DNS queries through Cloudflare's DNS-over-HTTPS to prevent DNS leaks when using proxies.
|
||||
:param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it.
|
||||
:param cookies: Set cookies for the next request.
|
||||
:param network_idle: Wait for the page until there are no network connections for at least 500 ms.
|
||||
:param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
|
||||
:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
|
||||
:param page_action: Added for automation. A function that takes the `page` object and does the automation you need.
|
||||
:param page_action: Added for automation. A function that takes the `page` object, runs after navigation, and does the automation you need.
|
||||
:param page_setup: A function that takes the `page` object, runs before navigation. Use it to register event listeners or routes that must be set up before the page loads.
|
||||
:param wait_selector: Wait for a specific CSS selector to be in a specific state.
|
||||
:param init_script: An absolute path to a JavaScript file to be executed on page creation for all pages in this session.
|
||||
:param locale: Specify user locale, for example, `en-GB`, `de-DE`, etc. Locale will affect navigator.language value, Accept-Language request header value as well as number and date formatting
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
from base64 import b64encode, b64decode
|
||||
from pathlib import Path
|
||||
|
||||
import orjson
|
||||
import anyio
|
||||
from anyio import Path as AsyncPath
|
||||
|
||||
from scrapling.core.utils import log
|
||||
from scrapling.core._types import Dict, Optional, Any
|
||||
from scrapling.engines.toolbelt.custom import Response
|
||||
|
||||
|
||||
class ResponseCacheManager:
|
||||
"""Caches HTTP responses to disk for replay during spider development."""
|
||||
|
||||
def __init__(self, cache_dir: str | Path):
|
||||
self._cache_dir = AsyncPath(cache_dir)
|
||||
|
||||
def _cache_path(self, fingerprint: bytes) -> AsyncPath:
|
||||
return self._cache_dir / f"{fingerprint.hex()}.json"
|
||||
|
||||
async def get(self, fingerprint: bytes) -> Optional[Response]:
|
||||
path = self._cache_path(fingerprint)
|
||||
if not await path.exists():
|
||||
return None
|
||||
|
||||
try:
|
||||
async with await anyio.open_file(path, "rb") as f:
|
||||
data: Dict[str, Any] = orjson.loads(await f.read())
|
||||
|
||||
return Response(
|
||||
url=data["url"],
|
||||
content=b64decode(data["content"]),
|
||||
status=data["status"],
|
||||
reason=data["reason"],
|
||||
encoding=data["encoding"],
|
||||
cookies=data["cookies"],
|
||||
headers=data["headers"],
|
||||
request_headers=data["request_headers"],
|
||||
method=data["method"],
|
||||
)
|
||||
except Exception as e:
|
||||
log.warning(f"Failed to read cached response for {fingerprint.hex()}: {e}")
|
||||
return None
|
||||
|
||||
async def put(self, fingerprint: bytes, response: Response, method: str = "GET") -> None:
|
||||
await self._cache_dir.mkdir(parents=True, exist_ok=True)
|
||||
temp_path = self._cache_path(fingerprint).with_suffix(".tmp")
|
||||
|
||||
try:
|
||||
serialized = orjson.dumps(
|
||||
{
|
||||
"url": response.url,
|
||||
"content": b64encode(response.body).decode("ascii"),
|
||||
"status": response.status,
|
||||
"reason": response.reason,
|
||||
"encoding": response.encoding,
|
||||
"cookies": dict(response.cookies) if isinstance(response.cookies, dict) else {},
|
||||
"headers": dict(response.headers),
|
||||
"request_headers": dict(response.request_headers),
|
||||
"method": method,
|
||||
}
|
||||
)
|
||||
async with await anyio.open_file(temp_path, "wb") as f:
|
||||
await f.write(serialized)
|
||||
|
||||
await temp_path.rename(self._cache_path(fingerprint))
|
||||
except Exception as e:
|
||||
if await temp_path.exists():
|
||||
await temp_path.unlink()
|
||||
log.warning(f"Failed to cache response for {fingerprint.hex()}: {e}")
|
||||
|
||||
async def clear(self) -> None:
|
||||
if not await self._cache_dir.exists():
|
||||
return
|
||||
async for entry in self._cache_dir.iterdir():
|
||||
if entry.suffix == ".json":
|
||||
await entry.unlink()
|
||||
log.info(f"Cleared response cache at {self._cache_dir}")
|
||||
+156
-49
@@ -1,16 +1,19 @@
|
||||
import json
|
||||
import pprint
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import anyio
|
||||
from anyio import Path as AsyncPath
|
||||
from anyio import create_task_group, CapacityLimiter, create_memory_object_stream, EndOfStream
|
||||
|
||||
from scrapling.core.utils import log
|
||||
from scrapling.spiders.request import Request
|
||||
from scrapling.spiders.scheduler import Scheduler
|
||||
from scrapling.spiders.session import SessionManager
|
||||
from scrapling.spiders.request import Request, Response
|
||||
from scrapling.spiders.robotstxt import RobotsTxtManager
|
||||
from scrapling.spiders.result import CrawlStats, ItemList
|
||||
from scrapling.spiders.cache import ResponseCacheManager
|
||||
from scrapling.spiders.checkpoint import CheckpointManager, CheckpointData
|
||||
from scrapling.core._types import Dict, Union, Optional, TYPE_CHECKING, Any, AsyncGenerator
|
||||
|
||||
@@ -41,10 +44,29 @@ class CrawlerEngine:
|
||||
)
|
||||
self.stats = CrawlStats()
|
||||
|
||||
if self.spider.robots_txt_obey:
|
||||
|
||||
async def _fetch_robots(url: str, sid: str) -> Response:
|
||||
return await self.session_manager.fetch(Request(url, sid=sid))
|
||||
|
||||
self._robots_manager: Optional[RobotsTxtManager] = RobotsTxtManager(_fetch_robots)
|
||||
else:
|
||||
self._robots_manager = None
|
||||
|
||||
if self.spider.development_mode:
|
||||
cache_dir = self.spider.development_cache_dir or f".scrapling_cache/{self.spider.name}"
|
||||
self._cache_manager: Optional[ResponseCacheManager] = ResponseCacheManager(cache_dir)
|
||||
log.warning("Development mode enabled -- responses will be cached to disk and replayed on subsequent runs")
|
||||
else:
|
||||
self._cache_manager = None
|
||||
|
||||
self._global_limiter = CapacityLimiter(spider.concurrent_requests)
|
||||
self._domain_limiters: dict[str, CapacityLimiter] = {}
|
||||
self._allowed_domains: set[str] = spider.allowed_domains or set()
|
||||
|
||||
if self.spider.robots_txt_obey:
|
||||
self._domain_delays: dict[str, float] = {}
|
||||
|
||||
self._active_tasks: int = 0
|
||||
self._running: bool = False
|
||||
self._items: ItemList = ItemList()
|
||||
@@ -68,11 +90,42 @@ class CrawlerEngine:
|
||||
return True
|
||||
return False
|
||||
|
||||
async def _get_domain_delay(self, request: Request) -> float:
|
||||
"""Resolve the effective download delay for a domain.
|
||||
|
||||
Takes the max of the spider's configured delay and any robots.txt
|
||||
directives (Crawl-delay / Request-rate). Result is cached per domain.
|
||||
"""
|
||||
robots_manager = self._robots_manager
|
||||
if robots_manager is None:
|
||||
return self.spider.download_delay
|
||||
|
||||
domain = request.domain
|
||||
|
||||
if domain in self._domain_delays:
|
||||
return self._domain_delays[domain]
|
||||
|
||||
# For domains covered by _prefetch_robots_txt this is a local parser read.
|
||||
# Domains discovered mid-crawl (not in start_urls) will fetch here.
|
||||
c_delay, r_rate = await robots_manager.get_delay_directives(request.url, request.sid)
|
||||
|
||||
delay = self.spider.download_delay
|
||||
|
||||
if r_rate:
|
||||
req_count, period = r_rate
|
||||
if req_count > 0:
|
||||
delay = max(delay, period / req_count)
|
||||
|
||||
if c_delay is not None:
|
||||
delay = max(delay, c_delay)
|
||||
|
||||
self._domain_delays[domain] = delay
|
||||
return delay
|
||||
|
||||
def _rate_limiter(self, domain: str) -> CapacityLimiter:
|
||||
"""Get or create a per-domain concurrency limiter if enabled, otherwise use the global limiter."""
|
||||
if self.spider.concurrent_requests_per_domain:
|
||||
if domain not in self._domain_limiters:
|
||||
self._domain_limiters[domain] = CapacityLimiter(self.spider.concurrent_requests_per_domain)
|
||||
self._domain_limiters.setdefault(domain, CapacityLimiter(self.spider.concurrent_requests_per_domain))
|
||||
return self._domain_limiters[domain]
|
||||
return self._global_limiter
|
||||
|
||||
@@ -85,47 +138,8 @@ class CrawlerEngine:
|
||||
if not request.sid:
|
||||
request.sid = self.session_manager.default_session_id
|
||||
|
||||
async def _process_request(self, request: Request) -> None:
|
||||
"""Download and process a single request."""
|
||||
async with self._rate_limiter(request.domain):
|
||||
if self.spider.download_delay:
|
||||
await anyio.sleep(self.spider.download_delay)
|
||||
|
||||
if request._session_kwargs.get("proxy"):
|
||||
self.stats.proxies.append(request._session_kwargs["proxy"])
|
||||
if request._session_kwargs.get("proxies"):
|
||||
self.stats.proxies.append(dict(request._session_kwargs["proxies"]))
|
||||
try:
|
||||
response = await self.session_manager.fetch(request)
|
||||
self.stats.increment_requests_count(request.sid or self.session_manager.default_session_id)
|
||||
self.stats.increment_response_bytes(request.domain, len(response.body))
|
||||
self.stats.increment_status(response.status)
|
||||
|
||||
except Exception as e:
|
||||
self.stats.failed_requests_count += 1
|
||||
await self.spider.on_error(request, e)
|
||||
return
|
||||
|
||||
if await self.spider.is_blocked(response):
|
||||
self.stats.blocked_requests_count += 1
|
||||
if request._retry_count < self.spider.max_blocked_retries:
|
||||
retry_request = request.copy()
|
||||
retry_request._retry_count += 1
|
||||
retry_request.priority -= 1 # Don't retry immediately
|
||||
retry_request.dont_filter = True
|
||||
retry_request._session_kwargs.pop("proxy", None)
|
||||
retry_request._session_kwargs.pop("proxies", None)
|
||||
|
||||
new_request = await self.spider.retry_blocked_request(retry_request, response)
|
||||
self._normalize_request(new_request)
|
||||
await self.scheduler.enqueue(new_request)
|
||||
log.info(
|
||||
f"Scheduled blocked request for retry ({retry_request._retry_count}/{self.spider.max_blocked_retries}): {request.url}"
|
||||
)
|
||||
else:
|
||||
log.warning(f"Max retries exceeded for blocked request: {request.url}")
|
||||
return
|
||||
|
||||
async def _run_callbacks(self, request: Request, response: Response) -> None:
|
||||
"""Dispatch response to the request's callback and process yielded items/requests."""
|
||||
callback = request.callback if request.callback else self.spider.parse
|
||||
try:
|
||||
async for result in callback(response):
|
||||
@@ -155,6 +169,75 @@ class CrawlerEngine:
|
||||
log.error(msg, exc_info=e)
|
||||
await self.spider.on_error(request, e)
|
||||
|
||||
async def _process_request(self, request: Request) -> None:
|
||||
"""Download and process a single request."""
|
||||
if self._robots_manager:
|
||||
can_fetch = await self._robots_manager.can_fetch(request.url, request.sid)
|
||||
if not can_fetch:
|
||||
self.stats.robots_disallowed_count += 1
|
||||
log.info(f"Request disallowed by robots.txt: {request.url}")
|
||||
return
|
||||
delay = await self._get_domain_delay(request)
|
||||
else:
|
||||
delay = self.spider.download_delay
|
||||
|
||||
if self._cache_manager and request._fp is not None:
|
||||
cached = await self._cache_manager.get(request._fp)
|
||||
if cached is not None:
|
||||
cached.request = request
|
||||
self.stats.cache_hits += 1
|
||||
self.stats.increment_requests_count(request.sid or self.session_manager.default_session_id)
|
||||
self.stats.increment_response_bytes(request.domain, len(cached.body))
|
||||
self.stats.increment_status(cached.status)
|
||||
log.debug(f"Cache hit: {request.url}")
|
||||
await self._run_callbacks(request, cached)
|
||||
return
|
||||
|
||||
async with self._rate_limiter(request.domain):
|
||||
if delay:
|
||||
await anyio.sleep(delay)
|
||||
|
||||
if request._session_kwargs.get("proxy"):
|
||||
self.stats.proxies.append(request._session_kwargs["proxy"])
|
||||
if request._session_kwargs.get("proxies"):
|
||||
self.stats.proxies.append(dict(request._session_kwargs["proxies"]))
|
||||
try:
|
||||
response = await self.session_manager.fetch(request)
|
||||
self.stats.increment_requests_count(request.sid or self.session_manager.default_session_id)
|
||||
self.stats.increment_response_bytes(request.domain, len(response.body))
|
||||
self.stats.increment_status(response.status)
|
||||
|
||||
except Exception as e:
|
||||
self.stats.failed_requests_count += 1
|
||||
await self.spider.on_error(request, e)
|
||||
return
|
||||
|
||||
if self._cache_manager and request._fp is not None:
|
||||
self.stats.cache_misses += 1
|
||||
await self._cache_manager.put(request._fp, response, request._session_kwargs.get("method", "GET"))
|
||||
|
||||
if await self.spider.is_blocked(response):
|
||||
self.stats.blocked_requests_count += 1
|
||||
if request._retry_count < self.spider.max_blocked_retries:
|
||||
retry_request = request.copy()
|
||||
retry_request._retry_count += 1
|
||||
retry_request.priority -= 1 # Don't retry immediately
|
||||
retry_request.dont_filter = True
|
||||
retry_request._session_kwargs.pop("proxy", None)
|
||||
retry_request._session_kwargs.pop("proxies", None)
|
||||
|
||||
new_request = await self.spider.retry_blocked_request(retry_request, response)
|
||||
self._normalize_request(new_request)
|
||||
await self.scheduler.enqueue(new_request)
|
||||
log.info(
|
||||
f"Scheduled blocked request for retry ({retry_request._retry_count}/{self.spider.max_blocked_retries}): {request.url}"
|
||||
)
|
||||
else:
|
||||
log.warning(f"Max retries exceeded for blocked request: {request.url}")
|
||||
return
|
||||
|
||||
await self._run_callbacks(request, response)
|
||||
|
||||
async def _task_wrapper(self, request: Request) -> None:
|
||||
"""Wrapper to track active task count."""
|
||||
try:
|
||||
@@ -219,6 +302,25 @@ class CrawlerEngine:
|
||||
|
||||
return True
|
||||
|
||||
async def _prefetch_robots_txt(self) -> None:
|
||||
"""Pre-warm the robots.txt cache before the crawl loop starts.
|
||||
|
||||
Extracts unique domains from start_urls, preserving the original scheme.
|
||||
"""
|
||||
if not self._robots_manager or not self.spider.start_urls:
|
||||
return
|
||||
|
||||
# Deduplicate by netloc, preserving the scheme from the first URL per domain
|
||||
seen: set[str] = set()
|
||||
seed_urls: list[str] = []
|
||||
for url in self.spider.start_urls:
|
||||
parsed = urlparse(url)
|
||||
if parsed.netloc not in seen:
|
||||
seen.add(parsed.netloc)
|
||||
seed_urls.append(f"{parsed.scheme}://{parsed.netloc}/")
|
||||
|
||||
await self._robots_manager.prefetch(seed_urls, self.session_manager.default_session_id)
|
||||
|
||||
async def crawl(self) -> CrawlStats:
|
||||
"""Run the spider and return CrawlStats."""
|
||||
self._running = True
|
||||
@@ -227,6 +329,9 @@ class CrawlerEngine:
|
||||
self._pause_requested = False
|
||||
self._force_stop = False
|
||||
self.stats = CrawlStats(start_time=anyio.current_time())
|
||||
self._domain_limiters.clear()
|
||||
if self._robots_manager:
|
||||
self._domain_delays.clear()
|
||||
|
||||
# Check for existing checkpoint
|
||||
resuming = (await self._restore_from_checkpoint()) if self._checkpoint_system_enabled else False
|
||||
@@ -238,6 +343,8 @@ class CrawlerEngine:
|
||||
self.stats.download_delay = self.spider.download_delay
|
||||
await self.spider.on_start(resuming=resuming)
|
||||
|
||||
await self._prefetch_robots_txt()
|
||||
|
||||
try:
|
||||
if not resuming:
|
||||
async for request in self.spider.start_requests():
|
||||
@@ -251,11 +358,7 @@ class CrawlerEngine:
|
||||
while self._running:
|
||||
if self._pause_requested:
|
||||
if self._active_tasks == 0 or self._force_stop:
|
||||
if self._force_stop:
|
||||
log.warning(f"Force stopping with {self._active_tasks} active tasks")
|
||||
tg.cancel_scope.cancel()
|
||||
|
||||
# Only save checkpoint if checkpoint system is enabled
|
||||
# Save checkpoint before canceling to avoid data loss
|
||||
if self._checkpoint_system_enabled:
|
||||
await self._save_checkpoint()
|
||||
self.paused = True
|
||||
@@ -263,6 +366,10 @@ class CrawlerEngine:
|
||||
else:
|
||||
log.info("Spider stopped gracefully")
|
||||
|
||||
if self._force_stop:
|
||||
log.warning(f"Force stopping with {self._active_tasks} active tasks")
|
||||
tg.cancel_scope.cancel()
|
||||
|
||||
self._running = False
|
||||
break
|
||||
|
||||
|
||||
@@ -47,6 +47,9 @@ class CrawlStats:
|
||||
concurrent_requests_per_domain: int = 0
|
||||
failed_requests_count: int = 0
|
||||
offsite_requests_count: int = 0
|
||||
robots_disallowed_count: int = 0
|
||||
cache_hits: int = 0
|
||||
cache_misses: int = 0
|
||||
response_bytes: int = 0
|
||||
items_scraped: int = 0
|
||||
items_dropped: int = 0
|
||||
@@ -95,6 +98,9 @@ class CrawlStats:
|
||||
"sessions_requests_count": self.sessions_requests_count,
|
||||
"failed_requests_count": self.failed_requests_count,
|
||||
"offsite_requests_count": self.offsite_requests_count,
|
||||
"robots_disallowed_count": self.robots_disallowed_count,
|
||||
"cache_hits": self.cache_hits,
|
||||
"cache_misses": self.cache_misses,
|
||||
"blocked_requests_count": self.blocked_requests_count,
|
||||
"response_status_count": self.response_status_count,
|
||||
"response_bytes": self.response_bytes,
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from anyio import create_task_group
|
||||
from protego import Protego
|
||||
|
||||
from scrapling.core._types import Dict, Optional, Callable, Awaitable
|
||||
from scrapling.core.utils import log
|
||||
|
||||
|
||||
class RobotsTxtManager:
|
||||
"""Manages fetching, parsing, and caching of robots.txt files."""
|
||||
|
||||
def __init__(self, fetch_fn: Callable[[str, str], Awaitable]):
|
||||
self._fetch_fn = fetch_fn
|
||||
self._cache: Dict[str, Protego] = {}
|
||||
|
||||
async def _get_parser(self, url: str, sid: str) -> Protego:
|
||||
parsed = urlparse(url)
|
||||
domain = parsed.netloc
|
||||
|
||||
if domain in self._cache:
|
||||
return self._cache[domain]
|
||||
|
||||
scheme = parsed.scheme or "https"
|
||||
robots_url = f"{scheme}://{domain}/robots.txt"
|
||||
content = ""
|
||||
try:
|
||||
response = await self._fetch_fn(robots_url, sid)
|
||||
if response.status == 200:
|
||||
content = response.body.decode(response.encoding, errors="replace")
|
||||
except Exception as e:
|
||||
log.warning(f"Failed to fetch robots.txt for {domain}: {e}")
|
||||
|
||||
try:
|
||||
parser = Protego.parse(content)
|
||||
except Exception as e:
|
||||
log.warning(f"Failed to parse robots.txt for {domain}: {e}")
|
||||
parser = Protego.parse("")
|
||||
|
||||
self._cache[domain] = parser
|
||||
return parser
|
||||
|
||||
async def can_fetch(self, url: str, sid: str) -> bool:
|
||||
"""Check if a URL can be fetched according to the domain's robots.txt.
|
||||
|
||||
:param url: The full URL to check
|
||||
:param sid: Session ID for fetching robots.txt if not yet cached
|
||||
"""
|
||||
parser = await self._get_parser(url, sid)
|
||||
return parser.can_fetch(url, "*")
|
||||
|
||||
async def get_delay_directives(self, url: str, sid: str) -> tuple[Optional[float], Optional[tuple[int, int]]]:
|
||||
"""Return both crawl-delay and request-rate in a single parser lookup.
|
||||
|
||||
:param url: Any URL on the domain to check
|
||||
:param sid: Session ID for fetching robots.txt if not yet cached
|
||||
"""
|
||||
parser = await self._get_parser(url, sid)
|
||||
c_delay = parser.crawl_delay("*")
|
||||
rate = parser.request_rate("*")
|
||||
return (
|
||||
float(c_delay) if c_delay is not None else None,
|
||||
(rate.requests, rate.seconds) if rate is not None else None,
|
||||
)
|
||||
|
||||
async def prefetch(self, urls: list[str], sid: str) -> None:
|
||||
"""Pre-warm the robots.txt cache for a list of seed URLs concurrently.
|
||||
|
||||
:param urls: Seed URLs whose domains should be pre-fetched (one per domain).
|
||||
:param sid: Session ID to use for the robots.txt fetch requests.
|
||||
"""
|
||||
if not urls:
|
||||
return
|
||||
log.debug(f"Pre-fetching robots.txt for {len(urls)} domain(s)")
|
||||
async with create_task_group() as tg:
|
||||
for url in urls:
|
||||
tg.start_soon(self._get_parser, url, sid)
|
||||
@@ -72,6 +72,13 @@ class Spider(ABC):
|
||||
start_urls: list[str] = []
|
||||
allowed_domains: Set[str] = set()
|
||||
|
||||
# Robots.txt compliance
|
||||
robots_txt_obey: bool = False
|
||||
|
||||
# Development mode
|
||||
development_mode: bool = False
|
||||
development_cache_dir: Optional[str] = None
|
||||
|
||||
# Concurrency settings
|
||||
concurrent_requests: int = 4
|
||||
concurrent_requests_per_domain: int = 0
|
||||
|
||||
+2
-2
@@ -14,12 +14,12 @@
|
||||
"mimeType": "image/png"
|
||||
}
|
||||
],
|
||||
"version": "0.4.4",
|
||||
"version": "0.4.7",
|
||||
"packages": [
|
||||
{
|
||||
"registryType": "pypi",
|
||||
"identifier": "scrapling",
|
||||
"version": "0.4.4",
|
||||
"version": "0.4.7",
|
||||
"runtimeHint": "uvx",
|
||||
"packageArguments": [
|
||||
{
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[metadata]
|
||||
name = scrapling
|
||||
version = 0.4.4
|
||||
version = 0.4.7
|
||||
author = Karim Shoair
|
||||
author_email = karim.shoair@pm.me
|
||||
description = Scrapling is an undetectable, powerful, flexible, high-performance Python library that makes Web Scraping easy and effortless as it should be!
|
||||
|
||||
@@ -177,6 +177,25 @@ class TestSessionManagement:
|
||||
with pytest.raises(ValueError, match="not found"):
|
||||
await server.fetch(url=test_url, session_id=session_id)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_open_session_with_custom_id(self, server):
|
||||
"""Test opening a session with a custom session_id"""
|
||||
result = await server.open_session(session_type="dynamic", session_id="my-session", headless=True)
|
||||
assert isinstance(result, SessionCreatedModel)
|
||||
assert result.session_id == "my-session"
|
||||
|
||||
await server.close_session("my-session")
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_open_session_duplicate_id_raises(self, server):
|
||||
"""Test that opening a session with a duplicate session_id raises an error"""
|
||||
await server.open_session(session_type="dynamic", session_id="dupe", headless=True)
|
||||
|
||||
with pytest.raises(ValueError, match="already exists"):
|
||||
await server.open_session(session_type="dynamic", session_id="dupe", headless=True)
|
||||
|
||||
await server.close_session("dupe")
|
||||
|
||||
|
||||
class TestNormalizeCredentials:
|
||||
"""Test the _normalize_credentials helper"""
|
||||
|
||||
@@ -84,6 +84,15 @@ class TestPagePool:
|
||||
with pytest.raises(RuntimeError):
|
||||
pool.add_page(Mock())
|
||||
|
||||
def test_proxy_rotation_pool_leak(self):
|
||||
pool = PagePool(max_pages=1)
|
||||
page_info = pool.add_page(Mock())
|
||||
assert pool.pages_count == 1
|
||||
pool.pages.remove(page_info)
|
||||
assert pool.pages_count == 0
|
||||
pool.add_page(Mock())
|
||||
assert pool.pages_count == 1
|
||||
|
||||
|
||||
|
||||
def test_cleanup_error_pages(self):
|
||||
|
||||
@@ -5,11 +5,9 @@ from scrapling.engines.toolbelt.navigation import (
|
||||
construct_proxy_dict,
|
||||
create_intercept_handler,
|
||||
create_async_intercept_handler,
|
||||
_is_domain_blocked,
|
||||
)
|
||||
from scrapling.engines.toolbelt.fingerprints import (
|
||||
get_os_name,
|
||||
generate_headers
|
||||
)
|
||||
from scrapling.engines.toolbelt.fingerprints import get_os_name, generate_headers
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
@@ -148,31 +146,19 @@ class TestConstructProxyDict:
|
||||
"""Test a basic proxy string"""
|
||||
result = construct_proxy_dict("http://proxy.example.com:8080")
|
||||
|
||||
expected = {
|
||||
"server": "http://proxy.example.com:8080",
|
||||
"username": "",
|
||||
"password": ""
|
||||
}
|
||||
expected = {"server": "http://proxy.example.com:8080", "username": "", "password": ""}
|
||||
assert result == expected
|
||||
|
||||
def test_proxy_string_with_auth(self):
|
||||
"""Test proxy string with authentication"""
|
||||
result = construct_proxy_dict("http://user:pass@proxy.example.com:8080")
|
||||
|
||||
expected = {
|
||||
"server": "http://proxy.example.com:8080",
|
||||
"username": "user",
|
||||
"password": "pass"
|
||||
}
|
||||
expected = {"server": "http://proxy.example.com:8080", "username": "user", "password": "pass"}
|
||||
assert result == expected
|
||||
|
||||
def test_proxy_dict_input(self):
|
||||
"""Test proxy dictionary input"""
|
||||
input_dict = {
|
||||
"server": "http://proxy.example.com:8080",
|
||||
"username": "user",
|
||||
"password": "pass"
|
||||
}
|
||||
input_dict = {"server": "http://proxy.example.com:8080", "username": "user", "password": "pass"}
|
||||
result = construct_proxy_dict(input_dict)
|
||||
|
||||
assert result == input_dict
|
||||
@@ -182,11 +168,7 @@ class TestConstructProxyDict:
|
||||
input_dict = {"server": "http://proxy.example.com:8080"}
|
||||
result = construct_proxy_dict(input_dict)
|
||||
|
||||
expected = {
|
||||
"server": "http://proxy.example.com:8080",
|
||||
"username": "",
|
||||
"password": ""
|
||||
}
|
||||
expected = {"server": "http://proxy.example.com:8080", "username": "", "password": ""}
|
||||
assert result == expected
|
||||
|
||||
def test_invalid_proxy_string(self):
|
||||
@@ -240,7 +222,7 @@ class TestResponse:
|
||||
cookies={"session": "abc123"},
|
||||
headers={"Content-Type": "text/html"},
|
||||
request_headers={"User-Agent": "Test"},
|
||||
encoding="utf-8"
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
assert response.url == "https://example.com"
|
||||
@@ -250,7 +232,7 @@ class TestResponse:
|
||||
|
||||
def test_response_with_bytes_content(self):
|
||||
"""Test Response with 'bytes' content"""
|
||||
content_bytes = "<html><body>Test</body></html>".encode('utf-8')
|
||||
content_bytes = "<html><body>Test</body></html>".encode("utf-8")
|
||||
|
||||
response = Response(
|
||||
url="https://example.com",
|
||||
@@ -259,7 +241,7 @@ class TestResponse:
|
||||
reason="OK",
|
||||
cookies={},
|
||||
headers={},
|
||||
request_headers={}
|
||||
request_headers={},
|
||||
)
|
||||
|
||||
# Should handle 'bytes' content properly
|
||||
@@ -268,6 +250,7 @@ class TestResponse:
|
||||
|
||||
class _MockRequest:
|
||||
"""Minimal mock for Playwright's Request object."""
|
||||
|
||||
def __init__(self, url: str, resource_type: str = "document"):
|
||||
self.url = url
|
||||
self.resource_type = resource_type
|
||||
@@ -275,6 +258,7 @@ class _MockRequest:
|
||||
|
||||
class _MockRoute:
|
||||
"""Minimal mock for Playwright's sync Route object."""
|
||||
|
||||
def __init__(self, url: str, resource_type: str = "document"):
|
||||
self.request = _MockRequest(url, resource_type)
|
||||
self.aborted = False
|
||||
@@ -289,6 +273,7 @@ class _MockRoute:
|
||||
|
||||
class _AsyncMockRoute:
|
||||
"""Minimal mock for Playwright's async Route object."""
|
||||
|
||||
def __init__(self, url: str, resource_type: str = "document"):
|
||||
self.request = _MockRequest(url, resource_type)
|
||||
self.aborted = False
|
||||
@@ -411,3 +396,90 @@ class TestCreateAsyncInterceptHandler:
|
||||
route = _AsyncMockRoute("https://notexample.com/page")
|
||||
await handler(route)
|
||||
assert route.continued
|
||||
|
||||
|
||||
class TestIsDomainBlocked:
|
||||
"""Test the frozenset-based domain matching helper."""
|
||||
|
||||
def test_exact_match(self):
|
||||
domains = frozenset({"doubleclick.net"})
|
||||
assert _is_domain_blocked("doubleclick.net", domains) is True
|
||||
|
||||
def test_subdomain_match(self):
|
||||
domains = frozenset({"doubleclick.net"})
|
||||
assert _is_domain_blocked("ads.doubleclick.net", domains) is True
|
||||
|
||||
def test_deep_subdomain_match(self):
|
||||
domains = frozenset({"doubleclick.net"})
|
||||
assert _is_domain_blocked("tracker.ads.doubleclick.net", domains) is True
|
||||
|
||||
def test_no_partial_match(self):
|
||||
domains = frozenset({"doubleclick.net"})
|
||||
assert _is_domain_blocked("notdoubleclick.net", domains) is False
|
||||
|
||||
def test_no_match(self):
|
||||
domains = frozenset({"doubleclick.net"})
|
||||
assert _is_domain_blocked("example.com", domains) is False
|
||||
|
||||
def test_empty_domains(self):
|
||||
assert _is_domain_blocked("example.com", frozenset()) is False
|
||||
|
||||
def test_multiple_domains(self):
|
||||
domains = frozenset({"ads.com", "tracker.io", "doubleclick.net"})
|
||||
assert _is_domain_blocked("cdn.ads.com", domains) is True
|
||||
assert _is_domain_blocked("tracker.io", domains) is True
|
||||
assert _is_domain_blocked("safe.example.com", domains) is False
|
||||
|
||||
|
||||
class TestAdDomains:
|
||||
"""Test the built-in ad domain list."""
|
||||
|
||||
def test_ad_domains_is_frozenset(self):
|
||||
from scrapling.engines.toolbelt.ad_domains import AD_DOMAINS
|
||||
|
||||
assert isinstance(AD_DOMAINS, frozenset)
|
||||
|
||||
def test_ad_domains_has_entries(self):
|
||||
from scrapling.engines.toolbelt.ad_domains import AD_DOMAINS
|
||||
|
||||
assert len(AD_DOMAINS) > 1000
|
||||
|
||||
def test_ad_domains_contains_known_entries(self):
|
||||
from scrapling.engines.toolbelt.ad_domains import AD_DOMAINS
|
||||
|
||||
assert "doubleclick.net" in AD_DOMAINS
|
||||
assert "googlesyndication.com" in AD_DOMAINS
|
||||
|
||||
|
||||
class TestBlockAdsConfig:
|
||||
"""Test that block_ads merges ad domains into blocked_domains at config level."""
|
||||
|
||||
def test_block_ads_populates_blocked_domains(self):
|
||||
from scrapling.engines._browsers._validators import PlaywrightConfig
|
||||
|
||||
config = PlaywrightConfig(block_ads=True)
|
||||
assert config.blocked_domains is not None
|
||||
assert len(config.blocked_domains) > 1000
|
||||
assert "doubleclick.net" in config.blocked_domains
|
||||
|
||||
def test_block_ads_false_leaves_blocked_domains_none(self):
|
||||
from scrapling.engines._browsers._validators import PlaywrightConfig
|
||||
|
||||
config = PlaywrightConfig(block_ads=False)
|
||||
assert config.blocked_domains is None
|
||||
|
||||
def test_block_ads_merges_with_user_domains(self):
|
||||
from scrapling.engines._browsers._validators import PlaywrightConfig
|
||||
|
||||
user_domains = {"my-custom-block.com"}
|
||||
config = PlaywrightConfig(block_ads=True, blocked_domains=user_domains)
|
||||
assert config.blocked_domains is not None
|
||||
assert "my-custom-block.com" in config.blocked_domains
|
||||
assert "doubleclick.net" in config.blocked_domains
|
||||
|
||||
def test_block_ads_does_not_modify_original_set(self):
|
||||
from scrapling.engines._browsers._validators import PlaywrightConfig
|
||||
|
||||
user_domains = {"my-custom-block.com"}
|
||||
_ = PlaywrightConfig(block_ads=True, blocked_domains=user_domains)
|
||||
assert len(user_domains) == 1
|
||||
|
||||
@@ -321,6 +321,50 @@ def test_selectors_generation(page):
|
||||
_traverse(page)
|
||||
|
||||
|
||||
def test_full_path_selector_no_duplicate_ids():
|
||||
"""Test that full path selectors don't duplicate id segments (regression test)"""
|
||||
html = '<html><body><div id="main"><p id="target">Hello</p></div></body></html>'
|
||||
page = Selector(html)
|
||||
target = page.css("#target").first
|
||||
|
||||
# CSS full path should not duplicate id selectors
|
||||
css_full = target.generate_full_css_selector
|
||||
assert css_full.count("#target") == 1, f"Duplicate #target in CSS full path: {css_full}"
|
||||
assert css_full.count("#main") == 1, f"Duplicate #main in CSS full path: {css_full}"
|
||||
|
||||
# XPath full path should not duplicate id selectors
|
||||
xpath_full = target.generate_full_xpath_selector
|
||||
assert xpath_full.count("@id='target'") == 1, f"Duplicate @id='target' in XPath full path: {xpath_full}"
|
||||
assert xpath_full.count("@id='main'") == 1, f"Duplicate @id='main' in XPath full path: {xpath_full}"
|
||||
|
||||
# The generated CSS selector should actually select the correct element
|
||||
result = page.css(css_full)
|
||||
assert len(result) == 1
|
||||
assert result.first.text == "Hello"
|
||||
|
||||
# The generated XPath selector should also select the correct element
|
||||
result = page.xpath(xpath_full)
|
||||
assert len(result) == 1, f"XPath '{xpath_full}' selected {len(result)} elements, expected 1"
|
||||
assert result.first.text == "Hello"
|
||||
|
||||
|
||||
def test_full_path_selector_mixed_id_and_no_id():
|
||||
"""Test full path selectors with a mix of elements with and without ids"""
|
||||
html = '<html><body><div id="wrapper"><section><p>Text</p></section></div></body></html>'
|
||||
page = Selector(html)
|
||||
target = page.css("p").first
|
||||
|
||||
css_full = target.generate_full_css_selector
|
||||
# p has no id, so it should appear as a tag name; div has id
|
||||
assert "#wrapper" in css_full
|
||||
assert css_full.count("#wrapper") == 1
|
||||
|
||||
# Verify the selector works
|
||||
result = page.css(css_full)
|
||||
assert len(result) == 1
|
||||
assert result.first.text == "Text"
|
||||
|
||||
|
||||
# Miscellaneous Tests
|
||||
def test_getting_all_text(page):
|
||||
"""Test getting all text from the page"""
|
||||
|
||||
@@ -0,0 +1,228 @@
|
||||
"""Tests for the ResponseCacheManager and development_mode integration."""
|
||||
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
import anyio
|
||||
import pytest
|
||||
|
||||
from scrapling.spiders.cache import ResponseCacheManager
|
||||
from scrapling.spiders.engine import CrawlerEngine
|
||||
from scrapling.spiders.request import Request
|
||||
from scrapling.spiders.session import SessionManager
|
||||
from scrapling.engines.toolbelt.custom import Response
|
||||
from scrapling.core._types import Any, Dict, Set, AsyncGenerator
|
||||
|
||||
|
||||
def _make_response(url: str = "https://example.com", body: bytes = b"<html>hello</html>", status: int = 200) -> Response:
|
||||
return Response(
|
||||
url=url,
|
||||
content=body,
|
||||
status=status,
|
||||
reason="OK",
|
||||
encoding="utf-8",
|
||||
cookies={},
|
||||
headers={"content-type": "text/html"},
|
||||
request_headers={"user-agent": "test"},
|
||||
method="GET",
|
||||
)
|
||||
|
||||
|
||||
class TestResponseCacheManager:
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_put_get_roundtrip(self):
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
cache = ResponseCacheManager(tmpdir)
|
||||
fp = b"\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10\x11\x12\x13\x14"
|
||||
original = _make_response(body=b"<html>test content</html>")
|
||||
|
||||
await cache.put(fp, original, "GET")
|
||||
restored = await cache.get(fp)
|
||||
|
||||
assert restored is not None
|
||||
assert restored.url == original.url
|
||||
assert restored.body == original.body
|
||||
assert restored.status == original.status
|
||||
assert restored.reason == original.reason
|
||||
assert restored.encoding == original.encoding
|
||||
assert dict(restored.headers) == dict(original.headers)
|
||||
assert dict(restored.request_headers) == dict(original.request_headers)
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_get_cache_miss(self):
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
cache = ResponseCacheManager(tmpdir)
|
||||
result = await cache.get(b"\x00" * 20)
|
||||
assert result is None
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_get_corrupt_file(self):
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
cache = ResponseCacheManager(tmpdir)
|
||||
fp = b"\xaa" * 20
|
||||
corrupt_path = Path(tmpdir) / f"{fp.hex()}.json"
|
||||
corrupt_path.write_text("not valid json{{{")
|
||||
|
||||
result = await cache.get(fp)
|
||||
assert result is None
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_clear(self):
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
cache = ResponseCacheManager(tmpdir)
|
||||
fp1 = b"\x01" * 20
|
||||
fp2 = b"\x02" * 20
|
||||
|
||||
await cache.put(fp1, _make_response(url="https://a.com"), "GET")
|
||||
await cache.put(fp2, _make_response(url="https://b.com"), "GET")
|
||||
|
||||
assert await cache.get(fp1) is not None
|
||||
assert await cache.get(fp2) is not None
|
||||
|
||||
await cache.clear()
|
||||
|
||||
assert await cache.get(fp1) is None
|
||||
assert await cache.get(fp2) is None
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_creates_cache_dir(self):
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
nested = Path(tmpdir) / "sub" / "dir"
|
||||
cache = ResponseCacheManager(str(nested))
|
||||
await cache.put(b"\x03" * 20, _make_response(), "GET")
|
||||
assert nested.exists()
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_preserves_binary_body(self):
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
cache = ResponseCacheManager(tmpdir)
|
||||
fp = b"\x04" * 20
|
||||
binary_body = bytes(range(256))
|
||||
await cache.put(fp, _make_response(body=binary_body), "GET")
|
||||
restored = await cache.get(fp)
|
||||
assert restored is not None
|
||||
assert restored.body == binary_body
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Integration tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class MockSession:
|
||||
def __init__(self):
|
||||
self._is_alive = False
|
||||
self.fetch_count = 0
|
||||
|
||||
async def __aenter__(self):
|
||||
self._is_alive = True
|
||||
return self
|
||||
|
||||
async def __aexit__(self, *args):
|
||||
self._is_alive = False
|
||||
|
||||
async def fetch(self, url: str, **kwargs):
|
||||
self.fetch_count += 1
|
||||
return _make_response(url=url, body=b"<html>fetched</html>")
|
||||
|
||||
|
||||
class _LogCounterStub:
|
||||
def get_counts(self) -> Dict[str, int]:
|
||||
return {"debug": 0, "info": 0, "warning": 0, "error": 0, "critical": 0}
|
||||
|
||||
|
||||
class MockSpider:
|
||||
def __init__(self, cache_dir: str):
|
||||
self.concurrent_requests = 4
|
||||
self.concurrent_requests_per_domain = 0
|
||||
self.download_delay = 0.0
|
||||
self.max_blocked_retries = 3
|
||||
self.allowed_domains: Set[str] = set()
|
||||
self.fp_include_kwargs = False
|
||||
self.fp_include_headers = False
|
||||
self.fp_keep_fragments = False
|
||||
self.robots_txt_obey = False
|
||||
self.development_mode = True
|
||||
self.development_cache_dir = cache_dir
|
||||
self.start_urls: list[str] = []
|
||||
self.name = "test_cache_spider"
|
||||
self._log_counter = _LogCounterStub()
|
||||
self.scraped_items: list[dict] = []
|
||||
|
||||
async def parse(self, response) -> AsyncGenerator[Dict[str, Any] | Request | None, None]:
|
||||
yield {"url": str(response)}
|
||||
|
||||
async def on_start(self, resuming: bool = False) -> None:
|
||||
pass
|
||||
|
||||
async def on_close(self) -> None:
|
||||
pass
|
||||
|
||||
async def on_error(self, request: Request, error: Exception) -> None:
|
||||
pass
|
||||
|
||||
async def on_scraped_item(self, item: Dict[str, Any]) -> Dict[str, Any] | None:
|
||||
self.scraped_items.append(item)
|
||||
return item
|
||||
|
||||
async def is_blocked(self, response) -> bool:
|
||||
return False
|
||||
|
||||
async def retry_blocked_request(self, request: Request, response) -> Request:
|
||||
return request
|
||||
|
||||
async def start_requests(self) -> AsyncGenerator[Request, None]:
|
||||
yield Request("https://example.com/page1", sid="default")
|
||||
|
||||
|
||||
class TestDevelopmentModeIntegration:
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_first_run_fetches_and_caches(self):
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
session = MockSession()
|
||||
spider = MockSpider(cache_dir=tmpdir)
|
||||
sm = SessionManager()
|
||||
sm.add("default", session)
|
||||
engine = CrawlerEngine(spider, sm)
|
||||
|
||||
await engine.crawl()
|
||||
|
||||
assert session.fetch_count == 1
|
||||
assert engine.stats.cache_misses == 1
|
||||
assert engine.stats.cache_hits == 0
|
||||
assert engine.stats.items_scraped == 1
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_second_run_uses_cache(self):
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
session = MockSession()
|
||||
spider = MockSpider(cache_dir=tmpdir)
|
||||
sm = SessionManager()
|
||||
sm.add("default", session)
|
||||
engine = CrawlerEngine(spider, sm)
|
||||
|
||||
await engine.crawl()
|
||||
assert session.fetch_count == 1
|
||||
|
||||
session2 = MockSession()
|
||||
spider2 = MockSpider(cache_dir=tmpdir)
|
||||
sm2 = SessionManager()
|
||||
sm2.add("default", session2)
|
||||
engine2 = CrawlerEngine(spider2, sm2)
|
||||
|
||||
await engine2.crawl()
|
||||
assert session2.fetch_count == 0
|
||||
assert engine2.stats.cache_hits == 1
|
||||
assert engine2.stats.cache_misses == 0
|
||||
assert engine2.stats.items_scraped == 1
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_disabled_by_default(self):
|
||||
spider = MockSpider(cache_dir="unused")
|
||||
spider.development_mode = False
|
||||
sm = SessionManager()
|
||||
sm.add("default", MockSession())
|
||||
engine = CrawlerEngine(spider, sm)
|
||||
assert engine._cache_manager is None
|
||||
@@ -8,6 +8,7 @@ import pytest
|
||||
|
||||
from scrapling.spiders.engine import CrawlerEngine, _dump
|
||||
from scrapling.spiders.request import Request
|
||||
from scrapling.spiders.robotstxt import RobotsTxtManager
|
||||
from scrapling.spiders.session import SessionManager
|
||||
from scrapling.spiders.result import CrawlStats, ItemList
|
||||
from scrapling.spiders.checkpoint import CheckpointData
|
||||
@@ -22,10 +23,11 @@ from scrapling.core._types import Any, Dict, Set, AsyncGenerator
|
||||
class MockResponse:
|
||||
"""Minimal Response stand-in."""
|
||||
|
||||
def __init__(self, status: int = 200, body: bytes = b"ok", url: str = "https://example.com"):
|
||||
def __init__(self, status: int = 200, body: bytes = b"ok", url: str = "https://example.com", encoding: str = "utf-8"):
|
||||
self.status = status
|
||||
self.body = body
|
||||
self.url = url
|
||||
self.encoding = encoding
|
||||
self.request: Any = None
|
||||
self.meta: Dict[str, Any] = {}
|
||||
|
||||
@@ -83,6 +85,8 @@ class MockSpider:
|
||||
is_blocked_fn=None,
|
||||
on_scraped_item_fn=None,
|
||||
retry_blocked_request_fn=None,
|
||||
robots_txt_obey: bool = False,
|
||||
start_urls: list[str] | None = None,
|
||||
):
|
||||
self.concurrent_requests = concurrent_requests
|
||||
self.concurrent_requests_per_domain = concurrent_requests_per_domain
|
||||
@@ -93,6 +97,10 @@ class MockSpider:
|
||||
self.fp_include_headers = fp_include_headers
|
||||
self.fp_keep_fragments = fp_keep_fragments
|
||||
self.name = "test_spider"
|
||||
self.robots_txt_obey = robots_txt_obey
|
||||
self.development_mode = False
|
||||
self.development_cache_dir = None
|
||||
self.start_urls = start_urls or []
|
||||
|
||||
# Tracking lists
|
||||
self.on_start_calls: list[dict] = []
|
||||
@@ -912,3 +920,71 @@ class TestPauseDuringCrawl:
|
||||
await engine.crawl()
|
||||
|
||||
assert engine.paused is False
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: _prefetch_robots_txt
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestPrefetchRobotsTxt:
|
||||
"""_prefetch_robots_txt warms the robots.txt cache before the crawl loop."""
|
||||
|
||||
@staticmethod
|
||||
def _make_counting_fetch():
|
||||
"""Return (fetch_fn, calls_list) where calls_list records every (url, sid) pair."""
|
||||
calls: list[tuple[str, str]] = []
|
||||
|
||||
async def _fetch(url: str, sid: str):
|
||||
calls.append((url, sid))
|
||||
return MockResponse(status=200, body=b"", url=url)
|
||||
|
||||
return _fetch, calls
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_prefetch_uses_start_urls(self):
|
||||
fetch_fn, calls = self._make_counting_fetch()
|
||||
spider = MockSpider(robots_txt_obey=True, start_urls=["https://example.com/page1"])
|
||||
engine = _make_engine(spider=spider)
|
||||
engine._robots_manager = RobotsTxtManager(fetch_fn)
|
||||
|
||||
await engine._prefetch_robots_txt()
|
||||
|
||||
assert len(calls) == 1
|
||||
assert calls[0][0] == "https://example.com/robots.txt"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_prefetch_noop_when_robots_disabled(self):
|
||||
fetch_fn, calls = self._make_counting_fetch()
|
||||
spider = MockSpider(robots_txt_obey=False)
|
||||
engine = _make_engine(spider=spider)
|
||||
|
||||
assert engine._robots_manager is None
|
||||
|
||||
await engine._prefetch_robots_txt()
|
||||
|
||||
assert calls == []
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_prefetch_noop_when_start_urls_empty(self):
|
||||
fetch_fn, calls = self._make_counting_fetch()
|
||||
spider = MockSpider(robots_txt_obey=True, start_urls=[])
|
||||
engine = _make_engine(spider=spider)
|
||||
engine._robots_manager = RobotsTxtManager(fetch_fn)
|
||||
|
||||
await engine._prefetch_robots_txt()
|
||||
|
||||
assert calls == []
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_prefetch_deduplicates_same_domain_in_start_urls(self):
|
||||
fetch_fn, calls = self._make_counting_fetch()
|
||||
spider = MockSpider(robots_txt_obey=True, start_urls=["https://example.com/a", "https://example.com/b"])
|
||||
engine = _make_engine(spider=spider)
|
||||
engine._robots_manager = RobotsTxtManager(fetch_fn)
|
||||
|
||||
await engine._prefetch_robots_txt()
|
||||
|
||||
# set of Request.domain values deduplicates to one task per domain
|
||||
assert len(calls) == 1
|
||||
assert calls[0][0] == "https://example.com/robots.txt"
|
||||
|
||||
@@ -0,0 +1,282 @@
|
||||
"""Tests for force-stop checkpoint preservation in CrawlerEngine.
|
||||
|
||||
Regression tests for the bug where force-stop (second Ctrl+C) called
|
||||
cancel_scope.cancel() BEFORE saving the checkpoint, causing:
|
||||
1. _save_checkpoint() to be aborted by anyio's Cancelled exception
|
||||
2. self.paused never set to True
|
||||
3. The finally block to DELETE the previous checkpoint (cleanup runs on non-paused exit)
|
||||
|
||||
Total progress loss: user's checkpoint from a long crawl is irrecoverably deleted.
|
||||
"""
|
||||
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
import anyio
|
||||
import pytest
|
||||
|
||||
from scrapling.spiders.engine import CrawlerEngine
|
||||
from scrapling.spiders.request import Request
|
||||
from scrapling.spiders.session import SessionManager
|
||||
from scrapling.spiders.checkpoint import CheckpointManager, CheckpointData
|
||||
from scrapling.core._types import Any, Dict, Set, AsyncGenerator
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Mock helpers (minimal, matching test_engine.py conventions)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class MockResponse:
|
||||
def __init__(self, status=200, body=b"ok", url="https://example.com"):
|
||||
self.status = status
|
||||
self.body = body
|
||||
self.url = url
|
||||
self.request: Any = None
|
||||
self.meta: Dict[str, Any] = {}
|
||||
|
||||
def __str__(self):
|
||||
return self.url
|
||||
|
||||
|
||||
class MockSession:
|
||||
def __init__(self, delay: float = 0.0):
|
||||
self._is_alive = False
|
||||
self._delay = delay
|
||||
|
||||
async def __aenter__(self):
|
||||
self._is_alive = True
|
||||
return self
|
||||
|
||||
async def __aexit__(self, *args):
|
||||
self._is_alive = False
|
||||
|
||||
async def fetch(self, url: str, **kwargs):
|
||||
if self._delay:
|
||||
await anyio.sleep(self._delay)
|
||||
resp = MockResponse(url=url)
|
||||
return resp
|
||||
|
||||
|
||||
class _LogCounterStub:
|
||||
def get_counts(self):
|
||||
return {"debug": 0, "info": 0, "warning": 0, "error": 0, "critical": 0}
|
||||
|
||||
|
||||
class SlowSpider:
|
||||
"""Spider with slow-responding requests to simulate in-flight tasks during force-stop."""
|
||||
|
||||
def __init__(self, num_urls: int = 10):
|
||||
self.concurrent_requests = 4
|
||||
self.concurrent_requests_per_domain = 0
|
||||
self.download_delay = 0.0
|
||||
self.max_blocked_retries = 3
|
||||
self.allowed_domains = set()
|
||||
self.fp_include_kwargs = False
|
||||
self.fp_include_headers = False
|
||||
self.fp_keep_fragments = False
|
||||
self.robots_txt_obey = False
|
||||
self.development_mode = False
|
||||
self.development_cache_dir = None
|
||||
self.start_urls = []
|
||||
self.name = "slow_spider"
|
||||
self._log_counter = _LogCounterStub()
|
||||
self._num_urls = num_urls
|
||||
self.on_start_calls = []
|
||||
self.on_close_calls = 0
|
||||
|
||||
async def parse(self, response) -> AsyncGenerator[Dict[str, Any] | Request | None, None]:
|
||||
yield {"url": str(response)}
|
||||
|
||||
async def on_start(self, resuming=False):
|
||||
self.on_start_calls.append({"resuming": resuming})
|
||||
|
||||
async def on_close(self):
|
||||
self.on_close_calls += 1
|
||||
|
||||
async def on_error(self, request, error):
|
||||
pass
|
||||
|
||||
async def on_scraped_item(self, item):
|
||||
return item
|
||||
|
||||
async def is_blocked(self, response):
|
||||
return False
|
||||
|
||||
async def retry_blocked_request(self, request, response):
|
||||
return request
|
||||
|
||||
async def start_requests(self) -> AsyncGenerator[Request, None]:
|
||||
for i in range(self._num_urls):
|
||||
yield Request(f"https://example.com/page/{i}", sid="default")
|
||||
|
||||
|
||||
def _make_engine(spider=None, session=None, crawldir=None, interval=300.0):
|
||||
spider = spider or SlowSpider()
|
||||
sm = SessionManager()
|
||||
sm.add("default", session or MockSession())
|
||||
return CrawlerEngine(spider, sm, crawldir=crawldir, interval=interval)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestForceStopCheckpointPreservation:
|
||||
"""Verify checkpoint is saved BEFORE cancel_scope.cancel() on force-stop."""
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_force_stop_saves_checkpoint_before_cancel(self):
|
||||
"""Core regression test: force-stop must save checkpoint, not delete it."""
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
spider = SlowSpider(num_urls=20)
|
||||
# Use a slow session so tasks are in-flight when we force-stop
|
||||
session = MockSession(delay=0.5)
|
||||
engine = _make_engine(spider, session, crawldir=tmpdir, interval=0)
|
||||
|
||||
checkpoint_path = Path(tmpdir) / "checkpoint.pkl"
|
||||
|
||||
async def force_stop_after_delay():
|
||||
"""Simulate two rapid Ctrl+C presses."""
|
||||
# Wait for some tasks to start
|
||||
await anyio.sleep(0.1)
|
||||
engine.request_pause() # First Ctrl+C
|
||||
await anyio.sleep(0.05)
|
||||
engine.request_pause() # Second Ctrl+C (force stop)
|
||||
|
||||
async with anyio.create_task_group() as tg:
|
||||
tg.start_soon(force_stop_after_delay)
|
||||
await engine.crawl()
|
||||
|
||||
# The checkpoint file MUST exist after force-stop
|
||||
assert checkpoint_path.exists(), (
|
||||
"Checkpoint file was not saved (or was deleted) after force-stop. "
|
||||
"This means the cancel_scope.cancel() ran before _save_checkpoint()."
|
||||
)
|
||||
# Engine must report as paused
|
||||
assert engine.paused is True
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_graceful_pause_still_saves_checkpoint(self):
|
||||
"""Single Ctrl+C (graceful pause) should save checkpoint as before."""
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
spider = SlowSpider(num_urls=5)
|
||||
session = MockSession(delay=0.3)
|
||||
engine = _make_engine(spider, session, crawldir=tmpdir, interval=0)
|
||||
|
||||
checkpoint_path = Path(tmpdir) / "checkpoint.pkl"
|
||||
|
||||
async def pause_after_delay():
|
||||
await anyio.sleep(0.1)
|
||||
engine.request_pause()
|
||||
|
||||
async with anyio.create_task_group() as tg:
|
||||
tg.start_soon(pause_after_delay)
|
||||
await engine.crawl()
|
||||
|
||||
assert checkpoint_path.exists(), "Checkpoint not saved on graceful pause"
|
||||
assert engine.paused is True
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_force_stop_checkpoint_is_loadable(self):
|
||||
"""Checkpoint saved during force-stop must be valid and loadable."""
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
spider = SlowSpider(num_urls=15)
|
||||
session = MockSession(delay=0.4)
|
||||
engine = _make_engine(spider, session, crawldir=tmpdir, interval=0)
|
||||
|
||||
async def force_stop():
|
||||
await anyio.sleep(0.1)
|
||||
engine.request_pause()
|
||||
await anyio.sleep(0.05)
|
||||
engine.request_pause()
|
||||
|
||||
async with anyio.create_task_group() as tg:
|
||||
tg.start_soon(force_stop)
|
||||
await engine.crawl()
|
||||
|
||||
# Load the checkpoint and verify it's valid
|
||||
manager = CheckpointManager(tmpdir)
|
||||
data = await manager.load()
|
||||
assert data is not None, "Checkpoint data could not be loaded"
|
||||
assert isinstance(data, CheckpointData)
|
||||
# seen set should have some entries (requests were enqueued)
|
||||
assert len(data.seen) > 0
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_normal_completion_cleans_up_checkpoint(self):
|
||||
"""Normal completion (no pause) should still clean up checkpoint files."""
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
spider = SlowSpider(num_urls=2)
|
||||
session = MockSession(delay=0.0)
|
||||
engine = _make_engine(spider, session, crawldir=tmpdir, interval=0)
|
||||
|
||||
await engine.crawl()
|
||||
|
||||
checkpoint_path = Path(tmpdir) / "checkpoint.pkl"
|
||||
# No pause → checkpoint should be cleaned up
|
||||
assert not checkpoint_path.exists()
|
||||
assert engine.paused is False
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_force_stop_without_checkpoint_system(self):
|
||||
"""Force-stop without crawldir should not crash."""
|
||||
spider = SlowSpider(num_urls=10)
|
||||
session = MockSession(delay=0.3)
|
||||
engine = _make_engine(spider, session, crawldir=None)
|
||||
|
||||
async def force_stop():
|
||||
await anyio.sleep(0.1)
|
||||
engine.request_pause()
|
||||
await anyio.sleep(0.05)
|
||||
engine.request_pause()
|
||||
|
||||
async with anyio.create_task_group() as tg:
|
||||
tg.start_soon(force_stop)
|
||||
await engine.crawl()
|
||||
|
||||
# Should not crash and should not be marked as paused
|
||||
# (no checkpoint system = no pause state)
|
||||
assert engine.paused is False
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_force_stop_preserves_existing_checkpoint(self):
|
||||
"""If a checkpoint already exists, force-stop must not delete it."""
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
# First run: do a graceful pause to create a checkpoint
|
||||
spider1 = SlowSpider(num_urls=10)
|
||||
session1 = MockSession(delay=0.2)
|
||||
engine1 = _make_engine(spider1, session1, crawldir=tmpdir, interval=0)
|
||||
|
||||
async def pause1():
|
||||
await anyio.sleep(0.1)
|
||||
engine1.request_pause()
|
||||
|
||||
async with anyio.create_task_group() as tg:
|
||||
tg.start_soon(pause1)
|
||||
await engine1.crawl()
|
||||
|
||||
checkpoint_path = Path(tmpdir) / "checkpoint.pkl"
|
||||
assert checkpoint_path.exists(), "First run should create checkpoint"
|
||||
first_checkpoint_size = checkpoint_path.stat().st_size
|
||||
|
||||
# Second run: force-stop (the fix ensures checkpoint is updated, not deleted)
|
||||
spider2 = SlowSpider(num_urls=10)
|
||||
session2 = MockSession(delay=0.3)
|
||||
engine2 = _make_engine(spider2, session2, crawldir=tmpdir, interval=0)
|
||||
|
||||
async def force_stop2():
|
||||
await anyio.sleep(0.1)
|
||||
engine2.request_pause()
|
||||
await anyio.sleep(0.05)
|
||||
engine2.request_pause()
|
||||
|
||||
async with anyio.create_task_group() as tg:
|
||||
tg.start_soon(force_stop2)
|
||||
await engine2.crawl()
|
||||
|
||||
# Checkpoint must still exist (updated, not deleted)
|
||||
assert checkpoint_path.exists(), (
|
||||
"Force-stop deleted the checkpoint instead of preserving it"
|
||||
)
|
||||
@@ -0,0 +1,506 @@
|
||||
"""Tests for RobotsTxtManager."""
|
||||
|
||||
import asyncio
|
||||
|
||||
import pytest
|
||||
|
||||
from scrapling.spiders.robotstxt import RobotsTxtManager
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fixtures and helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class MockResponse:
|
||||
"""Minimal response stub matching the shape _get_parser expects."""
|
||||
|
||||
def __init__(self, status: int = 200, body: bytes = b"", encoding: str = "utf-8"):
|
||||
self.status = status
|
||||
self.body = body
|
||||
self.encoding = encoding
|
||||
|
||||
|
||||
def make_fetch_fn(status: int = 200, content: str = "", encoding: str = "utf-8"):
|
||||
"""Return an async fetch callable that returns a fixed response.
|
||||
|
||||
Attaches a `.calls` list so tests can assert how many times it was invoked
|
||||
and with which arguments.
|
||||
"""
|
||||
calls: list[tuple] = []
|
||||
|
||||
async def _fetch(url: str, sid: str) -> MockResponse:
|
||||
calls.append((url, sid))
|
||||
return MockResponse(status=status, body=content.encode(encoding), encoding=encoding)
|
||||
|
||||
_fetch.calls = calls # type: ignore[attr-defined]
|
||||
return _fetch
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Shared robots.txt fixtures
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
ROBOTS_BASIC = """\
|
||||
User-agent: *
|
||||
Disallow: /admin/
|
||||
Crawl-delay: 2
|
||||
"""
|
||||
|
||||
ROBOTS_WITH_RATE = """\
|
||||
User-agent: *
|
||||
Request-rate: 1/10
|
||||
Disallow: /private/
|
||||
"""
|
||||
|
||||
ROBOTS_ALLOW_OVERRIDE = """\
|
||||
User-agent: *
|
||||
Disallow: /secret/
|
||||
Allow: /secret/public.html
|
||||
"""
|
||||
|
||||
ROBOTS_DISALLOW_ALL = """\
|
||||
User-agent: *
|
||||
Disallow: /
|
||||
"""
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: can_fetch
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestCanFetch:
|
||||
@pytest.mark.asyncio
|
||||
async def test_allowed_url_returns_true(self):
|
||||
mgr = RobotsTxtManager(make_fetch_fn(content=ROBOTS_BASIC))
|
||||
|
||||
assert await mgr.can_fetch("https://example.com/products", "s1") is True
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_disallowed_url_returns_false(self):
|
||||
mgr = RobotsTxtManager(make_fetch_fn(content=ROBOTS_BASIC))
|
||||
|
||||
assert await mgr.can_fetch("https://example.com/admin/", "s1") is False
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_disallowed_subpath_returns_false(self):
|
||||
mgr = RobotsTxtManager(make_fetch_fn(content=ROBOTS_BASIC))
|
||||
|
||||
assert await mgr.can_fetch("https://example.com/admin/users", "s1") is False
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_root_url_is_allowed(self):
|
||||
mgr = RobotsTxtManager(make_fetch_fn(content=ROBOTS_BASIC))
|
||||
|
||||
assert await mgr.can_fetch("https://example.com/", "s1") is True
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_allow_directive_overrides_disallow(self):
|
||||
mgr = RobotsTxtManager(make_fetch_fn(content=ROBOTS_ALLOW_OVERRIDE))
|
||||
|
||||
assert await mgr.can_fetch("https://example.com/secret/public.html", "s1") is True
|
||||
assert await mgr.can_fetch("https://example.com/secret/private.html", "s1") is False
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_disallow_all_blocks_every_path(self):
|
||||
mgr = RobotsTxtManager(make_fetch_fn(content=ROBOTS_DISALLOW_ALL))
|
||||
|
||||
assert await mgr.can_fetch("https://example.com/", "s1") is False
|
||||
assert await mgr.can_fetch("https://example.com/page", "s1") is False
|
||||
assert await mgr.can_fetch("https://example.com/a/b/c", "s1") is False
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_empty_robots_allows_everything(self):
|
||||
mgr = RobotsTxtManager(make_fetch_fn(content=""))
|
||||
|
||||
assert await mgr.can_fetch("https://example.com/anything", "s1") is True
|
||||
assert await mgr.can_fetch("https://example.com/admin/secret", "s1") is True
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_non_200_response_allows_everything(self):
|
||||
for status in [403, 404, 500, 503]:
|
||||
mgr = RobotsTxtManager(make_fetch_fn(status=status))
|
||||
result = await mgr.can_fetch("https://example.com/page", "s1")
|
||||
assert result is True, f"Expected True for HTTP {status}"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fetch_error_allows_everything(self):
|
||||
async def failing_fetch(url: str, sid: str) -> MockResponse:
|
||||
raise ConnectionError("network failure")
|
||||
|
||||
mgr = RobotsTxtManager(failing_fetch)
|
||||
|
||||
assert await mgr.can_fetch("https://example.com/page", "s1") is True
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_wildcard_path_pattern(self):
|
||||
content = "User-agent: *\nDisallow: /*.pdf$"
|
||||
mgr = RobotsTxtManager(make_fetch_fn(content=content))
|
||||
|
||||
assert await mgr.can_fetch("https://example.com/report.pdf", "s1") is False
|
||||
assert await mgr.can_fetch("https://example.com/report.html", "s1") is True
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_returns_bool(self):
|
||||
mgr = RobotsTxtManager(make_fetch_fn(content=ROBOTS_BASIC))
|
||||
result = await mgr.can_fetch("https://example.com/", "s1")
|
||||
assert isinstance(result, bool)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: get_delay_directives
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestGetDelayDirectives:
|
||||
@pytest.mark.asyncio
|
||||
async def test_returns_crawl_delay_when_set(self):
|
||||
mgr = RobotsTxtManager(make_fetch_fn(content=ROBOTS_BASIC))
|
||||
|
||||
c_delay, r_rate = await mgr.get_delay_directives("https://example.com/", "s1")
|
||||
|
||||
assert c_delay == 2.0
|
||||
assert isinstance(c_delay, float)
|
||||
assert r_rate is None
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_returns_request_rate_when_set(self):
|
||||
mgr = RobotsTxtManager(make_fetch_fn(content=ROBOTS_WITH_RATE))
|
||||
|
||||
c_delay, r_rate = await mgr.get_delay_directives("https://example.com/", "s1")
|
||||
|
||||
assert c_delay is None
|
||||
assert r_rate is not None
|
||||
assert r_rate == (1, 1)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_returns_both_none_when_not_set(self):
|
||||
content = "User-agent: *\nDisallow: /admin/"
|
||||
mgr = RobotsTxtManager(make_fetch_fn(content=content))
|
||||
|
||||
c_delay, r_rate = await mgr.get_delay_directives("https://example.com/", "s1")
|
||||
|
||||
assert c_delay is None
|
||||
assert r_rate is None
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_returns_both_none_for_empty_robots(self):
|
||||
mgr = RobotsTxtManager(make_fetch_fn(content=""))
|
||||
|
||||
c_delay, r_rate = await mgr.get_delay_directives("https://example.com/", "s1")
|
||||
|
||||
assert c_delay is None
|
||||
assert r_rate is None
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_returns_both_none_on_fetch_error(self):
|
||||
async def failing_fetch(url: str, sid: str) -> MockResponse:
|
||||
raise ConnectionError("network failure")
|
||||
|
||||
mgr = RobotsTxtManager(failing_fetch)
|
||||
|
||||
c_delay, r_rate = await mgr.get_delay_directives("https://example.com/", "s1")
|
||||
|
||||
assert c_delay is None
|
||||
assert r_rate is None
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fractional_crawl_delay(self):
|
||||
content = "User-agent: *\nCrawl-delay: 0.5"
|
||||
mgr = RobotsTxtManager(make_fetch_fn(content=content))
|
||||
|
||||
c_delay, _ = await mgr.get_delay_directives("https://example.com/", "s1")
|
||||
|
||||
assert c_delay == 0.5
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_url_path_does_not_affect_result(self):
|
||||
mgr = RobotsTxtManager(make_fetch_fn(content=ROBOTS_BASIC))
|
||||
|
||||
r1 = await mgr.get_delay_directives("https://example.com/", "s1")
|
||||
r2 = await mgr.get_delay_directives("https://example.com/deep/path/page.html", "s1")
|
||||
|
||||
assert r1 == r2
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: caching behaviour
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestCachingBehaviour:
|
||||
@pytest.mark.asyncio
|
||||
async def test_second_call_same_domain_uses_cache(self):
|
||||
fetch_fn = make_fetch_fn(content=ROBOTS_BASIC)
|
||||
mgr = RobotsTxtManager(fetch_fn)
|
||||
|
||||
await mgr.can_fetch("https://example.com/page1", "s1")
|
||||
await mgr.can_fetch("https://example.com/page2", "s1")
|
||||
|
||||
assert len(fetch_fn.calls) == 1
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_all_methods_share_cache(self):
|
||||
fetch_fn = make_fetch_fn(content=ROBOTS_BASIC)
|
||||
mgr = RobotsTxtManager(fetch_fn)
|
||||
|
||||
await mgr.can_fetch("https://example.com/", "s1")
|
||||
await mgr.get_delay_directives("https://example.com/", "s1")
|
||||
|
||||
assert len(fetch_fn.calls) == 1
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_different_sids_share_cache_entry(self):
|
||||
"""robots.txt is domain-level — different sessions share the same cached parser."""
|
||||
fetch_fn = make_fetch_fn(content=ROBOTS_BASIC)
|
||||
mgr = RobotsTxtManager(fetch_fn)
|
||||
|
||||
await mgr.can_fetch("https://example.com/", "s1")
|
||||
await mgr.can_fetch("https://example.com/", "s2")
|
||||
|
||||
assert len(fetch_fn.calls) == 1
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_different_domains_use_separate_cache_entries(self):
|
||||
fetch_fn = make_fetch_fn(content=ROBOTS_BASIC)
|
||||
mgr = RobotsTxtManager(fetch_fn)
|
||||
|
||||
await mgr.can_fetch("https://example.com/", "s1")
|
||||
await mgr.can_fetch("https://other.com/", "s1")
|
||||
|
||||
assert len(fetch_fn.calls) == 2
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cache_keyed_by_domain_not_path(self):
|
||||
fetch_fn = make_fetch_fn(content=ROBOTS_BASIC)
|
||||
mgr = RobotsTxtManager(fetch_fn)
|
||||
|
||||
await mgr.can_fetch("https://example.com/a/b/c", "s1")
|
||||
await mgr.can_fetch("https://example.com/x/y/z", "s1")
|
||||
await mgr.can_fetch("https://example.com/admin/", "s1")
|
||||
|
||||
assert len(fetch_fn.calls) == 1
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sid_is_passed_to_fetch_fn(self):
|
||||
fetch_fn = make_fetch_fn(content=ROBOTS_BASIC)
|
||||
mgr = RobotsTxtManager(fetch_fn)
|
||||
|
||||
await mgr.can_fetch("https://example.com/", "my_session")
|
||||
|
||||
_, received_sid = fetch_fn.calls[0]
|
||||
assert received_sid == "my_session"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: robots.txt URL construction
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestRobotsTxtUrlConstruction:
|
||||
@pytest.mark.asyncio
|
||||
async def test_http_scheme_preserved(self):
|
||||
fetch_fn = make_fetch_fn(content="")
|
||||
mgr = RobotsTxtManager(fetch_fn)
|
||||
|
||||
await mgr.can_fetch("http://example.com/page", "s1")
|
||||
|
||||
fetched_url, _ = fetch_fn.calls[0]
|
||||
assert fetched_url == "http://example.com/robots.txt"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_https_scheme_preserved(self):
|
||||
fetch_fn = make_fetch_fn(content="")
|
||||
mgr = RobotsTxtManager(fetch_fn)
|
||||
|
||||
await mgr.can_fetch("https://example.com/page", "s1")
|
||||
|
||||
fetched_url, _ = fetch_fn.calls[0]
|
||||
assert fetched_url == "https://example.com/robots.txt"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fetched_at_domain_root_regardless_of_request_path(self):
|
||||
fetch_fn = make_fetch_fn(content="")
|
||||
mgr = RobotsTxtManager(fetch_fn)
|
||||
|
||||
await mgr.can_fetch("https://example.com/deep/nested/path/page.html", "s1")
|
||||
|
||||
fetched_url, _ = fetch_fn.calls[0]
|
||||
assert fetched_url == "https://example.com/robots.txt"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_port_included_in_url(self):
|
||||
fetch_fn = make_fetch_fn(content="")
|
||||
mgr = RobotsTxtManager(fetch_fn)
|
||||
|
||||
await mgr.can_fetch("http://example.com:8080/page", "s1")
|
||||
|
||||
fetched_url, _ = fetch_fn.calls[0]
|
||||
assert fetched_url == "http://example.com:8080/robots.txt"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_different_ports_treated_as_different_domains(self):
|
||||
fetch_fn = make_fetch_fn(content="")
|
||||
mgr = RobotsTxtManager(fetch_fn)
|
||||
|
||||
await mgr.can_fetch("http://example.com:8000/page", "s1")
|
||||
await mgr.can_fetch("http://example.com:9000/page", "s1")
|
||||
|
||||
assert len(fetch_fn.calls) == 2
|
||||
urls = [call[0] for call in fetch_fn.calls]
|
||||
assert "http://example.com:8000/robots.txt" in urls
|
||||
assert "http://example.com:9000/robots.txt" in urls
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: encoding
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestEncoding:
|
||||
@pytest.mark.asyncio
|
||||
async def test_non_utf8_body_decoded_with_response_encoding(self):
|
||||
content = "User-agent: *\nDisallow: /admin/\nCrawl-delay: 3"
|
||||
body = content.encode("latin-1")
|
||||
|
||||
async def fetch_fn(url: str, sid: str) -> MockResponse:
|
||||
return MockResponse(status=200, body=body, encoding="latin-1")
|
||||
|
||||
mgr = RobotsTxtManager(fetch_fn)
|
||||
c_delay, _ = await mgr.get_delay_directives("https://example.com/", "s1")
|
||||
|
||||
assert c_delay == 3.0
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bytes_body_decoded_correctly(self):
|
||||
content = "User-agent: *\nDisallow: /private/"
|
||||
body = content.encode("utf-8")
|
||||
|
||||
async def fetch_fn(url: str, sid: str) -> MockResponse:
|
||||
return MockResponse(status=200, body=body, encoding="utf-8")
|
||||
|
||||
mgr = RobotsTxtManager(fetch_fn)
|
||||
|
||||
assert await mgr.can_fetch("https://example.com/private/", "s1") is False
|
||||
assert await mgr.can_fetch("https://example.com/public/", "s1") is True
|
||||
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: concurrent access
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestCacheAndConcurrency:
|
||||
@pytest.mark.asyncio
|
||||
async def test_cached_domain_not_refetched(self):
|
||||
"""Once a domain is cached, subsequent calls return the cached parser without fetching."""
|
||||
fetch_count = 0
|
||||
|
||||
async def counting_fetch(url: str, sid: str) -> MockResponse:
|
||||
nonlocal fetch_count
|
||||
fetch_count += 1
|
||||
return MockResponse(status=200, body=ROBOTS_BASIC.encode(), encoding="utf-8")
|
||||
|
||||
mgr = RobotsTxtManager(counting_fetch)
|
||||
|
||||
# First call fetches and caches
|
||||
await mgr.can_fetch("https://example.com/page1", "s1")
|
||||
# Subsequent calls hit the cache
|
||||
for i in range(7):
|
||||
await mgr.can_fetch(f"https://example.com/page{i + 2}", "s1")
|
||||
|
||||
assert fetch_count == 1
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_concurrent_calls_different_domains_fetch_independently(self):
|
||||
fetch_count = 0
|
||||
|
||||
async def slow_fetch(url: str, sid: str) -> MockResponse:
|
||||
nonlocal fetch_count
|
||||
fetch_count += 1
|
||||
await asyncio.sleep(0.01)
|
||||
return MockResponse(status=200, body=b"", encoding="utf-8")
|
||||
|
||||
mgr = RobotsTxtManager(slow_fetch)
|
||||
|
||||
await asyncio.gather(
|
||||
mgr.can_fetch("https://alpha.com/", "s1"),
|
||||
mgr.can_fetch("https://beta.com/", "s1"),
|
||||
mgr.can_fetch("https://gamma.com/", "s1"),
|
||||
)
|
||||
|
||||
assert fetch_count == 3
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_concurrent_calls_consistent_results(self):
|
||||
"""All concurrent callers should see the same allow/disallow result."""
|
||||
mgr = RobotsTxtManager(make_fetch_fn(content=ROBOTS_BASIC))
|
||||
|
||||
results = await asyncio.gather(*[
|
||||
mgr.can_fetch("https://example.com/admin/", "s1")
|
||||
for _ in range(6)
|
||||
])
|
||||
|
||||
assert all(r is False for r in results)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_different_sids_share_cache_after_first_fetch(self):
|
||||
"""After the first fetch, all sessions share the cached parser regardless of sid."""
|
||||
fetch_count = 0
|
||||
|
||||
async def counting_fetch(url: str, sid: str) -> MockResponse:
|
||||
nonlocal fetch_count
|
||||
fetch_count += 1
|
||||
return MockResponse(status=200, body=b"", encoding="utf-8")
|
||||
|
||||
mgr = RobotsTxtManager(counting_fetch)
|
||||
|
||||
# First call fetches and caches
|
||||
await mgr.can_fetch("https://example.com/", "s1")
|
||||
# s2 and s3 hit the cache — no additional fetches
|
||||
await mgr.can_fetch("https://example.com/", "s2")
|
||||
await mgr.can_fetch("https://example.com/", "s3")
|
||||
|
||||
assert fetch_count == 1
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: prefetch
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestPrefetch:
|
||||
@pytest.mark.asyncio
|
||||
async def test_prefetch_fetches_all_domains(self):
|
||||
fetch_fn = make_fetch_fn(content=ROBOTS_BASIC)
|
||||
mgr = RobotsTxtManager(fetch_fn)
|
||||
|
||||
await mgr.prefetch(["https://a.com/", "https://b.com/", "https://c.com/"], "s1")
|
||||
|
||||
assert len(fetch_fn.calls) == 3
|
||||
fetched = {url for url, _ in fetch_fn.calls}
|
||||
assert fetched == {"https://a.com/robots.txt", "https://b.com/robots.txt", "https://c.com/robots.txt"}
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_prefetch_warms_cache_for_subsequent_calls(self):
|
||||
fetch_fn = make_fetch_fn(content=ROBOTS_BASIC)
|
||||
mgr = RobotsTxtManager(fetch_fn)
|
||||
|
||||
await mgr.prefetch(["https://example.com/"], "s1")
|
||||
assert len(fetch_fn.calls) == 1
|
||||
|
||||
# Any subsequent call for the same domain hits the cache
|
||||
await mgr.can_fetch("https://example.com/products", "s1")
|
||||
await mgr.can_fetch("https://example.com/products", "s2")
|
||||
assert len(fetch_fn.calls) == 1
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_prefetch_empty_list_is_noop(self):
|
||||
fetch_fn = make_fetch_fn(content=ROBOTS_BASIC)
|
||||
mgr = RobotsTxtManager(fetch_fn)
|
||||
|
||||
await mgr.prefetch([], "s1")
|
||||
|
||||
assert len(fetch_fn.calls) == 0
|
||||
Reference in New Issue
Block a user