feat: Upload the library agent skill
This commit is contained in:
Karim shoair
@@ -0,0 +1,26 @@
|
||||
"""
|
||||
Example 1: Python - FetcherSession (persistent HTTP session with Chrome TLS fingerprint)
|
||||
|
||||
Scrapes all 10 pages of quotes.toscrape.com using a single HTTP session.
|
||||
No browser launched — fast and lightweight.
|
||||
|
||||
Best for: static or semi-static sites, APIs, pages that don't require JavaScript.
|
||||
"""
|
||||
|
||||
from scrapling.fetchers import FetcherSession
|
||||
|
||||
all_quotes = []
|
||||
|
||||
with FetcherSession(impersonate="chrome") as session:
|
||||
for i in range(1, 11):
|
||||
page = session.get(
|
||||
f"https://quotes.toscrape.com/page/{i}/",
|
||||
stealthy_headers=True,
|
||||
)
|
||||
quotes = page.css(".quote .text::text").getall()
|
||||
all_quotes.extend(quotes)
|
||||
print(f"Page {i}: {len(quotes)} quotes (status {page.status})")
|
||||
|
||||
print(f"\nTotal: {len(all_quotes)} quotes\n")
|
||||
for i, quote in enumerate(all_quotes, 1):
|
||||
print(f"{i:>3}. {quote}")
|
||||
@@ -0,0 +1,26 @@
|
||||
"""
|
||||
Example 2: Python - DynamicSession (Playwright browser automation, visible)
|
||||
|
||||
Scrapes all 10 pages of quotes.toscrape.com using a persistent browser session.
|
||||
The browser window stays open across all page requests for efficiency.
|
||||
|
||||
Best for: JavaScript-heavy pages, SPAs, sites with dynamic content loading.
|
||||
|
||||
Set headless=True to run the browser hidden.
|
||||
Set disable_resources=True to skip loading images/fonts for a speed boost.
|
||||
"""
|
||||
|
||||
from scrapling.fetchers import DynamicSession
|
||||
|
||||
all_quotes = []
|
||||
|
||||
with DynamicSession(headless=False, disable_resources=True) as session:
|
||||
for i in range(1, 11):
|
||||
page = session.fetch(f"https://quotes.toscrape.com/page/{i}/")
|
||||
quotes = page.css(".quote .text::text").getall()
|
||||
all_quotes.extend(quotes)
|
||||
print(f"Page {i}: {len(quotes)} quotes (status {page.status})")
|
||||
|
||||
print(f"\nTotal: {len(all_quotes)} quotes\n")
|
||||
for i, quote in enumerate(all_quotes, 1):
|
||||
print(f"{i:>3}. {quote}")
|
||||
@@ -0,0 +1,26 @@
|
||||
"""
|
||||
Example 3: Python - StealthySession (Patchright stealth browser, visible)
|
||||
|
||||
Scrapes all 10 pages of quotes.toscrape.com using a persistent stealth browser session.
|
||||
Bypasses anti-bot protections automatically (Cloudflare Turnstile, fingerprinting, etc.).
|
||||
|
||||
Best for: well-protected sites, Cloudflare-gated pages, sites that detect Playwright.
|
||||
|
||||
Set headless=True to run the browser hidden.
|
||||
Add solve_cloudflare=True to auto-solve Cloudflare challenges.
|
||||
"""
|
||||
|
||||
from scrapling.fetchers import StealthySession
|
||||
|
||||
all_quotes = []
|
||||
|
||||
with StealthySession(headless=False) as session:
|
||||
for i in range(1, 11):
|
||||
page = session.fetch(f"https://quotes.toscrape.com/page/{i}/")
|
||||
quotes = page.css(".quote .text::text").getall()
|
||||
all_quotes.extend(quotes)
|
||||
print(f"Page {i}: {len(quotes)} quotes (status {page.status})")
|
||||
|
||||
print(f"\nTotal: {len(all_quotes)} quotes\n")
|
||||
for i, quote in enumerate(all_quotes, 1):
|
||||
print(f"{i:>3}. {quote}")
|
||||
@@ -0,0 +1,58 @@
|
||||
"""
|
||||
Example 4: Python - Spider (auto-crawling framework)
|
||||
|
||||
Scrapes ALL pages of quotes.toscrape.com by following "Next" pagination links
|
||||
automatically. No manual page looping needed.
|
||||
|
||||
The spider yields structured items (text + author + tags) and exports them to JSON.
|
||||
|
||||
Best for: multi-page crawls, full-site scraping, anything needing pagination or
|
||||
link following across many pages.
|
||||
|
||||
Outputs:
|
||||
- Live stats to terminal during crawl
|
||||
- Final crawl stats at the end
|
||||
- quotes.json in the current directory
|
||||
"""
|
||||
|
||||
from scrapling.spiders import Spider, Response
|
||||
|
||||
|
||||
class QuotesSpider(Spider):
|
||||
name = "quotes"
|
||||
start_urls = ["https://quotes.toscrape.com/"]
|
||||
concurrent_requests = 5 # Fetch up to 5 pages at once
|
||||
|
||||
async def parse(self, response: Response):
|
||||
# Extract all quotes on the current page
|
||||
for quote in response.css(".quote"):
|
||||
yield {
|
||||
"text": quote.css(".text::text").get(),
|
||||
"author": quote.css(".author::text").get(),
|
||||
"tags": quote.css(".tags .tag::text").getall(),
|
||||
}
|
||||
|
||||
# Follow the "Next" button to the next page (if it exists)
|
||||
next_page = response.css(".next a")
|
||||
if next_page:
|
||||
yield response.follow(next_page[0].attrib["href"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
result = QuotesSpider().start()
|
||||
|
||||
print(f"\n{'=' * 50}")
|
||||
print(f"Scraped : {result.stats.items_scraped} quotes")
|
||||
print(f"Requests: {result.stats.requests_count}")
|
||||
print(f"Time : {result.stats.elapsed_seconds:.2f}s")
|
||||
print(f"Speed : {result.stats.requests_per_second:.2f} req/s")
|
||||
print(f"{'=' * 50}\n")
|
||||
|
||||
for i, item in enumerate(result.items, 1):
|
||||
print(f"{i:>3}. [{item['author']}] {item['text']}")
|
||||
if item["tags"]:
|
||||
print(f" Tags: {', '.join(item['tags'])}")
|
||||
|
||||
# Export to JSON
|
||||
result.items.to_json("quotes.json", indent=True)
|
||||
print("\nExported to quotes.json")
|
||||
@@ -0,0 +1,45 @@
|
||||
# Scrapling Examples
|
||||
|
||||
These examples scrape [quotes.toscrape.com](https://quotes.toscrape.com) — a safe, purpose-built scraping sandbox — and demonstrate every tool available in Scrapling, from plain HTTP to full browser automation and spiders.
|
||||
|
||||
All examples collect **all 100 quotes across 10 pages**.
|
||||
|
||||
## Quick Start
|
||||
|
||||
Make sure Scrapling is installed:
|
||||
|
||||
```bash
|
||||
pip install "scrapling[all]>=0.4.1"
|
||||
scrapling install --force
|
||||
```
|
||||
|
||||
## Examples
|
||||
|
||||
| File | Tool | Type | Best For |
|
||||
|--------------------------|-------------------|-----------------------------|---------------------------------------|
|
||||
| `01_fetcher_session.py` | `FetcherSession` | Python — persistent HTTP | APIs, fast multi-page scraping |
|
||||
| `02_dynamic_session.py` | `DynamicSession` | Python — browser automation | Dynamic/SPA pages |
|
||||
| `03_stealthy_session.py` | `StealthySession` | Python — stealth browser | Cloudflare, fingerprint bypass |
|
||||
| `04_spider.py` | `Spider` | Python — auto-crawling | Multi-page crawls, full-site scraping |
|
||||
|
||||
## Running
|
||||
|
||||
**Python scripts:**
|
||||
|
||||
```bash
|
||||
python examples/01_fetcher_session.py
|
||||
python examples/02_dynamic_session.py # Opens a visible browser
|
||||
python examples/03_stealthy_session.py # Opens a visible stealth browser
|
||||
python examples/04_spider.py # Auto-crawls all pages, exports quotes.json
|
||||
```
|
||||
|
||||
## Escalation Guide
|
||||
|
||||
Start with the fastest, lightest option and escalate only if needed:
|
||||
|
||||
```
|
||||
get / FetcherSession
|
||||
└─ If JS required → fetch / DynamicSession
|
||||
└─ If blocked → stealthy-fetch / StealthySession
|
||||
└─ If multi-page → Spider
|
||||
```
|
||||
Reference in New Issue
Block a user