docs: improving the code copy-paste experience and use less tokens for the agent skill
This commit is contained in:
Karim shoair
@@ -43,25 +43,24 @@ Once launched, you'll see the Scrapling banner and can immediately start scrapin
|
||||
|
||||
```python
|
||||
# No imports needed - everything is ready!
|
||||
>>> get('https://news.ycombinator.com')
|
||||
get('https://news.ycombinator.com')
|
||||
|
||||
>>> # Explore the page structure
|
||||
>>> page.css('a')[:5] # Look at first 5 links
|
||||
# Explore the page structure
|
||||
page.css('a')[:5] # Look at first 5 links
|
||||
|
||||
>>> # Refine your selectors
|
||||
>>> stories = page.css('.titleline>a')
|
||||
>>> len(stories)
|
||||
30
|
||||
# Refine your selectors
|
||||
stories = page.css('.titleline>a')
|
||||
len(stories) # 30
|
||||
|
||||
>>> # Extract specific data
|
||||
>>> for story in stories[:3]:
|
||||
# Extract specific data
|
||||
for story in stories[:3]:
|
||||
... title = story.text
|
||||
... url = story['href']
|
||||
... print(f"{title}: {url}")
|
||||
|
||||
>>> # Try different approaches
|
||||
>>> titles = page.css('.titleline>a::text') # Direct text extraction
|
||||
>>> urls = page.css('.titleline>a::attr(href)') # Direct attribute extraction
|
||||
# Try different approaches
|
||||
titles = page.css('.titleline>a::text') # Direct text extraction
|
||||
urls = page.css('.titleline>a::attr(href)') # Direct attribute extraction
|
||||
```
|
||||
|
||||
## Built-in Shortcuts
|
||||
@@ -86,12 +85,10 @@ The shell automatically tracks your requests and pages:
|
||||
The `page` and `response` commands are automatically updated with the last fetched page:
|
||||
|
||||
```python
|
||||
>>> get('https://quotes.toscrape.com')
|
||||
>>> # 'page' and 'response' both refer to the last fetched page
|
||||
>>> page.url
|
||||
'https://quotes.toscrape.com'
|
||||
>>> response.status # Same as page.status
|
||||
200
|
||||
get('https://quotes.toscrape.com')
|
||||
# 'page' and 'response' both refer to the last fetched page
|
||||
page.url # 'https://quotes.toscrape.com'
|
||||
response.status # Prints 200; Same as page.status
|
||||
```
|
||||
|
||||
- **Page History**
|
||||
@@ -99,20 +96,17 @@ The shell automatically tracks your requests and pages:
|
||||
The `pages` command keeps track of the last five pages (it's a `Selectors` object):
|
||||
|
||||
```python
|
||||
>>> get('https://site1.com')
|
||||
>>> get('https://site2.com')
|
||||
>>> get('https://site3.com')
|
||||
get('https://site1.com')
|
||||
get('https://site2.com')
|
||||
get('https://site3.com')
|
||||
|
||||
>>> # Access last 5 pages
|
||||
>>> len(pages) # `Selectors` object with `page` history
|
||||
3
|
||||
>>> pages[0].url # First page in history
|
||||
'https://site1.com'
|
||||
>>> pages[-1].url # Most recent page
|
||||
'https://site3.com'
|
||||
# Access last 5 pages
|
||||
len(pages) # `Selectors` object with `page` history -> 3
|
||||
pages[0].url # First page in history -> 'https://site1.com'
|
||||
pages[-1].url # Most recent page -> 'https://site3.com'
|
||||
|
||||
>>> # Work with historical pages
|
||||
>>> for i, old_page in enumerate(pages):
|
||||
# Work with historical pages
|
||||
for i, old_page in enumerate(pages):
|
||||
... print(f"Page {i}: {old_page.url} - {old_page.status}")
|
||||
```
|
||||
|
||||
@@ -123,8 +117,8 @@ The shell automatically tracks your requests and pages:
|
||||
View scraped pages in your browser:
|
||||
|
||||
```python
|
||||
>>> get('https://quotes.toscrape.com')
|
||||
>>> view(page) # Opens the page HTML in your default browser
|
||||
get('https://quotes.toscrape.com')
|
||||
view(page) # Opens the page HTML in your default browser
|
||||
```
|
||||
|
||||
### Curl Command Integration
|
||||
@@ -138,29 +132,24 @@ First, you need to copy a request as a curl command like the following:
|
||||
- **Convert Curl command to Request Object**
|
||||
|
||||
```python
|
||||
>>> curl_cmd = '''curl 'https://scrapling.requestcatcher.com/post' \
|
||||
curl_cmd = '''curl 'https://scrapling.requestcatcher.com/post' \
|
||||
... -X POST \
|
||||
... -H 'Content-Type: application/json' \
|
||||
... -d '{"name": "test", "value": 123}' '''
|
||||
|
||||
>>> request = uncurl(curl_cmd)
|
||||
>>> request.method
|
||||
'post'
|
||||
>>> request.url
|
||||
'https://scrapling.requestcatcher.com/post'
|
||||
>>> request.headers
|
||||
{'Content-Type': 'application/json'}
|
||||
request = uncurl(curl_cmd)
|
||||
request.method # -> 'post'
|
||||
request.url # -> 'https://scrapling.requestcatcher.com/post'
|
||||
request.headers # -> {'Content-Type': 'application/json'}
|
||||
```
|
||||
|
||||
- **Execute Curl Command Directly**
|
||||
|
||||
```python
|
||||
>>> # Convert and execute in one step
|
||||
>>> curl2fetcher(curl_cmd)
|
||||
>>> page.status
|
||||
200
|
||||
>>> page.json()['json']
|
||||
{'name': 'test', 'value': 123}
|
||||
# Convert and execute in one step
|
||||
curl2fetcher(curl_cmd)
|
||||
page.status # -> 200
|
||||
page.json()['json'] # -> {'name': 'test', 'value': 123}
|
||||
```
|
||||
|
||||
### IPython Features
|
||||
@@ -168,17 +157,17 @@ First, you need to copy a request as a curl command like the following:
|
||||
The shell inherits all IPython capabilities:
|
||||
|
||||
```python
|
||||
>>> # Magic commands
|
||||
>>> %time page = get('https://example.com') # Time execution
|
||||
>>> %history # Show command history
|
||||
>>> %save filename.py 1-10 # Save commands 1-10 to file
|
||||
# Magic commands
|
||||
%time page = get('https://example.com') # Time execution
|
||||
%history # Show command history
|
||||
%save filename.py 1-10 # Save commands 1-10 to file
|
||||
|
||||
>>> # Tab completion works everywhere
|
||||
>>> page.c<TAB> # Shows: css, cookies, headers, etc.
|
||||
>>> Fetcher.<TAB> # Shows all Fetcher methods
|
||||
# Tab completion works everywhere
|
||||
page.c<TAB> # Shows: css, cookies, headers, etc.
|
||||
Fetcher.<TAB> # Shows all Fetcher methods
|
||||
|
||||
>>> # Object inspection
|
||||
>>> get? # Show get documentation
|
||||
# Object inspection
|
||||
get? # Show get documentation
|
||||
```
|
||||
|
||||
## Examples
|
||||
@@ -188,23 +177,23 @@ Here are a few examples generated via AI:
|
||||
#### E-commerce Data Collection
|
||||
|
||||
```python
|
||||
>>> # Start with product listing page
|
||||
>>> catalog = get('https://shop.example.com/products')
|
||||
# Start with product listing page
|
||||
catalog = get('https://shop.example.com/products')
|
||||
|
||||
>>> # Find product links
|
||||
>>> product_links = catalog.css('.product-link::attr(href)')
|
||||
>>> print(f"Found {len(product_links)} products")
|
||||
# Find product links
|
||||
product_links = catalog.css('.product-link::attr(href)')
|
||||
print(f"Found {len(product_links)} products")
|
||||
|
||||
>>> # Sample a few products first
|
||||
>>> for link in product_links[:3]:
|
||||
# Sample a few products first
|
||||
for link in product_links[:3]:
|
||||
... product = get(f"https://shop.example.com{link}")
|
||||
... name = product.css('.product-name::text').get('')
|
||||
... price = product.css('.price::text').get('')
|
||||
... print(f"{name}: {price}")
|
||||
|
||||
>>> # Scale up with sessions for efficiency
|
||||
>>> from scrapling.fetchers import FetcherSession
|
||||
>>> with FetcherSession() as session:
|
||||
# Scale up with sessions for efficiency
|
||||
from scrapling.fetchers import FetcherSession
|
||||
with FetcherSession() as session:
|
||||
... products = []
|
||||
... for link in product_links:
|
||||
... product = session.get(f"https://shop.example.com{link}")
|
||||
|
||||
Reference in New Issue
Block a user