docs: updating http requests page and some corrections
This commit is contained in:
+57
-23
@@ -2,11 +2,11 @@
|
||||
|
||||
The `Fetcher` class provides rapid and lightweight HTTP requests using the high-performance `curl_cffi` library with a lot of stealth capabilities.
|
||||
|
||||
> 💡 **Prerequisites:**
|
||||
>
|
||||
> 1. You’ve completed or read the [Fetchers basics](../fetching/choosing.md) page to understand what the [Response object](../fetching/choosing.md#response-object) is and which fetcher to use.
|
||||
> 2. You’ve completed or read the [Querying elements](../parsing/selection.md) page to understand how to find/extract elements from the [Selector](../parsing/main_classes.md#selector)/[Response](../fetching/choosing.md#response-object) object.
|
||||
> 3. You’ve completed or read the [Main classes](../parsing/main_classes.md) page to know what properties/methods the [Response](../fetching/choosing.md#response-object) class is inheriting from the [Selector](../parsing/main_classes.md#selector) class.
|
||||
!!! success "Prerequisites"
|
||||
|
||||
1. You've completed or read the [Fetchers basics](../fetching/choosing.md) page to understand what the [Response object](../fetching/choosing.md#response-object) is and which fetcher to use.
|
||||
2. You've completed or read the [Querying elements](../parsing/selection.md) page to understand how to find/extract elements from the [Selector](../parsing/main_classes.md#selector)/[Response](../fetching/choosing.md#response-object) object.
|
||||
3. You've completed or read the [Main classes](../parsing/main_classes.md) page to know what properties/methods the [Response](../fetching/choosing.md#response-object) class is inheriting from the [Selector](../parsing/main_classes.md#selector) class.
|
||||
|
||||
## Basic Usage
|
||||
You have one primary way to import this Fetcher, which is the same for all fetchers.
|
||||
@@ -31,6 +31,7 @@ All methods for making requests here share some arguments, so let's discuss them
|
||||
- **proxy**: As the name implies, the proxy for this request is used to route all traffic (HTTP and HTTPS). The format accepted here is `http://username:password@localhost:8030`.
|
||||
- **proxy_auth**: HTTP basic auth for proxy, tuple of (username, password).
|
||||
- **proxies**: Dict of proxies to use. Format: `{"http": proxy_url, "https": proxy_url}`.
|
||||
- **proxy_rotator**: A `ProxyRotator` instance for automatic proxy rotation. Cannot be combined with `proxy` or `proxies`. Import from `scrapling.engines.toolbelt import ProxyRotator`.
|
||||
- **headers**: Headers to include in the request. Can override any header generated by the `stealthy_headers` argument
|
||||
- **max_redirects**: Maximum number of redirects. **Defaults to 30**, use -1 for unlimited.
|
||||
- **verify**: Whether to verify HTTPS certificates. **Defaults to True**.
|
||||
@@ -186,19 +187,51 @@ with FetcherSession(
|
||||
page1 = session.get('https://scrapling.requestcatcher.com/get')
|
||||
page2 = session.post('https://scrapling.requestcatcher.com/post', data={'key': 'value'})
|
||||
page3 = session.get('https://api.github.com/events')
|
||||
|
||||
|
||||
# All requests share the same session and connection pool
|
||||
```
|
||||
|
||||
You can also use a `ProxyRotator` with `FetcherSession` for automatic proxy rotation across requests:
|
||||
|
||||
```python
|
||||
from scrapling.fetchers import FetcherSession
|
||||
from scrapling.engines.toolbelt import ProxyRotator
|
||||
|
||||
rotator = ProxyRotator([
|
||||
'http://proxy1:8080',
|
||||
'http://proxy2:8080',
|
||||
'http://proxy3:8080',
|
||||
])
|
||||
|
||||
with FetcherSession(proxy_rotator=rotator, impersonate='chrome') as session:
|
||||
# Each request automatically uses the next proxy in rotation
|
||||
page1 = session.get('https://example.com/page1')
|
||||
page2 = session.get('https://example.com/page2')
|
||||
|
||||
# You can check which proxy was used via the response metadata
|
||||
print(page1.meta['proxy'])
|
||||
```
|
||||
|
||||
You can also override the session proxy (or rotator) for a specific request by passing `proxy=` directly to the request method:
|
||||
|
||||
```python
|
||||
with FetcherSession(proxy='http://default-proxy:8080') as session:
|
||||
# Uses the session proxy
|
||||
page1 = session.get('https://example.com/page1')
|
||||
|
||||
# Override the proxy for this specific request
|
||||
page2 = session.get('https://example.com/page2', proxy='http://special-proxy:9090')
|
||||
```
|
||||
|
||||
And here's an async example
|
||||
|
||||
```python
|
||||
async with FetcherSession(impersonate='firefox', http3=True) as session:
|
||||
# All standard HTTP methods available
|
||||
response = async session.get('https://example.com')
|
||||
response = async session.post('https://scrapling.requestcatcher.com/post', json={'data': 'value'})
|
||||
response = async session.put('https://scrapling.requestcatcher.com/put', data={'update': 'info'})
|
||||
response = async session.delete('https://scrapling.requestcatcher.com/delete')
|
||||
response = await session.get('https://example.com')
|
||||
response = await session.post('https://scrapling.requestcatcher.com/post', json={'data': 'value'})
|
||||
response = await session.put('https://scrapling.requestcatcher.com/put', data={'update': 'info'})
|
||||
response = await session.delete('https://scrapling.requestcatcher.com/delete')
|
||||
```
|
||||
or better
|
||||
```python
|
||||
@@ -239,11 +272,11 @@ page = Fetcher.get('https://example.com')
|
||||
# Check the status
|
||||
if page.status == 200:
|
||||
# Extract title
|
||||
title = page.css_first('title::text')
|
||||
title = page.css('title::text').get()
|
||||
print(f"Page title: {title}")
|
||||
|
||||
|
||||
# Extract all links
|
||||
links = page.css('a::attr(href)')
|
||||
links = page.css('a::attr(href)').getall()
|
||||
print(f"Found {len(links)} links")
|
||||
```
|
||||
|
||||
@@ -261,9 +294,9 @@ def scrape_products():
|
||||
results = []
|
||||
for product in products:
|
||||
results.append({
|
||||
'title': product.css_first('.title::text'),
|
||||
'price': product.css_first('.price::text').re_first(r'\d+\.\d{2}'),
|
||||
'description': product.css_first('.description::text'),
|
||||
'title': product.css('.title::text').get(),
|
||||
'price': product.css('.price::text').re_first(r'\d+\.\d{2}'),
|
||||
'description': product.css('.description::text').get(),
|
||||
'in_stock': product.has_class('in-stock')
|
||||
})
|
||||
|
||||
@@ -302,8 +335,8 @@ def scrape_all_pages():
|
||||
# Process products
|
||||
for product in products:
|
||||
all_products.append({
|
||||
'name': product.css_first('.name::text'),
|
||||
'price': product.css_first('.price::text')
|
||||
'name': product.css('.name::text').get(),
|
||||
'price': product.css('.price::text').get()
|
||||
})
|
||||
|
||||
# Next page
|
||||
@@ -329,7 +362,7 @@ response = Fetcher.post(
|
||||
# Check login success
|
||||
if response.status == 200:
|
||||
# Extract user info
|
||||
user_name = response.css_first('.user-name::text')
|
||||
user_name = response.css('.user-name::text').get()
|
||||
print(f"Logged in as: {user_name}")
|
||||
```
|
||||
|
||||
@@ -342,7 +375,7 @@ def extract_table():
|
||||
page = Fetcher.get('https://example.com/data')
|
||||
|
||||
# Find table
|
||||
table = page.css_first('table')
|
||||
table = page.css('table')[0]
|
||||
|
||||
# Extract headers
|
||||
headers = [
|
||||
@@ -367,12 +400,13 @@ def extract_menu():
|
||||
page = Fetcher.get('https://example.com')
|
||||
|
||||
# Find navigation
|
||||
nav = page.css_first('nav')
|
||||
nav = page.css('nav')[0]
|
||||
|
||||
menu = {}
|
||||
for item in nav.css('li'):
|
||||
link = item.css_first('a')
|
||||
if link:
|
||||
links = item.css('a')
|
||||
if links:
|
||||
link = links[0]
|
||||
menu[link.text] = {
|
||||
'url': link['href'],
|
||||
'has_submenu': bool(item.css('.submenu'))
|
||||
|
||||
Reference in New Issue
Block a user