docs: updating http requests page and some corrections

This commit is contained in:
Karim shoair
2026-02-10 19:07:09 +02:00
parent f25fe64d3c
commit 8df74dc57b
+57 -23
View File
@@ -2,11 +2,11 @@
The `Fetcher` class provides rapid and lightweight HTTP requests using the high-performance `curl_cffi` library with a lot of stealth capabilities. The `Fetcher` class provides rapid and lightweight HTTP requests using the high-performance `curl_cffi` library with a lot of stealth capabilities.
> 💡 **Prerequisites:** !!! success "Prerequisites"
>
> 1. Youve completed or read the [Fetchers basics](../fetching/choosing.md) page to understand what the [Response object](../fetching/choosing.md#response-object) is and which fetcher to use. 1. You've completed or read the [Fetchers basics](../fetching/choosing.md) page to understand what the [Response object](../fetching/choosing.md#response-object) is and which fetcher to use.
> 2. Youve completed or read the [Querying elements](../parsing/selection.md) page to understand how to find/extract elements from the [Selector](../parsing/main_classes.md#selector)/[Response](../fetching/choosing.md#response-object) object. 2. You've completed or read the [Querying elements](../parsing/selection.md) page to understand how to find/extract elements from the [Selector](../parsing/main_classes.md#selector)/[Response](../fetching/choosing.md#response-object) object.
> 3. Youve completed or read the [Main classes](../parsing/main_classes.md) page to know what properties/methods the [Response](../fetching/choosing.md#response-object) class is inheriting from the [Selector](../parsing/main_classes.md#selector) class. 3. You've completed or read the [Main classes](../parsing/main_classes.md) page to know what properties/methods the [Response](../fetching/choosing.md#response-object) class is inheriting from the [Selector](../parsing/main_classes.md#selector) class.
## Basic Usage ## Basic Usage
You have one primary way to import this Fetcher, which is the same for all fetchers. You have one primary way to import this Fetcher, which is the same for all fetchers.
@@ -31,6 +31,7 @@ All methods for making requests here share some arguments, so let's discuss them
- **proxy**: As the name implies, the proxy for this request is used to route all traffic (HTTP and HTTPS). The format accepted here is `http://username:password@localhost:8030`. - **proxy**: As the name implies, the proxy for this request is used to route all traffic (HTTP and HTTPS). The format accepted here is `http://username:password@localhost:8030`.
- **proxy_auth**: HTTP basic auth for proxy, tuple of (username, password). - **proxy_auth**: HTTP basic auth for proxy, tuple of (username, password).
- **proxies**: Dict of proxies to use. Format: `{"http": proxy_url, "https": proxy_url}`. - **proxies**: Dict of proxies to use. Format: `{"http": proxy_url, "https": proxy_url}`.
- **proxy_rotator**: A `ProxyRotator` instance for automatic proxy rotation. Cannot be combined with `proxy` or `proxies`. Import from `scrapling.engines.toolbelt import ProxyRotator`.
- **headers**: Headers to include in the request. Can override any header generated by the `stealthy_headers` argument - **headers**: Headers to include in the request. Can override any header generated by the `stealthy_headers` argument
- **max_redirects**: Maximum number of redirects. **Defaults to 30**, use -1 for unlimited. - **max_redirects**: Maximum number of redirects. **Defaults to 30**, use -1 for unlimited.
- **verify**: Whether to verify HTTPS certificates. **Defaults to True**. - **verify**: Whether to verify HTTPS certificates. **Defaults to True**.
@@ -186,19 +187,51 @@ with FetcherSession(
page1 = session.get('https://scrapling.requestcatcher.com/get') page1 = session.get('https://scrapling.requestcatcher.com/get')
page2 = session.post('https://scrapling.requestcatcher.com/post', data={'key': 'value'}) page2 = session.post('https://scrapling.requestcatcher.com/post', data={'key': 'value'})
page3 = session.get('https://api.github.com/events') page3 = session.get('https://api.github.com/events')
# All requests share the same session and connection pool # All requests share the same session and connection pool
``` ```
You can also use a `ProxyRotator` with `FetcherSession` for automatic proxy rotation across requests:
```python
from scrapling.fetchers import FetcherSession
from scrapling.engines.toolbelt import ProxyRotator
rotator = ProxyRotator([
'http://proxy1:8080',
'http://proxy2:8080',
'http://proxy3:8080',
])
with FetcherSession(proxy_rotator=rotator, impersonate='chrome') as session:
# Each request automatically uses the next proxy in rotation
page1 = session.get('https://example.com/page1')
page2 = session.get('https://example.com/page2')
# You can check which proxy was used via the response metadata
print(page1.meta['proxy'])
```
You can also override the session proxy (or rotator) for a specific request by passing `proxy=` directly to the request method:
```python
with FetcherSession(proxy='http://default-proxy:8080') as session:
# Uses the session proxy
page1 = session.get('https://example.com/page1')
# Override the proxy for this specific request
page2 = session.get('https://example.com/page2', proxy='http://special-proxy:9090')
```
And here's an async example And here's an async example
```python ```python
async with FetcherSession(impersonate='firefox', http3=True) as session: async with FetcherSession(impersonate='firefox', http3=True) as session:
# All standard HTTP methods available # All standard HTTP methods available
response = async session.get('https://example.com') response = await session.get('https://example.com')
response = async session.post('https://scrapling.requestcatcher.com/post', json={'data': 'value'}) response = await session.post('https://scrapling.requestcatcher.com/post', json={'data': 'value'})
response = async session.put('https://scrapling.requestcatcher.com/put', data={'update': 'info'}) response = await session.put('https://scrapling.requestcatcher.com/put', data={'update': 'info'})
response = async session.delete('https://scrapling.requestcatcher.com/delete') response = await session.delete('https://scrapling.requestcatcher.com/delete')
``` ```
or better or better
```python ```python
@@ -239,11 +272,11 @@ page = Fetcher.get('https://example.com')
# Check the status # Check the status
if page.status == 200: if page.status == 200:
# Extract title # Extract title
title = page.css_first('title::text') title = page.css('title::text').get()
print(f"Page title: {title}") print(f"Page title: {title}")
# Extract all links # Extract all links
links = page.css('a::attr(href)') links = page.css('a::attr(href)').getall()
print(f"Found {len(links)} links") print(f"Found {len(links)} links")
``` ```
@@ -261,9 +294,9 @@ def scrape_products():
results = [] results = []
for product in products: for product in products:
results.append({ results.append({
'title': product.css_first('.title::text'), 'title': product.css('.title::text').get(),
'price': product.css_first('.price::text').re_first(r'\d+\.\d{2}'), 'price': product.css('.price::text').re_first(r'\d+\.\d{2}'),
'description': product.css_first('.description::text'), 'description': product.css('.description::text').get(),
'in_stock': product.has_class('in-stock') 'in_stock': product.has_class('in-stock')
}) })
@@ -302,8 +335,8 @@ def scrape_all_pages():
# Process products # Process products
for product in products: for product in products:
all_products.append({ all_products.append({
'name': product.css_first('.name::text'), 'name': product.css('.name::text').get(),
'price': product.css_first('.price::text') 'price': product.css('.price::text').get()
}) })
# Next page # Next page
@@ -329,7 +362,7 @@ response = Fetcher.post(
# Check login success # Check login success
if response.status == 200: if response.status == 200:
# Extract user info # Extract user info
user_name = response.css_first('.user-name::text') user_name = response.css('.user-name::text').get()
print(f"Logged in as: {user_name}") print(f"Logged in as: {user_name}")
``` ```
@@ -342,7 +375,7 @@ def extract_table():
page = Fetcher.get('https://example.com/data') page = Fetcher.get('https://example.com/data')
# Find table # Find table
table = page.css_first('table') table = page.css('table')[0]
# Extract headers # Extract headers
headers = [ headers = [
@@ -367,12 +400,13 @@ def extract_menu():
page = Fetcher.get('https://example.com') page = Fetcher.get('https://example.com')
# Find navigation # Find navigation
nav = page.css_first('nav') nav = page.css('nav')[0]
menu = {} menu = {}
for item in nav.css('li'): for item in nav.css('li'):
link = item.css_first('a') links = item.css('a')
if link: if links:
link = links[0]
menu[link.text] = { menu[link.text] = {
'url': link['href'], 'url': link['href'],
'has_submenu': bool(item.css('.submenu')) 'has_submenu': bool(item.css('.submenu'))