diff --git a/docs/fetching/static.md b/docs/fetching/static.md index 22ce625..cc02da5 100644 --- a/docs/fetching/static.md +++ b/docs/fetching/static.md @@ -2,11 +2,11 @@ The `Fetcher` class provides rapid and lightweight HTTP requests using the high-performance `curl_cffi` library with a lot of stealth capabilities. -> 💡 **Prerequisites:** -> -> 1. You’ve completed or read the [Fetchers basics](../fetching/choosing.md) page to understand what the [Response object](../fetching/choosing.md#response-object) is and which fetcher to use. -> 2. You’ve completed or read the [Querying elements](../parsing/selection.md) page to understand how to find/extract elements from the [Selector](../parsing/main_classes.md#selector)/[Response](../fetching/choosing.md#response-object) object. -> 3. You’ve completed or read the [Main classes](../parsing/main_classes.md) page to know what properties/methods the [Response](../fetching/choosing.md#response-object) class is inheriting from the [Selector](../parsing/main_classes.md#selector) class. +!!! success "Prerequisites" + + 1. You've completed or read the [Fetchers basics](../fetching/choosing.md) page to understand what the [Response object](../fetching/choosing.md#response-object) is and which fetcher to use. + 2. You've completed or read the [Querying elements](../parsing/selection.md) page to understand how to find/extract elements from the [Selector](../parsing/main_classes.md#selector)/[Response](../fetching/choosing.md#response-object) object. + 3. You've completed or read the [Main classes](../parsing/main_classes.md) page to know what properties/methods the [Response](../fetching/choosing.md#response-object) class is inheriting from the [Selector](../parsing/main_classes.md#selector) class. ## Basic Usage You have one primary way to import this Fetcher, which is the same for all fetchers. @@ -31,6 +31,7 @@ All methods for making requests here share some arguments, so let's discuss them - **proxy**: As the name implies, the proxy for this request is used to route all traffic (HTTP and HTTPS). The format accepted here is `http://username:password@localhost:8030`. - **proxy_auth**: HTTP basic auth for proxy, tuple of (username, password). - **proxies**: Dict of proxies to use. Format: `{"http": proxy_url, "https": proxy_url}`. +- **proxy_rotator**: A `ProxyRotator` instance for automatic proxy rotation. Cannot be combined with `proxy` or `proxies`. Import from `scrapling.engines.toolbelt import ProxyRotator`. - **headers**: Headers to include in the request. Can override any header generated by the `stealthy_headers` argument - **max_redirects**: Maximum number of redirects. **Defaults to 30**, use -1 for unlimited. - **verify**: Whether to verify HTTPS certificates. **Defaults to True**. @@ -186,19 +187,51 @@ with FetcherSession( page1 = session.get('https://scrapling.requestcatcher.com/get') page2 = session.post('https://scrapling.requestcatcher.com/post', data={'key': 'value'}) page3 = session.get('https://api.github.com/events') - + # All requests share the same session and connection pool ``` +You can also use a `ProxyRotator` with `FetcherSession` for automatic proxy rotation across requests: + +```python +from scrapling.fetchers import FetcherSession +from scrapling.engines.toolbelt import ProxyRotator + +rotator = ProxyRotator([ + 'http://proxy1:8080', + 'http://proxy2:8080', + 'http://proxy3:8080', +]) + +with FetcherSession(proxy_rotator=rotator, impersonate='chrome') as session: + # Each request automatically uses the next proxy in rotation + page1 = session.get('https://example.com/page1') + page2 = session.get('https://example.com/page2') + + # You can check which proxy was used via the response metadata + print(page1.meta['proxy']) +``` + +You can also override the session proxy (or rotator) for a specific request by passing `proxy=` directly to the request method: + +```python +with FetcherSession(proxy='http://default-proxy:8080') as session: + # Uses the session proxy + page1 = session.get('https://example.com/page1') + + # Override the proxy for this specific request + page2 = session.get('https://example.com/page2', proxy='http://special-proxy:9090') +``` + And here's an async example ```python async with FetcherSession(impersonate='firefox', http3=True) as session: # All standard HTTP methods available - response = async session.get('https://example.com') - response = async session.post('https://scrapling.requestcatcher.com/post', json={'data': 'value'}) - response = async session.put('https://scrapling.requestcatcher.com/put', data={'update': 'info'}) - response = async session.delete('https://scrapling.requestcatcher.com/delete') + response = await session.get('https://example.com') + response = await session.post('https://scrapling.requestcatcher.com/post', json={'data': 'value'}) + response = await session.put('https://scrapling.requestcatcher.com/put', data={'update': 'info'}) + response = await session.delete('https://scrapling.requestcatcher.com/delete') ``` or better ```python @@ -239,11 +272,11 @@ page = Fetcher.get('https://example.com') # Check the status if page.status == 200: # Extract title - title = page.css_first('title::text') + title = page.css('title::text').get() print(f"Page title: {title}") - + # Extract all links - links = page.css('a::attr(href)') + links = page.css('a::attr(href)').getall() print(f"Found {len(links)} links") ``` @@ -261,9 +294,9 @@ def scrape_products(): results = [] for product in products: results.append({ - 'title': product.css_first('.title::text'), - 'price': product.css_first('.price::text').re_first(r'\d+\.\d{2}'), - 'description': product.css_first('.description::text'), + 'title': product.css('.title::text').get(), + 'price': product.css('.price::text').re_first(r'\d+\.\d{2}'), + 'description': product.css('.description::text').get(), 'in_stock': product.has_class('in-stock') }) @@ -302,8 +335,8 @@ def scrape_all_pages(): # Process products for product in products: all_products.append({ - 'name': product.css_first('.name::text'), - 'price': product.css_first('.price::text') + 'name': product.css('.name::text').get(), + 'price': product.css('.price::text').get() }) # Next page @@ -329,7 +362,7 @@ response = Fetcher.post( # Check login success if response.status == 200: # Extract user info - user_name = response.css_first('.user-name::text') + user_name = response.css('.user-name::text').get() print(f"Logged in as: {user_name}") ``` @@ -342,7 +375,7 @@ def extract_table(): page = Fetcher.get('https://example.com/data') # Find table - table = page.css_first('table') + table = page.css('table')[0] # Extract headers headers = [ @@ -367,12 +400,13 @@ def extract_menu(): page = Fetcher.get('https://example.com') # Find navigation - nav = page.css_first('nav') + nav = page.css('nav')[0] menu = {} for item in nav.css('li'): - link = item.css_first('a') - if link: + links = item.css('a') + if links: + link = links[0] menu[link.text] = { 'url': link['href'], 'has_submenu': bool(item.css('.submenu'))