freedak/Scrapling
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

v0.3

Browse Source
This commit is contained in:
Karim shoair
2025-09-01 07:26:15 +03:00
committed by GitHub
parent 9dfca4c6f2 812cf2d21b
commit d5aaeb9816
108 changed files with 12421 additions and 3849 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.bandit.yml
+1 -2
View File
@@ -1,9 +1,8 @@
skips:
- B101
- B311
- B320
- B410
- B113 # `Requests call without timeout` these requests are done in the benchmark and examples scripts only
- B403 # We are using pickle for tests only
- B404 # Using subprocess library
- B602 # subprocess call with shell=True identified
- B110 # Try, Except, Pass detected.
.flake8
-3
View File
@@ -1,3 +0,0 @@
[flake8]
ignore = E501, F401
exclude = .git,.venv,__pycache__,docs,.github,build,dist,tests,benchmarks.py
.github/workflows/publish.yml
-33
View File
@@ -1,33 +0,0 @@
name: Publish Python 🐍 distributions 📦 to PyPI
on:
release:
types: [created,published]
jobs:
build-n-publish:
name: Build and publish Python 🐍 distributions 📦 to PyPI
runs-on: ubuntu-latest
environment:
name: PyPI
url: https://pypi.org/p/scrapling
permissions:
id-token: write
steps:
- uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: 3.12
- name: Upgrade pip
run: python3 -m pip install --upgrade pip
- name: Install build
run: python3 -m pip install --upgrade build twine setuptools
- name: Build a binary wheel and a source tarball
run: python3 -m build --sdist --wheel --outdir dist/
- name: Publish distribution 📦 to PyPI
uses: pypa/gh-action-pypi-publish@release/v1
.github/workflows/auto-release.yml → .github/workflows/release-and-publish.yml
+25 -4
View File
@@ -1,5 +1,5 @@
name: Create Release
# Creates a GitHub release when a PR is merged to main, using the PR title as the version (must start with 'v') and PR body as release notes.
name: Create Release and Publish to PyPI
# Creates a GitHub release when a PR is merged to main (using PR title as version and body as release notes), then publishes to PyPI.
on:
pull_request:
@@ -8,11 +8,15 @@ on:
- main
jobs:
create-release:
create-release-and-publish:
if: github.event.pull_request.merged == true
runs-on: ubuntu-latest
environment:
name: PyPI
url: https://pypi.org/p/scrapling
permissions:
contents: write
id-token: write
steps:
- uses: actions/checkout@v4
with:
@@ -50,4 +54,21 @@ jobs:
draft: false
prerelease: false
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: 3.12
- name: Upgrade pip
run: python3 -m pip install --upgrade pip
- name: Install build
run: python3 -m pip install --upgrade build twine setuptools
- name: Build a binary wheel and a source tarball
run: python3 -m build --sdist --wheel --outdir dist/
- name: Publish distribution 📦 to PyPI
uses: pypa/gh-action-pypi-publish@release/v1
.github/workflows/tests.yml
+49 -22
View File
@@ -17,26 +17,22 @@ jobs:
fail-fast: false
matrix:
include:
- python-version: "3.9"
os: ubuntu-latest
env:
TOXENV: py
- python-version: "3.10"
os: ubuntu-latest
os: macos-latest
env:
TOXENV: py
TOXENV: py310
- python-version: "3.11"
os: ubuntu-latest
os: macos-latest
env:
TOXENV: py
TOXENV: py311
- python-version: "3.12"
os: ubuntu-latest
os: macos-latest
env:
TOXENV: py
TOXENV: py312
- python-version: "3.13"
os: ubuntu-latest
os: macos-latest
env:
TOXENV: py
TOXENV: py313
steps:
- uses: actions/checkout@v4
@@ -47,31 +43,62 @@ jobs:
python-version: ${{ matrix.python-version }}
cache: 'pip'
cache-dependency-path: |
setup.py
requirements*.txt
pyproject.toml
tox.ini
- name: Install Camoufox Dependencies
- name: Install all browsers dependencies
run: |
python3 -m pip install --upgrade pip
python3 -m pip install playwright camoufox
python3 -m pip install playwright==1.52.0 rebrowser-playwright==1.52.0 camoufox
- name: Retrieve Playwright browsers from cache if any
id: playwright-cache
uses: actions/cache@v4
with:
path: |
~/.cache/ms-playwright
~/Library/Caches/ms-playwright
~/.ms-playwright
key: ${{ runner.os }}-playwright-${{ hashFiles('pyproject.toml') }}
restore-keys: |
${{ runner.os }}-playwright-
- name: Install Playwright browsers
if: steps.playwright-cache.outputs.cache-hit != 'true'
run: |
python3 -m playwright install chromium
python3 -m playwright install-deps chromium firefox
- name: Retrieve Camoufox browser from cache if any
id: camoufox-cache
uses: actions/cache@v4
with:
path: |
~/.cache/camoufox
~/Library/Caches/camoufox
key: ${{ runner.os }}-camoufox-${{ hashFiles('pyproject.toml') }}
restore-keys: |
${{ runner.os }}-camoufox-
- name: Install Camoufox browser
if: steps.camoufox-cache.outputs.cache-hit != 'true'
run: |
python3 -m camoufox fetch --browserforge
# Cache tox environments
- name: Cache tox environments
uses: actions/cache@v3
uses: actions/cache@v4
with:
path: .tox
# Include python version and os in cache key
key: tox-v1-${{ runner.os }}-py${{ matrix.python-version }}-${{ hashFiles('tox.ini', 'setup.py', 'requirements*.txt') }}
# Include python version and os in the cache key
key: tox-v1-${{ runner.os }}-py${{ matrix.python-version }}-${{ hashFiles('tox.ini', 'pyproject.toml') }}
restore-keys: |
tox-v1-${{ runner.os }}-py${{ matrix.python-version }}-
tox-v1-${{ runner.os }}-
- name: Install tox
run: pip install -U tox
- name: Run tests
env: ${{ matrix.env }}
run: |
pip install -U tox
tox
run: tox
.pre-commit-config.yaml
+10 -9
View File
@@ -1,19 +1,20 @@
repos:
- repo: https://github.com/PyCQA/bandit
rev: 1.8.0
rev: 1.8.3
hooks:
- id: bandit
args: [-r, -c, .bandit.yml]
- repo: https://github.com/PyCQA/flake8
rev: 7.1.1
- repo: https://github.com/astral-sh/ruff-pre-commit
# Ruff version.
rev: v0.11.5
hooks:
- id: flake8
- repo: https://github.com/pycqa/isort
rev: 5.13.2
hooks:
- id: isort
# Run the linter.
- id: ruff
args: [ --fix ]
# Run the formatter.
- id: ruff-format
- repo: https://github.com/netromdk/vermin
rev: v1.6.0
hooks:
- id: vermin
args: ['-t=3.9-', '--violations', '--eval-annotations', '--no-tips']
args: ['-t=3.10-', '--violations', '--eval-annotations', '--no-tips']
README.md
+195 -99
View File
@@ -2,8 +2,7 @@
<br>
<a href="https://scrapling.readthedocs.io/en/latest/" target="_blank"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/poster.png" style="width: 50%; height: 100%;"/></a>
<br>
<i>Easy, effortless Web Scraping as it should be!</i>
<br>
<i><code>Easy, effortless Web Scraping as it should be!</code></i>
</p>
<p align="center">
<a href="https://github.com/D4Vinci/Scrapling/actions/workflows/tests.yml" alt="Tests">
@@ -11,7 +10,7 @@
<a href="https://badge.fury.io/py/Scrapling" alt="PyPI version">
<img alt="PyPI version" src="https://badge.fury.io/py/Scrapling.svg"></a>
<a href="https://pepy.tech/project/scrapling" alt="PyPI Downloads">
<img alt="PyPI Downloads" src="https://static.pepy.tech/badge/scrapling"></a>
<img alt="PyPI Downloads" src="https://static.pepy.tech/personalized-badge/scrapling?period=total&units=INTERNATIONAL_SYSTEM&left_color=GRAY&right_color=GREEN&left_text=Downloads"></a>
<br/>
<a href="https://discord.gg/EMgGbDceNQ" alt="Discord" target="_blank">
<img alt="Discord" src="https://img.shields.io/discord/1360786381042880532?style=social&logo=discord&link=https%3A%2F%2Fdiscord.gg%2FEMgGbDceNQ">
@@ -46,20 +45,22 @@
</a>
</p>
Dealing with failing web scrapers due to anti-bot protections or website changes? Meet Scrapling.
**Stop fighting anti-bot systems. Stop rewriting selectors after every website update.**
Scrapling is a high-performance, intelligent web scraping library for Python that automatically adapts to website changes while significantly outperforming popular alternatives. For both beginners and experts, Scrapling provides powerful features while maintaining simplicity.
Scrapling isn't just another Web Scraping library. It's the first **adaptive** scraping library that learns from website changes and evolves with them. While other libraries break when websites update their structure, Scrapling automatically relocates your elements and keeps your scrapers running.
Built for the modern Web, Scrapling has its own rapid parsing engine and its fetchers to handle all Web Scraping challenges you are facing or will face. Built by Web Scrapers for Web Scrapers and regular users, there's something for everyone.
```python
>> from scrapling.fetchers import Fetcher, AsyncFetcher, StealthyFetcher, PlayWrightFetcher
>> StealthyFetcher.auto_match = True
>> from scrapling.fetchers import Fetcher, AsyncFetcher, StealthyFetcher, DynamicFetcher
>> StealthyFetcher.adaptive = True
# Fetch websites' source under the radar!
>> page = StealthyFetcher.fetch('https://example.com', headless=True, network_idle=True)
>> print(page.status)
200
>> products = page.css('.product', auto_save=True) # Scrape data that survives website design changes!
>> # Later, if the website structure changes, pass `auto_match=True`
>> products = page.css('.product', auto_match=True) # and Scrapling still finds them!
>> # Later, if the website structure changes, pass `adaptive=True`
>> products = page.css('.product', adaptive=True) # and Scrapling still finds them!
```
# Sponsors
@@ -79,148 +80,243 @@ Scrapling is a high-performance, intelligent web scraping library for Python tha
## Key Features
### Fetch websites as you prefer with async support
- **HTTP Requests**: Fast and stealthy HTTP requests with the `Fetcher` class.
- **Dynamic Loading & Automation**: Fetch dynamic websites with the `PlayWrightFetcher` class through your real browser, Scrapling's stealth mode, Playwright's Chrome browser, or [NSTbrowser](https://app.nstbrowser.io/r/1vO5e5)'s browserless!
- **Anti-bot Protections Bypass**: Easily bypass protections with the `StealthyFetcher` and `PlayWrightFetcher` classes.
### Advanced Websites Fetching with Session Support
- **HTTP Requests**: Fast and stealthy HTTP requests with the `Fetcher` class. Can impersonate browsers' TLS fingerprint, headers, and use HTTP3.
- **Dynamic Loading**: Fetch dynamic websites with full browser automation through the `DynamicFetcher` class supporting Playwright's Chromium, real Chrome, and custom stealth mode.
- **Anti-bot Bypass**: Advanced stealth capabilities with `StealthyFetcher` using a modified version of Firefox and fingerprint spoofing. Can bypass all levels of Cloudflare's Turnstile with automation easily.
- **Session Management**: Persistent session support with `FetcherSession`, `StealthySession`, and `DynamicSession` classes for cookie and state management across requests.
- **Async Support**: Complete async support across all fetchers and dedicated async session classes.
### Adaptive Scraping
- 🔄 **Smart Element Tracking**: Relocate elements after website changes using an intelligent similarity system and integrated storage.
- 🎯 **Flexible Selection**: CSS selectors, XPath selectors, filter-based search, text search, regex search, and more.
- 🔍 **Find Similar Elements**: Automatically locate elements similar to the element you found!
- 🧠 **Smart Content Scraping**: Extract data from multiple websites using Scrapling's powerful features without specific selectors.
### Adaptive Scraping & AI Integration
- 🔄 **Smart Element Tracking**: Relocate elements after website changes using intelligent similarity algorithms.
- 🎯 **Smart Flexible Selection**: CSS selectors, XPath selectors, filter-based search, text search, regex search, and more.
- 🔍 **Find Similar Elements**: Automatically locate elements similar to found elements.
- 🤖 **MCP Server to be used with AI**: Built-in MCP server for AI-assisted Web Scraping and data extraction. The MCP server features custom, powerful capabilities that utilize Scrapling to extract targeted content before passing it to the AI (Claude/Cursor/etc), thereby speeding up operations and reducing costs by minimizing token usage.
### High Performance
- 🚀 **Lightning Fast**: Built from the ground up with performance in mind, outperforming most popular Python scraping libraries.
- 🔋 **Memory Efficient**: Optimized data structures for minimal memory footprint.
- ⚡ **Fast JSON serialization**: 10x faster than standard library.
### High-Performance & battle-tested Architecture
- 🚀 **Lightning Fast**: Optimized performance outperforming most Python scraping libraries.
- 🔋 **Memory Efficient**: Optimized data structures and lazy loading for a minimal memory footprint.
- ⚡ **Fast JSON Serialization**: 10x faster than the standard library.
- 🏗️ **Battle tested**: Not only does Scrapling have 92% test coverage and full type hints coverage, but it has been used daily by hundreds of Web Scrapers over the past year.
### Developer Friendly
- 🛠️ **Powerful Navigation API**: Easy DOM traversal in all directions.
- 🧬 **Rich Text Processing**: All strings have built-in regex, cleaning methods, and more. All elements' attributes are optimized dictionaries with added methods that consume less memory than standard dictionaries.
- 📝 **Auto Selectors Generation**: Generate robust short and full CSS/XPath selectors for any element.
- 🔌 **Familiar API**: Similar to Scrapy/BeautifulSoup and the same pseudo-elements used in Scrapy.
- 📘 **Type hints**: Complete type/doc-strings coverage for future-proofing and best autocompletion support.
### Developer/Web Scraper Friendly Experience
- 🎯 **Interactive Web Scraping Shell**: Optional built-in IPython shell with Scrapling integration, shortcuts, and new tools to speed up Web Scraping scripts development, like converting curl requests to Scrapling requests and viewing requests results in your browser.
- 🚀 **Use it directly from the Terminal**: Optionally, you can use Scrapling to scrape a URL without writing a single code!
- 🛠️ **Rich Navigation API**: Advanced DOM traversal with parent, sibling, and child navigation methods.
- 🧬 **Enhanced Text Processing**: Built-in regex, cleaning methods, and optimized string operations.
- 📝 **Auto Selector Generation**: Generate robust CSS/XPath selectors for any element.
- 🔌 **Familiar API**: Similar to Scrapy/BeautifulSoup with the same pseudo-elements used in Scrapy/Parsel.
- 📘 **Complete Type Coverage**: Full type hints for excellent IDE support and code completion.
### New Session Architecture
Scrapling 0.3 introduces a completely revamped session system:
- **Persistent Sessions**: Maintain cookies, headers, and authentication across multiple requests
- **Automatic Session Management**: Smart session lifecycle handling with proper cleanup
- **Session Inheritance**: All fetchers support both one-off requests and persistent session usage
- **Concurrent Session Support**: Run multiple isolated sessions simultaneously
## Getting Started
### Basic Usage
```python
from scrapling.fetchers import Fetcher, StealthyFetcher, DynamicFetcher
from scrapling.fetchers import FetcherSession, StealthySession, DynamicSession
# HTTP requests with session support
with FetcherSession(impersonate='chrome') as session: # Use latest version of Chrome's TLS fingerprint
page = session.get('https://quotes.toscrape.com/', stealthy_headers=True)
quotes = page.css('.quote .text::text')
# Or use one-off requests
page = Fetcher.get('https://quotes.toscrape.com/')
quotes = page.css('.quote .text::text')
# Advanced stealth mode (Keep the browser open until you finish)
with StealthySession(headless=True, solve_cloudflare=True) as session:
page = session.fetch('https://nopecha.com/demo/cloudflare')
data = page.css('#padded_content a')
# Or use one-off request style, it opens the browser for this request, then closes it after finishing
page = StealthyFetcher.fetch('https://nopecha.com/demo/cloudflare')
data = page.css('#padded_content a')
# Full browser automation (Keep the browser open until you finish)
with DynamicSession(headless=True, disable_resources=False, network_idle=True) as session:
page = session.fetch('https://quotes.toscrape.com/')
data = page.xpath('//span[@class="text"]/text()') # XPath selector if you prefer it
# Or use one-off request style, it opens the browser for this request, then closes it after finishing
page = DynamicFetcher.fetch('https://quotes.toscrape.com/')
data = page.css('.quote .text::text')
```
### Advanced Parsing & Navigation
```python
from scrapling.fetchers import Fetcher
# Do HTTP GET request to a web page and create an Adaptor instance
page = Fetcher.get('https://quotes.toscrape.com/', stealthy_headers=True)
# Get all text content from all HTML tags in the page except the `script` and `style` tags
page.get_all_text(ignore_tags=('script', 'style'))
# Rich element selection and navigation
page = Fetcher.get('https://quotes.toscrape.com/')
# Get all quotes elements; any of these methods will return a list of strings directly (TextHandlers)
quotes = page.css('.quote .text::text') # CSS selector
quotes = page.xpath('//span[@class="text"]/text()') # XPath
quotes = page.css('.quote').css('.text::text') # Chained selectors
quotes = [element.text for element in page.css('.quote .text')] # Slower than bulk query above
# Get the first quote element
quote = page.css_first('.quote') # same as page.css('.quote').first or page.css('.quote')[0]
# Tired of selectors? Use find_all/find
# Get all 'div' HTML tags that one of its 'class' values is 'quote'
quotes = page.find_all('div', {'class': 'quote'})
# Get quotes with multiple selection methods
quotes = page.css('.quote') # CSS selector
quotes = page.xpath('//div[@class="quote"]') # XPath
quotes = page.find_all('div', {'class': 'quote'}) # BeautifulSoup-style
# Same as
quotes = page.find_all('div', class_='quote')
quotes = page.find_all(['div'], class_='quote')
quotes = page.find_all(class_='quote') # and so on...
# Find element by text content
quotes = page.find_by_text('quote', tag='div')
# Working with elements
quote.html_content # Get the Inner HTML of this element
quote.prettify() # Prettified version of Inner HTML above
quote.attrib # Get that element's attributes
quote.path # DOM path to element (List of all ancestors from <html> tag till the element itself)
# Advanced navigation
first_quote = page.css_first('.quote')
quote_text = first_quote.css('.text::text')
quote_text = page.css('.quote').css_first('.text::text') # Chained selectors
quote_text = page.css_first('.quote .text').text # Using `css_first` is faster than `css` if you want the first element
author = first_quote.next_sibling.css('.author::text')
parent_container = first_quote.parent
# Element relationships and similarity
similar_elements = first_quote.find_similar()
below_elements = first_quote.below_elements()
```
You can use the parser right away if you don't want to fetch websites like below:
```python
from scrapling.parser import Selector
page = Selector("<html>...</html>")
```
And it works exactly the same!
### Async Session Management Examples
```python
import asyncio
from scrapling.fetchers import FetcherSession, AsyncStealthySession, AsyncDynamicSession
async with FetcherSession(http3=True) as session: # `FetcherSession` is context-aware and can work in both sync/async patterns
page1 = session.get('https://quotes.toscrape.com/')
page2 = session.get('https://quotes.toscrape.com/', impersonate='firefox135')
# Async session usage
async with AsyncStealthySession(max_pages=2) as session:
tasks = []
urls = ['https://example.com/page1', 'https://example.com/page2']
for url in urls:
task = session.fetch(url)
tasks.append(task)
print(session.get_pool_stats()) # Optional - The status of the browser tabs pool (busy/free/error)
results = await asyncio.gather(*tasks)
print(session.get_pool_stats())
```
## CLI & Interactive Shell
Scrapling v0.3 includes a powerful command-line interface:
```bash
# Launch interactive Web Scraping shell
scrapling shell
# Extract pages to a file directly without programming (Extracts the content inside `body` tag by default)
# If the output file ends with `.txt`, then the text content of the target will be extracted.
# If ended with `.md`, it will be a markdown representation of the HTML content, and `.html` will be the HTML content right away.
scrapling extract get 'https://example.com' content.md
scrapling extract get 'https://example.com' content.txt --css-selector '#fromSkipToProducts' --impersonate 'chrome' # All elements matching the CSS selector '#fromSkipToProducts'
scrapling extract fetch 'https://example.com' content.md --css-selector '#fromSkipToProducts' --no-headless
scrapling extract stealthy-fetch 'https://nopecha.com/demo/cloudflare' captchas.html --css-selector '#padded_content a' --solve-cloudflare
```
To keep it simple, all methods can be chained on top of each other!
> [!NOTE]
> Check out the full documentation from [here](https://scrapling.readthedocs.io/en/latest/)
> There are many additional features, but we want to keep this page short, like the MCP server and the interactive Web Scraping Shell. Check out the full documentation [here](https://scrapling.readthedocs.io/en/latest/)
## Parsing Performance
## Performance Benchmarks
Scrapling isn't just powerful - it's also blazing fast. Scrapling implements many best practices, design patterns, and numerous optimizations to save fractions of seconds. All of that while focusing exclusively on parsing HTML documents.
Here are benchmarks comparing Scrapling to popular Python libraries in two tests.
### Text Extraction Speed Test (5000 nested elements).
This test consists of extracting the text content of 5000 nested div elements.
Scrapling isn't just powerful—it's also blazing fast, and version 0.3 delivers exceptional performance improvements across all operations!
### Text Extraction Speed Test (5000 nested elements)
| # | Library | Time (ms) | vs Scrapling |
|---|:-----------------:|:---------:|:------------:|
| 1 | Scrapling | 5.44 | 1.0x |
| 2 | Parsel/Scrapy | 5.53 | 1.017x |
| 3 | Raw Lxml | 6.76 | 1.243x |
| 4 | PyQuery | 21.96 | 4.037x |
| 5 | Selectolax | 67.12 | 12.338x |
| 6 | BS4 with Lxml | 1307.03 | 240.263x |
| 7 | MechanicalSoup | 1322.64 | 243.132x |
| 8 | BS4 with html5lib | 3373.75 | 620.175x |
| 1 | Scrapling | 1.88 | 1.0x |
| 2 | Parsel/Scrapy | 1.96 | 1.043x |
| 3 | Raw Lxml | 2.32 | 1.234x |
| 4 | PyQuery | 20.2 | ~11x |
| 5 | Selectolax | 85.2 | ~45x |
| 6 | MechanicalSoup | 1305.84 | ~695x |
| 7 | BS4 with Lxml | 1307.92 | ~696x |
| 8 | BS4 with html5lib | 3336.28 | ~1775x |
As you see, Scrapling is on par with Scrapy and slightly faster than Lxml, which both libraries are built on top of. These are the closest results to Scrapling. PyQuery is also built on top of Lxml, but Scrapling is four times faster.
### Element Similarity & Text Search Performance
### Extraction By Text Speed Test
Scrapling can find elements based on its text content and find elements similar to these elements. The only known library with these two features, too, is AutoScraper.
So, we compared this to see how fast Scrapling can be in these two tasks compared to AutoScraper.
Here are the results:
Scrapling's adaptive element finding capabilities significantly outperform alternatives:
| Library | Time (ms) | vs Scrapling |
|-------------|:---------:|:------------:|
| Scrapling | 2.51 | 1.0x |
| AutoScraper | 11.41 | 4.546x |
| Scrapling | 2.02 | 1.0x |
| AutoScraper | 10.26 | 5.08x |
Scrapling can find elements with more methods and returns the entire element's `Adaptor` object, not only text like AutoScraper. So, to make this test fair, both libraries will extract an element with text, find similar elements, and then extract the text content for all of them.
As you see, Scrapling is still 4.5 times faster at the same task.
If we made Scrapling extract the elements only without stopping to extract each element's text, we would get speed twice as fast as this, but as I said, to make it fair comparison a bit :smile:
> All benchmarks' results are an average of 100 runs. See our [benchmarks.py](https://github.com/D4Vinci/Scrapling/blob/main/benchmarks.py) for methodology and to run your comparisons.
> All benchmarks represent averages of 100+ runs. See [benchmarks.py](https://github.com/D4Vinci/Scrapling/blob/main/benchmarks.py) for methodology.
## Installation
Scrapling is a breeze to get started with. Starting from version 0.2.9, we require at least Python 3.9 to work.
Scrapling requires Python 3.10 or higher:
```bash
pip3 install scrapling
pip install scrapling
```
Then run this command to install browsers' dependencies needed to use Fetcher classes
#### Fetchers Setup
If you are going to use any of the fetchers or their classes, then install browser dependencies with
```bash
scrapling install
```
If you have any installation issues, please open an issue.
This downloads all browsers with their system dependencies and fingerprint manipulation dependencies.
### Optional Dependencies
- Install the MCP server feature:
```bash
pip install "scrapling[ai]"
```
- Install shell features (Web Scraping shell and the `extract` command):
```bash
pip install "scrapling[shell]"
```
- Install everything:
```bash
pip install "scrapling[all]"
```
## Contributing
Everybody is invited and welcome to contribute to Scrapling. There is a lot to do!
Please read the [contributing file](https://github.com/D4Vinci/Scrapling/blob/main/CONTRIBUTING.md) before doing anything.
We welcome contributions! Please read our [contributing guidelines](https://github.com/D4Vinci/Scrapling/blob/main/CONTRIBUTING.md) before getting started.
## Disclaimer
## Disclaimer for Scrapling Project
> [!CAUTION]
> This library is provided for educational and research purposes only. By using this library, you agree to comply with local and international data scraping and privacy laws. The authors and contributors are not responsible for any misuse of this software. This library should not be used to violate the rights of others, for unethical purposes, or to use data in an unauthorized or illegal manner. Do not use it on any website unless you have permission from the website owner or within their allowed rules, such as the `robots.txt` file.
> This library is provided for educational and research purposes only. By using this library, you agree to comply with local and international data scraping and privacy laws. The authors and contributors are not responsible for any misuse of this software. Always respect website terms of service and robots.txt files.
## License
This work is licensed under BSD-3
This work is licensed under the BSD-3-Clause License.
## Acknowledgments
This project includes code adapted from:
- Parsel (BSD License) - Used for [translator](https://github.com/D4Vinci/Scrapling/blob/main/scrapling/translator.py) submodule
- Parsel (BSD License)—Used for [translator](https://github.com/D4Vinci/Scrapling/blob/main/scrapling/core/translator.py) submodule
## Thanks and References
- [Daijro](https://github.com/daijro)'s brilliant work on both [BrowserForge](https://github.com/daijro/browserforge) and [Camoufox](https://github.com/daijro/camoufox)
- [Vinyzu](https://github.com/Vinyzu)'s work on Playwright's mock on [Botright](https://github.com/Vinyzu/Botright)
- [brotector](https://github.com/kaliiiiiiiiii/brotector)
- [fakebrowser](https://github.com/kkoooqq/fakebrowser)
- [rebrowser-patches](https://github.com/rebrowser/rebrowser-patches)
## Known Issues
- In the auto-matching save process, the unique properties of the first element from the selection results are the only ones that get saved. If the selector you are using selects different elements on the page in different locations, auto-matching will return the first element to you only when you relocate it later. This doesn't include combined CSS selectors (Using commas to combine more than one selector, for example), as these selectors get separated, and each selector gets executed alone.
- [Daijro](https://github.com/daijro)'s brilliant work on [BrowserForge](https://github.com/daijro/browserforge) and [Camoufox](https://github.com/daijro/camoufox)
- [Vinyzu](https://github.com/Vinyzu)'s work on [Botright](https://github.com/Vinyzu/Botright)
- [brotector](https://github.com/kaliiiiiiiiii/brotector) for browser detection bypass techniques
- [fakebrowser](https://github.com/kkoooqq/fakebrowser) for fingerprinting research
- [rebrowser-patches](https://github.com/rebrowser/rebrowser-patches) for stealth improvements
---
<div align="center"><small>Designed & crafted with ❤️ by Karim Shoair.</small></div><br>
ROADMAP.md
+6 -6
View File
@@ -1,14 +1,14 @@
## TODOs
- [x] Add more tests and increase the code coverage.
- [x] Structure the tests folder in a better way.
- [ ] Add more documentation.
- [x] Add more documentation.
- [x] Add the browsing ability.
- [ ] Create detailed documentation for 'readthedocs' website, preferably add Github action for deploying it.
- [x] Create detailed documentation for the 'readthedocs' website, preferably add GitHub action for deploying it.
- [ ] Create a Scrapy plugin/decorator to make it replace parsel in the response argument when needed.
- [ ] Need to add more functionality to `AttributesHandler` and more navigation functions to `Adaptor` object (ex: functions similar to map, filter, and reduce functions but here pass it to the element and the function is executed on children, siblings, next elements, etc...)
- [x] Add `.filter` method to `Adaptors` object and other similar methods.
- [x] Need to add more functionality to `AttributesHandler` and more navigation functions to `Selector` object (ex: functions similar to map, filter, and reduce functions but here pass it to the element and the function is executed on children, siblings, next elements, etc...)
- [x] Add `.filter` method to `Selectors` object and other similar methods.
- [ ] Add functionality to automatically detect pagination URLs
- [ ] Add the ability to auto-detect schemas in pages and manipulate them.
- [ ] Add `analyzer` ability that tries to learn about the page through meta elements and return what it learned
- [ ] Add ability to generate a regex from a group of elements (Like for all href attributes)
- [ ] Add `analyzer` ability that tries to learn about the page through meta-elements and return what it learned
- [ ] Add the ability to generate a regex from a group of elements (Like for all href attributes)
-
benchmarks.py
+36 -30
View File
@@ -12,21 +12,29 @@ from parsel import Selector
from pyquery import PyQuery as pq
from selectolax.parser import HTMLParser
from scrapling import Adaptor
from scrapling import Selector as ScraplingSelector
large_html = '<html><body>' + '<div class="item">' * 5000 + '</div>' * 5000 + '</body></html>'
large_html = (
"<html><body>" + '<div class="item">' * 5000 + "</div>" * 5000 + "</body></html>"
)
def benchmark(func):
@functools.wraps(func)
def wrapper(*args, **kwargs):
benchmark_name = func.__name__.replace('test_', '').replace('_', ' ')
benchmark_name = func.__name__.replace("test_", "").replace("_", " ")
print(f"-> {benchmark_name}", end=" ", flush=True)
# Warm-up phase
timeit.repeat(lambda: func(*args, **kwargs), number=2, repeat=2, globals=globals())
timeit.repeat(
lambda: func(*args, **kwargs), number=2, repeat=2, globals=globals()
)
# Measure time (1 run, repeat 100 times, take average)
times = timeit.repeat(
lambda: func(*args, **kwargs), number=1, repeat=100, globals=globals(), timer=time.process_time
lambda: func(*args, **kwargs),
number=1,
repeat=100,
globals=globals(),
timer=time.process_time,
)
min_time = round(mean(times) * 1000, 2) # Convert to milliseconds
print(f"average execution time: {min_time} ms")
@@ -41,58 +49,59 @@ def test_lxml():
e.text
for e in etree.fromstring(
large_html,
# Scrapling and Parsel use the same parser inside so this is just to make it fair
parser=html.HTMLParser(recover=True, huge_tree=True)
).cssselect('.item')]
# Scrapling and Parsel use the same parser inside, so this is just to make it fair
parser=html.HTMLParser(recover=True, huge_tree=True),
).cssselect(".item")
]
@benchmark
def test_bs4_lxml():
return [e.text for e in BeautifulSoup(large_html, 'lxml').select('.item')]
return [e.text for e in BeautifulSoup(large_html, "lxml").select(".item")]
@benchmark
def test_bs4_html5lib():
return [e.text for e in BeautifulSoup(large_html, 'html5lib').select('.item')]
return [e.text for e in BeautifulSoup(large_html, "html5lib").select(".item")]
@benchmark
def test_pyquery():
return [e.text() for e in pq(large_html)('.item').items()]
return [e.text() for e in pq(large_html)(".item").items()]
@benchmark
def test_scrapling():
# No need to do `.extract()` like parsel to extract text
# Also, this is faster than `[t.text for t in Adaptor(large_html, auto_match=False).css('.item')]`
# Also, this is faster than `[t.text for t in Selector(large_html, adaptive=False).css('.item')]`
# for obvious reasons, of course.
return Adaptor(large_html, auto_match=False).css('.item::text')
return ScraplingSelector(large_html, adaptive=False).css(".item::text")
@benchmark
def test_parsel():
return Selector(text=large_html).css('.item::text').extract()
return Selector(text=large_html).css(".item::text").extract()
@benchmark
def test_mechanicalsoup():
browser = StatefulBrowser()
browser.open_fake_page(large_html)
return [e.text for e in browser.page.select('.item')]
return [e.text for e in browser.page.select(".item")]
@benchmark
def test_selectolax():
return [node.text() for node in HTMLParser(large_html).css('.item')]
return [node.text() for node in HTMLParser(large_html).css(".item")]
def display(results):
# Sort and display results
sorted_results = sorted(results.items(), key=lambda x: x[1]) # Sort by time
scrapling_time = results['Scrapling']
scrapling_time = results["Scrapling"]
print("\nRanked Results (fastest to slowest):")
print(f" i. {'Library tested':<18} | {'avg. time (ms)':<15} | vs Scrapling")
print('-' * 50)
print("-" * 50)
for i, (test_name, test_time) in enumerate(sorted_results, 1):
compare = round(test_time / scrapling_time, 3)
print(f" {i}. {test_name:<18} | {str(test_time):<15} | {compare}")
@@ -100,27 +109,24 @@ def display(results):
@benchmark
def test_scrapling_text(request_html):
# Will loop over resulted elements to get text too to make comparison even more fair otherwise Scrapling will be even faster
return [
element.text for element in Adaptor(
request_html, auto_match=False
).find_by_text('Tipping the Velvet', first_match=True).find_similar(ignore_attributes=['title'])
]
return ScraplingSelector(request_html, adaptive=False).find_by_text("Tipping the Velvet", first_match=True, clean_match=False).find_similar(ignore_attributes=["title"])
@benchmark
def test_autoscraper(request_html):
# autoscraper by default returns elements text
return AutoScraper().build(html=request_html, wanted_list=['Tipping the Velvet'])
return AutoScraper().build(html=request_html, wanted_list=["Tipping the Velvet"])
if __name__ == "__main__":
print(' Benchmark: Speed of parsing and retrieving the text content of 5000 nested elements \n')
print(
" Benchmark: Speed of parsing and retrieving the text content of 5000 nested elements \n"
)
results1 = {
"Raw Lxml": test_lxml(),
"Parsel/Scrapy": test_parsel(),
"Scrapling": test_scrapling(),
'Selectolax': test_selectolax(),
"Selectolax": test_selectolax(),
"PyQuery": test_pyquery(),
"BS4 with Lxml": test_bs4_lxml(),
"MechanicalSoup": test_mechanicalsoup(),
@@ -128,10 +134,10 @@ if __name__ == "__main__":
}
display(results1)
print('\n' + "="*25)
req = requests.get('https://books.toscrape.com/index.html')
print("\n" + "=" * 25)
req = requests.get("https://books.toscrape.com/index.html")
print(
' Benchmark: Speed of searching for an element by text content, and retrieving the text of similar elements\n'
" Benchmark: Speed of searching for an element by text content, and retrieving the text of similar elements\n"
)
results2 = {
"Scrapling": test_scrapling_text(req.text),
cleanup.py
+8 -8
View File
@@ -9,12 +9,12 @@ def clean():
# Directories and patterns to clean
cleanup_patterns = [
'build',
'dist',
'*.egg-info',
'__pycache__',
'.eggs',
'.pytest_cache'
"build",
"dist",
"*.egg-info",
"__pycache__",
".eggs",
".pytest_cache",
]
# Clean directories
@@ -30,7 +30,7 @@ def clean():
print(f"Could not remove {path}: {e}")
# Remove compiled Python files
for path in base_dir.rglob('*.py[co]'):
for path in base_dir.rglob("*.py[co]"):
try:
path.unlink()
print(f"Removed compiled file: {path}")
@@ -38,5 +38,5 @@ def clean():
print(f"Could not remove {path}: {e}")
if __name__ == '__main__':
if __name__ == "__main__":
clean()
docs/ai/mcp-server.md
+253
View File
@@ -0,0 +1,253 @@
# Scrapling MCP Server Guide
The **Scrapling MCP Server** is a new feature that brings Scrapling's powerful Web Scraping capabilities directly to your favorite AI chatbot or AI agent. This integration allows you to scrape websites, extract data, and bypass anti-bot protections conversationally through Claude's AI interface or any other chatbot that supports MCP.
## Features
The Scrapling MCP Server provides six powerful tools for web scraping:
### 🚀 Basic HTTP Scraping
- **`get`**: Fast HTTP requests with browser fingerprint impersonation, generating real browser headers matching the TLS version, HTTP/3, and more!
- **`bulk_get`**: An async version of the above tool that allows scraping of multiple URLs at the same time!
### 🌐 Dynamic Content Scraping
- **`fetch`**: Rapidly fetch dynamic content with Chromium/Chrome browser with complete control over the request/browser, stealth mode, and more!
- **`bulk_fetch`**: An async version of the above tool that allows scraping of multiple URLs in different browser tabs at the same time!
### 🔒 Stealth Scraping
- **`stealthy_fetch`**: Uses our modified version of Camoufox browser to bypass Cloudflare Turnstile and other anti-bot systems with complete control over the request/browser!
- **`bulk_stealthy_fetch`**: An async version of the above tool that allows stealth scraping of multiple URLs in different browser tabs at the same time!
### Key Capabilities
- **Smart Content Extraction**: Convert web pages/elements to Markdown, HTML, or extract a clean version of the text content
- **CSS Selector Support**: Use the Scrapling engine to target specific elements with precision before handing the content to the AI
- **Anti-Bot Bypass**: Handle Cloudflare Turnstile and other protections
- **Proxy Support**: Use proxies for anonymity and geo-targeting
- **Browser Impersonation**: Mimic real browsers with TLS fingerprinting, real browser headers matching that version, and more
- **Parallel Processing**: Scrape multiple URLs concurrently for efficiency
#### But why use Scrapling MCP Server instead of other available tools?
Aside from its stealth capabilities and ability to bypass Cloudflare Turnstile, Scrapling's server is the only one that allows you to pass a CSS selector in the prompt to extract specific elements before handing the content to the AI.
The way other servers work is that they extract the content, then pass it all to the AI to extract the fields you want. This causes the AI to consume a lot more tokens that are not needed (from irrelevant content). Scrapling solves this problem by allowing you to pass a CSS selector to narrow down the content you want before passing it to the AI, which makes the whole process much faster and more efficient.
If you don't know how to write/use CSS selectors, don't worry. You can tell the AI in the prompt to write selectors to match possible fields for you and watch it try different combinations until it finds the right one, as we will show in the examples section.
## Installation
Install Scrapling with MCP Support, then double-check that the browser dependencies are installed.
```bash
# Install Scrapling with MCP server dependencies
pip install "scrapling[ai]"
# Install browser dependencies
scrapling install
```
## Setting up the MCP Server
Here we will explain how to add Scrapling MCP Server to [Claude Desktop](https://claude.ai/download) and [Claude Code](https://www.anthropic.com/claude-code), but the same logic applies to any other chatbot that supports MCP:
### Claude Desktop
1. Open Claude Desktop
2. Click the hamburger menu (☰) at the top left → Settings → Developer → Edit Config
3. Add the Scrapling MCP server configuration:
```json
"ScraplingServer": {
"command": "scrapling",
"args": [
"mcp"
]
}
```
If that's the first MCP server you're adding, set the content of the file to this:
```json
{
"mcpServers": {
"ScraplingServer": {
"command": "scrapling",
"args": [
"mcp"
]
}
}
}
```
As per the [official article](https://modelcontextprotocol.io/quickstart/user), this action creates a new configuration file if one doesn’t exist or opens your existing configuration. The file is located at
1. **MacOS**: `~/Library/Application Support/Claude/claude_desktop_config.json`
2. **Windows**: `%APPDATA%\Claude\claude_desktop_config.json`
To ensure it's working, it's best to use the full path to the `scrapling` executable. Open the terminal and execute the following command:
1. **MacOS**: `which scrapling`
2. **Windows**: `where scrapling`
For me, on my Mac, it returned `/Users/<MyUsername>/.venv/bin/scrapling`, so the config I used in the end is:
```json
{
"mcpServers": {
"ScraplingServer": {
"command": "/Users/<MyUsername>/.venv/bin/scrapling",
"args": [
"mcp"
]
}
}
}
```
The same logic applies to [Cursor](https://docs.cursor.com/en/context/mcp), [WindSurf](https://windsurf.com/university/tutorials/configuring-first-mcp-server), and others.
### Claude Code
Here it's much simpler to do. If you have [Claude Code](https://www.anthropic.com/claude-code) installed, open the terminal and execute the following command:
```bash
claude mcp add ScraplingServer "/Users/<MyUsername>/.venv/bin/scrapling" mcp
```
Same as above, to get Scrapling's executable path, open the terminal and execute the following command:
1. **MacOS**: `which scrapling`
2. **Windows**: `where scrapling`
Here's the main article from Anthropic on [how to add MCP servers to Claude code](https://docs.anthropic.com/en/docs/claude-code/mcp#option-1%3A-add-a-local-stdio-server) for further details.
Then, after you've added the server, you need to completely quit and restart the app you used above. In Claude Desktop, you should see an MCP server indicator (🔧) in the bottom-right corner of the chat input or see `ScraplingServer` in the `Search and tools` dropdown in the chat input box.
## Examples
Now we will show you some examples of prompts we used while testing the MCP server, but you are probably more creative than we are and better at prompt engineering than we are :)
We will gradually go from simple prompts to more complex ones. We will use Claude Desktop for the examples, but the same logic applies to the rest, of course.
1. **Basic Web Scraping**
Extract the main content from a webpage as Markdown:
```
Scrape the main content from https://example.com and convert it to markdown format.
```
Claude will use the `get` tool to fetch the page and return clean, readable content. If it fails, it will continue retrying every second for three attempts, unless you instruct it to do otherwise. If it fails to retrieve content for any reason, such as protection or if it's a dynamic website, it will automatically try the other tools. If Claude didn't do that automatically for some reason, you can add that to the prompt.
A more optimized version of the same prompt would be:
```
Use regular requests to scrape the main content from https://example.com and convert it to markdown format.
```
This tells Claude about the right tool to use here, so it doesn't have to guess. Sometimes it will start using normal requests on its own, and at other times, it will assume browsers are better suited for this website without any apparent reason. As a general rule of thumb, you should always tell Claude what tool to use if you want to save time, money, and get consistent results.
2. **Targeted Data Extraction**
Extract specific elements using CSS selectors:
```
Get all product titles from https://shop.example.com using the CSS selector '.product-title'. If the request fails, retry up to 5 times every 10 seconds.
```
The server will extract only the elements matching your selector and return them as a structured list. Notice I told it to set the tool to only try three times in case the website has connection issues, but the default setting should be fine for most cases.
3. **E-commerce Data Collection**
Another example of a bit more complex prompt:
```
Extract product information from these e-commerce URLs using bulk browser fetches:
- https://shop1.com/product-a
- https://shop2.com/product-b
- https://shop3.com/product-c
Get the product names, prices, and descriptions from each page.
```
Claude will use `bulk_fetch` to scrape all URLs concurrently, then analyze the extracted data.
4. **More advanced workflow**
Let's say I want to get all the action games available on PlayStation's store first page right now. I can use the following prompt to do that:
```
Extract the URLs of all games in this page, then do a bulk request to them and return a list of all action games: https://store.playstation.com/en-us/pages/browse
```
Note that I instructed it to use a bulk request for all the URLs collected. If I hadn't mentioned it, sometimes it works as intended, and other times it makes a separate request to each URL, which takes significantly longer. This prompt takes approximately one minute to complete.
However, because I wasn't specific enough, it actually used the `stealthy_fetch` here and the `bulk_stealthy_fetch` in the second step, which unnecessarily consumed a large number of tokens. A better prompt would be:
```
Use normal requests to extract the URLs of all games in this page, then do a bulk request to them and return a list of all action games: https://store.playstation.com/en-us/pages/browse
```
And if you know how to write CSS selectors, you can instruct Claude to apply the selectors to the elements you want, and it will nearly complete the task immediately.
```
Use normal requests to extract the URLs of all games on the page below, then perform a bulk request to them and return a list of all action games.
The selector for games in the first page is `[href*="/concept/"]` and the selector for the genre in the second request is `[data-qa="gameInfo#releaseInformation#genre-value"]`
URL: https://store.playstation.com/en-us/pages/browse
```
5. **Get data from a website with Cloudflare protection**
If you think the website you are targeting has Cloudflare protection, you should tell Claude instead of letting it discover that on its own.
```
What's the price of this product? Be cautious, as it utilizes Cloudflare's Turnstile protection. Make the browser visible while you work.
https://ao.com/product/oo101uk-ninja-woodfire-outdoor-pizza-oven-brown-99357-685.aspx
```
6. **Long workflow**
You can, for example, use a prompt like this:
```
Extract all the product URLs in the following category, then return the prices and the details of the first three products.
https://www.arnotts.ie/furniture/bedroom/bed-frames/
```
But a better prompt would be:
```
Go to the following category URL and extract all product URLs using the CSS selector "a". Then, fetch the first 3 product pages in parallel and extract each product’s price and details.
Keep the output in markdown format to reduce irrelevant content.
Category URL:
https://www.arnotts.ie/furniture/bedroom/bed-frames/
```
And so on, you get the idea. Your creativity is the key here.
## Best Practices
Here is some technical advice for you.
### 1. Choose the Right Tool
- **`get`**: Fast, simple websites
- **`fetch`**: Sites with JavaScript/dynamic content
- **`stealthy_fetch`**: Protected sites, Cloudflare, anti-bot systems
### 2. Optimize Performance
- Use bulk tools for multiple URLs
- Disable unnecessary resources
- Set appropriate timeouts
- Use CSS selectors for targeted extraction
### 3. Handle Dynamic Content
- Use `network_idle` for SPAs
- Set `wait_selector` for specific elements
- Increase timeout for slow-loading sites
### 4. Data Quality
- Use `main_content_only=true` to avoid navigation/ads
- Choose an appropriate `extraction_type` for your use case
## Legal and Ethical Considerations
⚠️ **Important Guidelines:**
- **Check robots.txt**: Visit `https://website.com/robots.txt` to see scraping rules
- **Respect rate limits**: Don't overwhelm servers with requests
- **Terms of Service**: Read and comply with website terms
- **Copyright**: Respect intellectual property rights
- **Privacy**: Be mindful of personal data protection laws
- **Commercial use**: Ensure you have permission for business purposes
---
*Built with ❤️ by the Scrapling team. Happy scraping!*
docs/api-reference/adaptor.md
-25
View File
@@ -1,25 +0,0 @@
---
search:
exclude: true
---
# Adaptor Class
The `Adaptor` class is the core parsing engine in Scrapling that provides HTML parsing and element selection capabilities.
Here's the reference information for the `Adaptor` class, with all its parameters, attributes, and methods.
You can import the `Adaptor` class directly from `scrapling`:
```python
from scrapling.parser import Adaptor
```
## ::: scrapling.parser.Adaptor
handler: python
:docstring:
## ::: scrapling.parser.Adaptors
handler: python
:docstring:
docs/api-reference/fetchers.md
+35 -2
View File
@@ -10,7 +10,10 @@ Here's the reference information for all fetcher-type classes' parameters, attri
You can import all of them directly like below:
```python
from scrapling.fetchers import Fetcher, AsyncFetcher, StealthyFetcher, PlayWrightFetcher
from scrapling.fetchers import (
Fetcher, AsyncFetcher, StealthyFetcher, DynamicFetcher,
FetcherSession, AsyncStealthySession, StealthySession, DynamicSession, AsyncDynamicSession
)
```
## ::: scrapling.fetchers.Fetcher
@@ -21,10 +24,40 @@ from scrapling.fetchers import Fetcher, AsyncFetcher, StealthyFetcher, PlayWrigh
handler: python
:docstring:
## ::: scrapling.fetchers.PlayWrightFetcher
## ::: scrapling.fetchers.DynamicFetcher
handler: python
:docstring:
## ::: scrapling.fetchers.StealthyFetcher
handler: python
:docstring:
## Session Classes
### HTTP Sessions
## ::: scrapling.fetchers.FetcherSession
handler: python
:docstring:
### Stealth Sessions
## ::: scrapling.fetchers.StealthySession
handler: python
:docstring:
## ::: scrapling.fetchers.AsyncStealthySession
handler: python
:docstring:
### Dynamic Sessions
## ::: scrapling.fetchers.DynamicSession
handler: python
:docstring:
## ::: scrapling.fetchers.AsyncDynamicSession
handler: python
:docstring:
docs/api-reference/mcp-server.md
+39
View File
@@ -0,0 +1,39 @@
---
search:
exclude: true
---
# MCP Server API Reference
The **Scrapling MCP Server** provides six powerful tools for web scraping through the Model Context Protocol (MCP). This server integrates Scrapling's capabilities directly into AI chatbots and agents, allowing conversational web scraping with advanced anti-bot bypass features.
You can start the MCP server by running:
```bash
scrapling mcp
```
Or import the server class directly:
```python
from scrapling.core.ai import ScraplingMCPServer
server = ScraplingMCPServer()
server.serve()
```
## Response Model
The standardized response structure that's returned by all MCP server tools:
## ::: scrapling.core.ai.ResponseModel
handler: python
:docstring:
## MCP Server Class
The main MCP server class that provides all web scraping tools:
## ::: scrapling.core.ai.ScraplingMCPServer
handler: python
:docstring:
docs/api-reference/selector.md
+25
View File
@@ -0,0 +1,25 @@
---
search:
exclude: true
---
# Selector Class
The `Selector` class is the core parsing engine in Scrapling that provides HTML parsing and element selection capabilities.
Here's the reference information for the `Selector` class, with all its parameters, attributes, and methods.
You can import the `Selector` class directly from `scrapling`:
```python
from scrapling.parser import Selector
```
## ::: scrapling.parser.Selector
handler: python
:docstring:
## ::: scrapling.parser.Selectors
handler: python
:docstring:
docs/assets/scrapling_shell_curl.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 527 KiB

docs/benchmarks.md
+16 -33
View File
@@ -1,44 +1,27 @@
Scrapling isn't just powerful - it's also blazing fast. Scrapling implements many best practices, design patterns, and numerous optimizations to save fractions of seconds. All of that while focusing exclusively on parsing HTML documents.
# Performance Benchmarks
Here are benchmarks comparing Scrapling's parsing speed to popular Python libraries in two tests.
Scrapling isn't just powerful—it's also blazing fast, and version 0.3 delivers exceptional performance improvements across all operations!
### Text Extraction Speed Test
This test consists of extracting the text content of 5000 nested div elements.
Here are the results comparing Scrapling to all well-known parsing libraries:
## Benchmark Results
### Text Extraction Speed Test (5000 nested elements)
| # | Library | Time (ms) | vs Scrapling |
|---|:-----------------:|:---------:|:------------:|
| 1 | Scrapling | 5.44 | 1.0x |
| 2 | Parsel/Scrapy | 5.53 | 1.017x |
| 3 | Raw Lxml | 6.76 | 1.243x |
| 4 | PyQuery | 21.96 | 4.037x |
| 5 | Selectolax | 67.12 | 12.338x |
| 6 | BS4 with Lxml | 1307.03 | 240.263x |
| 7 | MechanicalSoup | 1322.64 | 243.132x |
| 8 | BS4 with html5lib | 3373.75 | 620.175x |
| 1 | Scrapling | 1.88 | 1.0x |
| 2 | Parsel/Scrapy | 1.96 | 1.043x |
| 3 | Raw Lxml | 2.32 | 1.234x |
| 4 | PyQuery | 20.2 | ~11x |
| 5 | Selectolax | 85.2 | ~45x |
| 6 | MechanicalSoup | 1305.84 | ~695x |
| 7 | BS4 with Lxml | 1307.92 | ~696x |
| 8 | BS4 with html5lib | 3336.28 | ~1775x |
As you see, Scrapling is on par with Scrapy and slightly faster than Lxml, which both libraries are built on top of. These are the closest results to Scrapling. PyQuery is also built on top of Lxml, but Scrapling is four times faster.
### Element Similarity & Text Search Performance
### Extraction By Text Speed Test
Scrapling can find elements based on its text content and find elements similar to these elements. The only known library with these two features, too, is AutoScraper.
So, we compared this to see how fast Scrapling can be in these two tasks compared to AutoScraper.
Here are the results:
Scrapling's adaptive element finding capabilities significantly outperform alternatives:
| Library | Time (ms) | vs Scrapling |
|-------------|:---------:|:------------:|
| Scrapling | 2.51 | 1.0x |
| AutoScraper | 11.41 | 4.546x |
Scrapling can find elements with more methods and returns the entire element's `Adaptor` object, not only text like AutoScraper. So, to make this test fair, both libraries will extract an element with text, find similar elements, and then extract the text content for all of them.
As you see, Scrapling is still 4.5 times faster at the same task.
If we made Scrapling extract the elements only without stopping to extract each element's text, we would get speed twice as fast as this, but as I said, to make it fair comparison a bit :smile:
> All benchmarks' results are an average of 100 runs. See our [benchmarks.py](https://github.com/D4Vinci/Scrapling/blob/main/benchmarks.py) for methodology and to run your comparisons.
| Scrapling | 2.02 | 1.0x |
| AutoScraper | 10.26 | 5.08x |
docs/cli/extract-commands.md
+348
View File
@@ -0,0 +1,348 @@
# Scrapling Extract Command Guide
**Web Scraping through the terminal without requiring any programming!**
The `scrapling extract` Command lets you download and extract content from websites directly from your terminal without writing any code. Ideal for beginners, researchers, and anyone requiring rapid web data extraction.
## What is the Extract Command group?
The extract command is a set of simple terminal tools that:
- **Downloads web pages** and saves their content to files.
- **Converts HTML to readable formats** like Markdown, keeps it as HTML, or just extracts the text content of the page.
- **Supports custom CSS selectors** to extract specific parts of the page.
- **Handles HTTP requests and fetching through browsers**
- **Highly customizable** with custom headers, cookies, proxies, and the rest of the options. Almost all the options available through the code are also accessible through the command line.
## Quick Start
- **Basic Website Download**
Download a website's text content as clean, readable text:
```bash
scrapling extract get "https://example.com" page_content.txt
```
This does an HTTP GET request and saves the text content of the webpage to `page_content.txt`.
- **Save as Different Formats**
Choose your output format by changing the file extension:
```bash
# Convert the HTML content to Markdown, then save it to the file (great for documentation)
scrapling extract get "https://blog.example.com" article.md
# Save the HTML content as it is to the file
scrapling extract get "https://example.com" page.html
# Save a clean version of the text content of the webpage to the file
scrapling extract get "https://example.com" content.txt
```
- **Extract Specific Content**
All commands can use CSS selectors to extract specific parts of the page through `--css-selector` or `-s` as you will see in the examples below.
## Available Commands
You can display the available commands through `scrapling extract --help` to get the following list:
```bash
Usage: scrapling extract [OPTIONS] COMMAND [ARGS]...
Fetch web pages using various fetchers and extract full/selected HTML content as HTML, Markdown, or extract text content.
Options:
--help Show this message and exit.
Commands:
get Perform a GET request and save the content to a file.
post Perform a POST request and save the content to a file.
put Perform a PUT request and save the content to a file.
delete Perform a DELETE request and save the content to a file.
fetch Use DynamicFetcher to fetch content with browser...
stealthy-fetch Use StealthyFetcher to fetch content with advanced...
```
We will go through each Command in detail below.
### HTTP Requests
1. **GET Request**
The most common Command for downloading website content:
```bash
scrapling extract get [URL] [OUTPUT_FILE] [OPTIONS]
```
**Examples:**
```bash
# Basic download
scrapling extract get "https://news.site.com" news.md
# Download with custom timeout
scrapling extract get "https://example.com" content.txt --timeout 60
# Extract only specific content using CSS selectors
scrapling extract get "https://blog.example.com" articles.md --css-selector "article"
# Send a request with cookies
scrapling extract get "https://scrapling.requestcatcher.com" content.md --cookies "session=abc123; user=john"
# Add user agent
scrapling extract get "https://api.site.com" data.json -H "User-Agent: MyBot 1.0"
# Add multiple headers
scrapling extract get "https://site.com" page.html -H "Accept: text/html" -H "Accept-Language: en-US"
```
Get the available options for the Command with `scrapling extract get --help` as follows:
```bash
Usage: scrapling extract get [OPTIONS] URL OUTPUT_FILE
Perform a GET request and save the content to a file.
The output file path can be an HTML file, a Markdown file of the HTML content, or the text content itself. Use file extensions (`.html`/`.md`/`.txt`) respectively.
Options:
-H, --headers TEXT HTTP headers in format "Key: Value" (can be used multiple times)
--cookies TEXT Cookies string in format "name1=value1;name2=value2"
--timeout INTEGER Request timeout in seconds (default: 30)
--proxy TEXT Proxy URL in format "http://username:password@host:port"
-s, --css-selector TEXT CSS selector to extract specific content from the page. It returns all matches.
-p, --params TEXT Query parameters in format "key=value" (can be used multiple times)
--follow-redirects / --no-follow-redirects Whether to follow redirects (default: True)
--verify / --no-verify Whether to verify SSL certificates (default: True)
--impersonate TEXT Browser to impersonate (e.g., chrome, firefox).
--stealthy-headers / --no-stealthy-headers Use stealthy browser headers (default: True)
--help Show this message and exit.
```
Note that the options will work in the same way for all other request commands, so no need to repeat them.
2. **Post Request**
```bash
scrapling extract post [URL] [OUTPUT_FILE] [OPTIONS]
```
**Examples:**
```bash
# Submit form data
scrapling extract post "https://api.site.com/search" results.html --data "query=python&type=tutorial"
# Send JSON data
scrapling extract post "https://api.site.com" response.json --json '{"username": "test", "action": "search"}'
```
Get the available options for the Command with `scrapling extract post --help` as follows:
```bash
Usage: scrapling extract post [OPTIONS] URL OUTPUT_FILE
Perform a POST request and save the content to a file.
The output file path can be an HTML file, a Markdown file of the HTML content, or the text content itself. Use file extensions (`.html`/`.md`/`.txt`) respectively.
Options:
-d, --data TEXT Form data to include in the request body (as string, ex: "param1=value1&param2=value2")
-j, --json TEXT JSON data to include in the request body (as string)
-H, --headers TEXT HTTP headers in format "Key: Value" (can be used multiple times)
--cookies TEXT Cookies string in format "name1=value1;name2=value2"
--timeout INTEGER Request timeout in seconds (default: 30)
--proxy TEXT Proxy URL in format "http://username:password@host:port"
-s, --css-selector TEXT CSS selector to extract specific content from the page. It returns all matches.
-p, --params TEXT Query parameters in format "key=value" (can be used multiple times)
--follow-redirects / --no-follow-redirects Whether to follow redirects (default: True)
--verify / --no-verify Whether to verify SSL certificates (default: True)
--impersonate TEXT Browser to impersonate (e.g., chrome, firefox).
--stealthy-headers / --no-stealthy-headers Use stealthy browser headers (default: True)
--help Show this message and exit.
```
3. **Put Request**
```bash
scrapling extract put [URL] [OUTPUT_FILE] [OPTIONS]
```
**Examples:**
```bash
# Send data
scrapling extract put "https://scrapling.requestcatcher.com/put" results.html --data "update=info" --impersonate "firefox"
# Send JSON data
scrapling extract put "https://scrapling.requestcatcher.com/put" response.json --json '{"username": "test", "action": "search"}'
```
Get the available options for the Command with `scrapling extract put --help` as follows:
```bash
Usage: scrapling extract put [OPTIONS] URL OUTPUT_FILE
Perform a PUT request and save the content to a file.
The output file path can be an HTML file, a Markdown file of the HTML content, or the text content itself. Use file extensions (`.html`/`.md`/`.txt`) respectively.
Options:
-d, --data TEXT Form data to include in the request body
-j, --json TEXT JSON data to include in the request body (as string)
-H, --headers TEXT HTTP headers in format "Key: Value" (can be used multiple times)
--cookies TEXT Cookies string in format "name1=value1;name2=value2"
--timeout INTEGER Request timeout in seconds (default: 30)
--proxy TEXT Proxy URL in format "http://username:password@host:port"
-s, --css-selector TEXT CSS selector to extract specific content from the page. It returns all matches.
-p, --params TEXT Query parameters in format "key=value" (can be used multiple times)
--follow-redirects / --no-follow-redirects Whether to follow redirects (default: True)
--verify / --no-verify Whether to verify SSL certificates (default: True)
--impersonate TEXT Browser to impersonate (e.g., chrome, firefox).
--stealthy-headers / --no-stealthy-headers Use stealthy browser headers (default: True)
--help Show this message and exit.
```
4. **Delete Request**
```bash
scrapling extract delete [URL] [OUTPUT_FILE] [OPTIONS]
```
**Examples:**
```bash
# Send data
scrapling extract delete "https://scrapling.requestcatcher.com/delete" results.html
# Send JSON data
scrapling extract delete "https://scrapling.requestcatcher.com/" response.txt --impersonate "chrome"
```
Get the available options for the Command with `scrapling extract delete --help` as follows:
```bash
Usage: scrapling extract delete [OPTIONS] URL OUTPUT_FILE
Perform a DELETE request and save the content to a file.
The output file path can be an HTML file, a Markdown file of the HTML content, or the text content itself. Use file extensions (`.html`/`.md`/`.txt`) respectively.
Options:
-H, --headers TEXT HTTP headers in format "Key: Value" (can be used multiple times)
--cookies TEXT Cookies string in format "name1=value1;name2=value2"
--timeout INTEGER Request timeout in seconds (default: 30)
--proxy TEXT Proxy URL in format "http://username:password@host:port"
-s, --css-selector TEXT CSS selector to extract specific content from the page. It returns all matches.
-p, --params TEXT Query parameters in format "key=value" (can be used multiple times)
--follow-redirects / --no-follow-redirects Whether to follow redirects (default: True)
--verify / --no-verify Whether to verify SSL certificates (default: True)
--impersonate TEXT Browser to impersonate (e.g., chrome, firefox).
--stealthy-headers / --no-stealthy-headers Use stealthy browser headers (default: True)
--help Show this message and exit.
```
### Browsers fetching
1. **fetch - Handle Dynamic Content**
For websites that load content with dynamic content or have slight protection
```bash
scrapling extract fetch [URL] [OUTPUT_FILE] [OPTIONS]
```
**Examples:**
```bash
# Wait for JavaScript to load content and finish network activity
scrapling extract fetch "https://scrapling.requestcatcher.com/" content.md --network-idle
# Wait for specific content to appear
scrapling extract fetch "https://scrapling.requestcatcher.com/" data.txt --wait-selector ".content-loaded"
# Run in visible browser mode (helpful for debugging)
scrapling extract fetch "https://scrapling.requestcatcher.com/" page.html --no-headless --disable-resources
```
Get the available options for the Command with `scrapling extract fetch --help` as follows:
```bash
Usage: scrapling extract fetch [OPTIONS] URL OUTPUT_FILE
Use DynamicFetcher to fetch content with browser automation.
The output file path can be an HTML file, a Markdown file of the HTML content, or the text content itself. Use file extensions (`.html`/`.md`/`.txt`) respectively.
Options:
--headless / --no-headless Run browser in headless mode (default: True)
--disable-resources / --enable-resources Drop unnecessary resources for speed boost (default: False)
--network-idle / --no-network-idle Wait for network idle (default: False)
--timeout INTEGER Timeout in milliseconds (default: 30000)
--wait INTEGER Additional wait time in milliseconds after page load (default: 0)
-s, --css-selector TEXT CSS selector to extract specific content from the page. It returns all matches.
--wait-selector TEXT CSS selector to wait for before proceeding
--locale TEXT Browser locale (default: en-US)
--stealth / --no-stealth Enable stealth mode (default: False)
--hide-canvas / --show-canvas Add noise to canvas operations (default: False)
--disable-webgl / --enable-webgl Disable WebGL support (default: False)
--proxy TEXT Proxy URL in format "http://username:password@host:port"
-H, --extra-headers TEXT Extra headers in format "Key: Value" (can be used multiple times)
--help Show this message and exit.
```
2. **stealthy-fetch - Bypass Protection**
For websites with anti-bot protection or Cloudflare protection
```bash
scrapling extract stealthy-fetch [URL] [OUTPUT_FILE] [OPTIONS]
```
**Examples:**
```bash
# Bypass basic protection
scrapling extract stealthy-fetch "https://scrapling.requestcatcher.com" content.md
# Solve Cloudflare challenges
scrapling extract stealthy-fetch "https://nopecha.com/demo/cloudflare" data.txt --solve-cloudflare --css-selector "#padded_content a"
# Use proxy for anonymity
scrapling extract stealthy-fetch "https://site.com" content.md --proxy "http://proxy-server:8080"
```
Get the available options for the Command with `scrapling extract stealthy-fetch --help` as follows:
```bash
Usage: scrapling extract stealthy-fetch [OPTIONS] URL OUTPUT_FILE
Use StealthyFetcher to fetch content with advanced stealth features.
The output file path can be an HTML file, a Markdown file of the HTML content, or the text content itself. Use file extensions (`.html`/`.md`/`.txt`) respectively.
Options:
--headless / --no-headless Run browser in headless mode (default: True)
--block-images / --allow-images Block image loading (default: False)
--disable-resources / --enable-resources Drop unnecessary resources for speed boost (default: False)
--block-webrtc / --allow-webrtc Block WebRTC entirely (default: False)
--humanize / --no-humanize Humanize cursor movement (default: False)
--solve-cloudflare / --no-solve-cloudflare Solve Cloudflare challenges (default: False)
--allow-webgl / --block-webgl Allow WebGL (default: True)
--network-idle / --no-network-idle Wait for network idle (default: False)
--disable-ads / --allow-ads Install uBlock Origin addon (default: False)
--timeout INTEGER Timeout in milliseconds (default: 30000)
--wait INTEGER Additional wait time in milliseconds after page load (default: 0)
-s, --css-selector TEXT CSS selector to extract specific content from the page. It returns all matches.
--wait-selector TEXT CSS selector to wait for before proceeding
--geoip / --no-geoip Use IP/Proxy geolocation for timezone/locale (default: False)
--proxy TEXT Proxy URL in format "http://username:password@host:port"
-H, --extra-headers TEXT Extra headers in format "Key: Value" (can be used multiple times)
--help Show this message and exit.
```
## When to use each Command
If you are not a Web Scraping expert and can't decide what to choose, you can use the following formula to help you decide:
- Use **`get`** with simple websites, blogs, or news articles
- Use **`fetch`** with modern web apps, or sites with dynamic content
- Use **`stealthy-fetch`** with protected sites, Cloudflare, or anti-bot systems
## Legal and Ethical Considerations
⚠️ **Important Guidelines:**
- **Check robots.txt**: Visit `https://website.com/robots.txt` to see scraping rules
- **Respect rate limits**: Don't overwhelm servers with requests
- **Terms of Service**: Read and comply with website terms
- **Copyright**: Respect intellectual property rights
- **Privacy**: Be mindful of personal data protection laws
- **Commercial use**: Ensure you have permission for business purposes
---
*Happy scraping! Remember to always respect website policies and comply with all applicable legal requirements.*
docs/cli/interactive-shell.md
+235
View File
@@ -0,0 +1,235 @@
# Scrapling Interactive Shell Guide
<script src="https://asciinema.org/a/736339.js" id="asciicast-736339" async data-autoplay="1" data-loop="1" data-cols="225" data-rows="40" data-start-at="00:06" data-speed="1.5"></script>
**Powerful Web Scraping REPL for Developers and Data Scientists**
The Scrapling Interactive Shell is an enhanced IPython-based environment designed specifically for Web Scraping tasks. It provides instant access to all Scrapling features, clever shortcuts, automatic page management, and advanced tools like curl command conversion.
## Why use the Interactive Shell?
The interactive shell transforms web scraping from a slow script-and-run cycle into a fast, exploratory experience. It's perfect for:
- **Rapid prototyping**: Test scraping strategies instantly
- **Data exploration**: Interactively navigate and extract from websites
- **Learning Scrapling**: Experiment with features in real-time
- **Debugging scrapers**: Step through requests and inspect results
- **Converting workflows**: Transform curl commands from browser DevTools to a Fetcher request in a one-liner
## Getting Started
### Launch the Shell
```bash
# Start the interactive shell
scrapling shell
# Execute code and exit (useful for scripting)
scrapling shell -c "get('https://quotes.toscrape.com'); print(len(page.css('.quote')))"
# Set logging level
scrapling shell --loglevel info
```
Once launched, you'll see the Scrapling banner and can immediately start scraping as the video above shows:
```python
# No imports needed - everything is ready!
>>> get('https://news.ycombinator.com')
>>> # Explore the page structure
>>> page.css('a')[:5] # Look at first 5 links
>>> # Refine your selectors
>>> stories = page.css('.titleline>a')
>>> len(stories)
30
>>> # Extract specific data
>>> for story in stories[:3]:
... title = story.text
... url = story['href']
... print(f"{title}: {url}")
>>> # Try different approaches
>>> titles = page.css('.titleline>a::text') # Direct text extraction
>>> urls = page.css('.titleline>a::attr(href)') # Direct attribute extraction
```
## Built-in Shortcuts
The shell provides convenient shortcuts that eliminate boilerplate code:
- **`get(url, **kwargs)`** - HTTP GET request (instead of `Fetcher.get`)
- **`post(url, **kwargs)`** - HTTP POST request (instead of `Fetcher.post`)
- **`put(url, **kwargs)`** - HTTP PUT request (instead of `Fetcher.put`)
- **`delete(url, **kwargs)`** - HTTP DELETE request (instead of `Fetcher.delete`)
- **`fetch(url, **kwargs)`** - Browser-based fetch (instead of `DynamicFetcher.fetch`)
- **`stealthy_fetch(url, **kwargs)`** - Stealthy browser fetch (instead of `StealthyFetcher.fetch`)
The most commonly used classes are automatically available without any import, including `Fetcher`, `AsyncFetcher`, `DynamicFetcher`, `StealthyFetcher`, and `Selector`.
### Smart Page Management
The shell automatically tracks your requests and pages:
- **Current Page Access**
The `page` and `response` commands are automatically updated with the last fetched page:
```python
>>> get('https://quotes.toscrape.com')
>>> # 'page' and 'response' both refer to the last fetched page
>>> page.url
'https://quotes.toscrape.com'
>>> response.status # Same as page.status
200
```
- **Page History**
The `pages` command keeps track of the last five pages (it's a `Selectors` object):
```python
>>> get('https://site1.com')
>>> get('https://site2.com')
>>> get('https://site3.com')
>>> # Access last 5 pages
>>> len(pages) # `Selectors` object with `page` history
3
>>> pages[0].url # First page in history
'https://site1.com'
>>> pages[-1].url # Most recent page
'https://site3.com'
>>> # Work with historical pages
>>> for i, old_page in enumerate(pages):
... print(f"Page {i}: {old_page.url} - {old_page.status}")
```
## Additional helpful commands
### Page Visualization
View scraped pages in your browser:
```python
>>> get('https://quotes.toscrape.com')
>>> view(page) # Opens the page HTML in your default browser
```
### Curl Command Integration
The shell provides a few functions to help you convert curl commands from the browser DevTools to `Fetcher` requests, which are `uncurl` and `curl2fetcher`. First, you need to copy a request as a curl command like the following:
<img src="../../assets/scrapling_shell_curl.png" title="Copying a request as a curl command from Chrome" alt="Copying a request as a curl command from Chrome" style="width: 70%;"/>
- **Convert Curl command to Request Object**
```python
>>> curl_cmd = '''curl 'https://scrapling.requestcatcher.com/post' \
... -X POST \
... -H 'Content-Type: application/json' \
... -d '{"name": "test", "value": 123}' '''
>>> request = uncurl(curl_cmd)
>>> request.method
'post'
>>> request.url
'https://scrapling.requestcatcher.com/post'
>>> request.headers
{'Content-Type': 'application/json'}
```
- **Execute Curl Command Directly**
```python
>>> # Convert and execute in one step
>>> curl2fetcher(curl_cmd)
>>> page.status
200
>>> page.json()['json']
{'name': 'test', 'value': 123}
```
### IPython Features
The shell inherits all IPython capabilities:
```python
>>> # Magic commands
>>> %time page = get('https://example.com') # Time execution
>>> %history # Show command history
>>> %save filename.py 1-10 # Save commands 1-10 to file
>>> # Tab completion works everywhere
>>> page.c<TAB> # Shows: css, css_first, cookies, etc.
>>> Fetcher.<TAB> # Shows all Fetcher methods
>>> # Object inspection
>>> get? # Show get documentation
```
## Examples
Here are a few examples generated via AI:
#### E-commerce Data Collection
```python
>>> # Start with product listing page
>>> catalog = get('https://shop.example.com/products')
>>> # Find product links
>>> product_links = catalog.css('.product-link::attr(href)')
>>> print(f"Found {len(product_links)} products")
>>> # Sample a few products first
>>> for link in product_links[:3]:
... product = get(f"https://shop.example.com{link}")
... name = product.css('.product-name::text').get('')
... price = product.css('.price::text').get('')
... print(f"{name}: {price}")
>>> # Scale up with sessions for efficiency
>>> from scrapling.fetchers import FetcherSession
>>> with FetcherSession() as session:
... products = []
... for link in product_links:
... product = session.get(f"https://shop.example.com{link}")
... products.append({
... 'name': product.css('.product-name::text').get(''),
... 'price': product.css('.price::text').get(''),
... 'url': link
... })
```
#### API Integration and Testing
```python
>>> # Test API endpoints interactively
>>> response = get('https://jsonplaceholder.typicode.com/posts/1')
>>> response.json()
{'userId': 1, 'id': 1, 'title': 'sunt aut...', 'body': 'quia et...'}
>>> # Test POST requests
>>> new_post = post('https://jsonplaceholder.typicode.com/posts',
... json={'title': 'Test Post', 'body': 'Test content', 'userId': 1})
>>> new_post.json()['id']
101
>>> # Test with different data
>>> updated = put(f'https://jsonplaceholder.typicode.com/posts/{new_post.json()["id"]}',
... json={'title': 'Updated Title'})
```
## Getting Help
If you need help other than what is available in-terminal, you can:
- [Scrapling Documentation](https://scrapling.readthedocs.io/)
- [Discord Community](https://discord.gg/EMgGbDceNQ)
- [GitHub Issues](https://github.com/D4Vinci/Scrapling/issues)
And that's it! Happy scraping! The shell makes web scraping as easy as a conversation.
docs/cli/overview.md
+30
View File
@@ -0,0 +1,30 @@
# Command Line Interface
Since v0.3, Scrapling includes a powerful command-line interface that provides three main capabilities:
1. **Interactive Shell**: An interactive Web Scraping shell based on IPython that provides many shortcuts and useful tools
2. **Extract Commands**: Scrape websites from the terminal without any programming
3. **Utility Commands**: Installation and management tools
```bash
# Launch interactive shell
scrapling shell
# Convert the content of a page to markdown and save it to a file
scrapling extract get "https://example.com" content.md
# Get help for any command
scrapling --help
scrapling extract --help
```
## Requirements
This section requires you to install the extra `shell` dependency group, like the following:
```bash
pip install "scrapling[shell]"
```
and the installation of the fetchers' dependencies with the following command
```bash
scrapling install
```
This downloads all browsers with their system dependencies and fingerprint manipulation dependencies.
docs/contributing.md
+1 -1
View File
@@ -68,7 +68,7 @@ We use:
Example:
```
feat: add auto-matching for similar elements
feat: add `adaptive` for similar elements
- Added find_similar() method
- Implemented pattern matching
docs/development/automatch_storage_system.md → docs/development/adaptive_storage_system.md
+7 -7
View File
@@ -1,22 +1,22 @@
Scrapling uses SQLite by default, but this tutorial covers writing your storage system to store element properties there for auto-matching.
Scrapling uses SQLite by default, but this tutorial covers writing your storage system to store element properties there for `adaptive` feature.
You might want to use FireBase, for example, and share the database between multiple spiders on different machines. It's a great idea to use an online database like that because the spiders will share with each other.
So first, to make your storage class work, it must do the big 3:
1. Inherit from the abstract class `scrapling.core.storage_adaptors.StorageSystemMixin` and accept a string argument, which will be the `url` argument to maintain the library logic.
1. Inherit from the abstract class `scrapling.core.storage.StorageSystemMixin` and accept a string argument, which will be the `url` argument to maintain the library logic.
2. Use the decorator `functools.lru_cache` on top of the class to follow the Singleton design pattern as other classes.
3. Implement methods `save` and `retrieve`, as you see from the type hints:
- The method `save` returns nothing and will get two arguments from the library
* The first one is of type `lxml.html.HtmlElement`, which is the element itself. It must be converted to a dictionary using the function `element_to_dict` in submodule `scrapling.core.utils._StorageTools` to keep the same format and save it to your database as you wish.
* The second one is a string, the identifier used for retrieval. The combination result of this identifier and the `url` argument from initialization must be unique for each row, or the auto-match will be messed up.
* The second one is a string, the identifier used for retrieval. The combination result of this identifier and the `url` argument from initialization must be unique for each row, or the `adaptive` data will be messed up.
- The method `retrieve` takes a string, which is the identifier; using it with the `url` passed on initialization, the element's dictionary is retrieved from the database and returned if it exists; otherwise, it returns `None`.
> If the instructions weren't clear enough for you, you can check my implementation using SQLite3 in [storage_adaptors](https://github.com/D4Vinci/Scrapling/blob/main/scrapling/core/storage_adaptors.py) file
> If the instructions weren't clear enough for you, you can check my implementation using SQLite3 in [storage_adaptors](https://github.com/D4Vinci/Scrapling/blob/main/scrapling/core/storage.py) file
If your class meets these criteria, the rest is easy. If you plan to use the library in a threaded application, ensure your class supports it. The default used class is thread-safe.
If your class meets these criteria, the rest is straightforward. If you plan to use the library in a threaded application, ensure your class supports it. The default used class is thread-safe.
Some helper functions are added to the abstract class if you want to use them. It's easier to see it for yourself in the [code](https://github.com/D4Vinci/Scrapling/blob/main/scrapling/core/storage_adaptors.py); it's heavily commented :)
Some helper functions are added to the abstract class if you want to use them. It's easier to see it for yourself in the [code](https://github.com/D4Vinci/Scrapling/blob/main/scrapling/core/storage.py); it's heavily commented :)
## Real-World Example: Redis Storage
@@ -27,7 +27,7 @@ Here's a more practical example generated by AI using Redis:
import redis
import orjson
from functools import lru_cache
from scrapling.core.storage_adaptors import StorageSystemMixin
from scrapling.core.storage import StorageSystemMixin
from scrapling.core.utils import _StorageTools
@lru_cache(None)
docs/donate.md
+18 -2
View File
@@ -1,7 +1,23 @@
I've been working on Scrapling and other public projects in my spare time and have invested considerable resources and effort to provide these projects for free to the community. By becoming a sponsor, you could directly fund my coffee reserves, helping me continuously update existing projects and create new ones.
I've been working on Scrapling and other public projects in my spare time and have invested considerable resources and effort to provide these projects for free to the community. By becoming a sponsor, you would directly fund my coffee reserves, helping me continuously update existing projects and create new ones.
You can sponsor me directly through [Github sponsors program](https://github.com/sponsors/D4Vinci) or [Buy Me A Coffe](https://buymeacoffee.com/d4vinci). If you are a **company** and looking to **advertise** your business through Scrapling or another project, check out the available plans on my [Github Sponsors page](https://github.com/sponsors/D4Vinci).
You can sponsor me directly through [GitHub sponsors program](https://github.com/sponsors/D4Vinci) or [Buy Me A Coffe](https://buymeacoffee.com/d4vinci).
Thank you, stay curious, and hack the planet! ❤️
## Advertisement
If you are looking to **advertise** your business through Scrapling and take advantage of our target audience, check out the [available tiers](https://github.com/sponsors/D4Vinci):
### [The Silver tier](https://github.com/sponsors/D4Vinci/sponsorships?tier_id=435496) ($50/month)
Perks:
- Your logo will be featured at [the top of Scrapling's project page](https://github.com/D4Vinci/Scrapling?tab=readme-ov-file#sponsors).
- The same logo will be featured at [the top of Scrapling's PyPI page](https://pypi.org/project/scrapling/).
### [The Gold tier](https://github.com/sponsors/D4Vinci/sponsorships?tier_id=435495) ($100/month)
Perks:
- Your logo will be featured at [the top of Scrapling's project page](https://github.com/D4Vinci/Scrapling?tab=readme-ov-file#sponsors).
- The same logo will be featured at [the top of Scrapling's PyPI page](https://pypi.org/project/scrapling/).
- Your logo will be featured as a top sponsor on [Scrapling's website](https://scrapling.readthedocs.io/en/latest/) main page.
- A Shoutout with each [Release note](https://github.com/D4Vinci/Scrapling/release).
docs/fetching/choosing.md
+24 -24
View File
@@ -1,66 +1,66 @@
## Introduction
Fetchers are classes that can do requests or fetch pages for you easily in a single-line fashion with many features and then return a [Response](#response-object) object.
Fetchers are classes that can do requests or fetch pages for you easily in a single-line fashion with many features and then return a [Response](#response-object) object. Starting with v0.3, all fetchers have other classes to keep the session running, so for example, a fetcher that uses a browser will keep the browser open till you finish all your requests through it instead of opening multiple browsers. So it depends on your use case.
This feature was introduced because the only option before v0.2 was to fetch the page as you wanted, then pass it manually to the `Adaptor` class and start playing with it.
This feature was introduced because, before v0.2, Scrapling was only a parsing engine; therefore, we wanted to gradually transition to become your one-stop shop for all Web Scraping needs.
> Fetchers are not wrappers built on top of other libraries, but they use these libraries as an engine to make requests/fetch pages easily for you while fully utilizing that engine and adding features for you that aren't included in those engines
> Fetchers are not wrappers built on top of other libraries. However, they utilize these libraries as an engine to request/fetch pages easily for you, while fully leveraging that engine and adding features for you. Some fetchers don't even use the official library for requests; instead, they use their own custom version. For example, `StealthyFetcher` utilizes `Camoufox` browser directly, without relying on its Python library for anything except launch options. This last part might change soon as well.
## Fetchers Overview
Scrapling provides three different fetcher classes, each designed for specific use cases.
Scrapling provides three different fetcher classes with their session classes; each fetcher is designed for a specific use case.
The following table compares them and can be quickly used for guidance.
| Feature | Fetcher | PlayWrightFetcher | StealthyFetcher |
|--------------------|----------------|--------------------------------------------------------------------------------|--------------------------------------------------------------------------------------|
| Relative speed | ⭐⭐⭐⭐⭐ | ⭐⭐⭐ | ⭐⭐ |
| Stealth | ⭐ | ⭐⭐ | ⭐⭐⭐⭐ |
| Anti-Bot options | ⭐ | ⭐⭐ | ⭐⭐⭐⭐ |
| JavaScript loading | ❌ | ✅ | ✅ |
| Memory Usage | ⭐ | ⭐⭐⭐ | ⭐⭐⭐ |
| Best used for | Basic scraping | - Dynamically loaded websites <br/>- Small automation<br/>- Slight protections | - Dynamically loaded websites <br/>- Small automation <br/>- Complicated protections |
| Browser(s) | ❌ | Chromium and Google Chrome | Modified Firefox |
| Browser API used | ❌ | PlayWright | PlayWright |
| Setup Complexity | Simple | Simple | Simple |
| Feature | Fetcher | DynamicFetcher | StealthyFetcher |
|--------------------|---------------------------------------------------|--------------------------------------------------------------------------------|--------------------------------------------------------------------------------------|
| Relative speed | 🐇🐇🐇🐇🐇 | 🐇🐇🐇 | 🐇🐇 |
| Stealth | ⭐⭐ | ⭐⭐⭐ | ⭐⭐⭐⭐⭐ |
| Anti-Bot options | ⭐⭐ | ⭐⭐⭐ | ⭐⭐⭐⭐⭐ |
| JavaScript loading | ❌ | ✅ | ✅ |
| Memory Usage | ⭐ | ⭐⭐⭐ | ⭐⭐⭐ |
| Best used for | Basic scraping when HTTP requests alone can do it | - Dynamically loaded websites <br/>- Small automation<br/>- Slight protections | - Dynamically loaded websites <br/>- Small automation <br/>- Complicated protections |
| Browser(s) | ❌ | Chromium and Google Chrome | Modified Firefox |
| Browser API used | ❌ | PlayWright | PlayWright |
| Setup Complexity | Simple | Simple | Simple |
In the following pages, we will talk about each one in detail.
## Parser configuration in all fetchers
All fetchers classes share the same import, as you will see in the upcoming pages
All fetchers share the same import method, as you will see in the upcoming pages
```python
>>> from scrapling.fetchers import Fetcher, AsyncFetcher, StealthyFetcher, PlayWrightFetcher
>>> from scrapling.fetchers import Fetcher, AsyncFetcher, StealthyFetcher, DynamicFetcher
```
Then you use it right away without initializing like this, and it will use the default parser settings:
```python
>>> page = StealthyFetcher.fetch('https://example.com')
```
If you want to configure the parser ([Adaptor class](../parsing/main_classes.md#adaptor)) that will be used on the response before returning it for you, then do this first:
If you want to configure the parser ([Selector class](../parsing/main_classes.md#selector)) that will be used on the response before returning it for you, then do this first:
```python
>>> from scrapling.fetchers import Fetcher
>>> Fetcher.configure(auto_match=True, encoding="utf8", keep_comments=False, keep_cdata=False) # and the rest
>>> Fetcher.configure(adaptive=True, encoding="utf8", keep_comments=False, keep_cdata=False) # and the rest
```
or
```python
>>> from scrapling.fetchers import Fetcher
>>> Fetcher.auto_match=True
>>> Fetcher.adaptive=True
>>> Fetcher.encoding="utf8"
>>> Fetcher.keep_comments=False
>>> Fetcher.keep_cdata=False # and the rest
```
Then, continue your code as usual.
The available configuration arguments are: `auto_match`, `huge_tree`, `keep_comments`, `keep_cdata`, `storage`, and `storage_args`, which are the same ones you give to the `Adaptor` class. You can display the current configuration anytime by running `<fetcher_class>.display_config()`.
The available configuration arguments are: `adaptive`, `huge_tree`, `keep_comments`, `keep_cdata`, `storage`, and `storage_args`, which are the same ones you give to the [Selector](../parsing/main_classes.md#selector) class. You can display the current configuration anytime by running `<fetcher_class>.display_config()`.
> Note: The `auto_match` argument is disabled by default; you must enable it to use that feature.
> Note: The `adaptive` argument is disabled by default; you must enable it to use that feature.
### Set parser config per request
As you probably understood, the logic above for setting the parser config will work globally for all requests/fetches done through that class, and it's intended for simplicity.
As you probably understand, the logic above for setting the parser config will apply globally to all requests/fetches made through that class, and it's intended for simplicity.
If your use case requires a different configuration for each request/fetch, you can pass a dictionary to the request method (`fetch`/`get`/`post`/...) to an argument named `custom_config`.
## Response Object
The `Response` object is the same as the [Adaptor](../parsing/main_classes.md#adaptor) class, but it has added details about the response like response headers, status, cookies, etc... as shown below:
The `Response` object is the same as the [Selector](../parsing/main_classes.md#selector) class, but it has additional details about the response, like response headers, status, cookies, etc., as shown below:
```python
>>> from scrapling.fetchers import Fetcher
>>> page = Fetcher.get('https://example.com')
docs/fetching/dynamic.md
+119 -71
View File
@@ -1,57 +1,57 @@
# Introduction
Here, we will discuss the `PlayWrightFetcher` class. This class provides flexible browser automation with multiple configuration options and some stealth capabilities. It uses [PlayWright](https://playwright.dev/python/docs/intro) as an engine for fetching websites.
Here, we will discuss the `DynamicFetcher` class (previously known as `PlayWrightFetcher`). This class provides flexible browser automation with multiple configuration options and some stealth capabilities.
As we will explain later, to automate the page, you need some knowledge of [PlayWright's Page API](https://playwright.dev/python/docs/api/class-page).
As we will explain later, to automate the page, you need some knowledge of [Playwright's Page API](https://playwright.dev/python/docs/api/class-page).
## Basic Usage
You have one primary way to import this Fetcher, which is the same for all fetchers.
```python
>>> from scrapling.fetchers import PlayWrightFetcher
>>> from scrapling.fetchers import DynamicFetcher
```
Check out how to configure the parsing options [here](choosing.md#parser-configuration-in-all-fetchers)
Now we will go over most of the arguments one by one with examples if you want to jump to a table of all arguments for quick reference [click here](#full-list-of-arguments)
Now, we will review most of the arguments one by one, using examples. If you want to jump to a table of all arguments for quick reference, [click here](#full-list-of-arguments)
> Notes:
>
> 1. Every time you fetch a website with this fetcher, it waits by default for all JavaScript to fully load and execute, so you don't have to (waits for the `domcontentloaded` state).
> 1. Every time you fetch a website with this fetcher, it waits by default for all JavaScript to fully load and execute, so you don't have to (wait for the `domcontentloaded` state).
> 2. Of course, the async version of the `fetch` method is the `async_fetch` method.
This fetcher currently provides 4 main run options, but they can be mixed as you want.
This fetcher currently provides four main run options, which can be mixed as desired.
Which are:
### 1. Vanilla Playwright
```python
PlayWrightFetcher.fetch('https://example.com')
DynamicFetcher.fetch('https://example.com')
```
Using it like that will open a Chromium browser and fetch the page. There are no tricks or extra features; it's just a plain PlayWright API.
Using it in that manner will open a Chromium browser and load the page. There are no tricks or extra features unless you enable some; it's just a plain PlayWright API.
### 2. Stealth Mode
```python
PlayWrightFetcher.fetch('https://example.com', stealth=True)
DynamicFetcher.fetch('https://example.com', stealth=True)
```
It's the same as the vanilla PlayWright option, but it provides a simple stealth mode suitable for websites with a small-to-medium protection layer(s).
It's the same as the vanilla Playwright option, but it provides a simple stealth mode suitable for websites with a small to medium protection layer(s).
Some of the things this fetcher's stealth mode does include:
* Patching the CDP runtime fingerprint.
* Mimics some of the real browsers' properties by injecting several JS files and using custom options.
* Custom flags are used on launch to hide Playwright even more and make it faster.
* Generates real browser headers of the same type and user OS, then append them to the request's headers.
* Generates real browser headers of the same type and user OS, then appends them to the request's headers.
### 3. Real Chrome
```python
PlayWrightFetcher.fetch('https://example.com', real_chrome=True)
DynamicFetcher.fetch('https://example.com', real_chrome=True)
```
If you have a Google Chrome browser installed, use this option. It's the same as the first option but will use the Google Chrome browser you installed on your device instead of Chromium.
If you have a Google Chrome browser installed, use this option. It's the same as the first option, but will use the Google Chrome browser you installed on your device instead of Chromium.
This will make your requests look more like requests coming from an actual human, so it's less detectable, and you can even use the `stealth=True` mode with it for better results like below:
This will make your requests look more authentic, so it's less detectable, and you can even use the `stealth=True` mode with it for better results, like below:
```python
PlayWrightFetcher.fetch('https://example.com', real_chrome=True, stealth=True)
DynamicFetcher.fetch('https://example.com', real_chrome=True, stealth=True)
```
If you don't have Google Chrome installed and want to use this option, you can use the command below in the terminal to install it for the library instead of installing it manually:
```commandline
@@ -60,52 +60,45 @@ playwright install chrome
### 4. CDP Connection
```python
PlayWrightFetcher.fetch('https://example.com', cdp_url='ws://localhost:9222')
DynamicFetcher.fetch('https://example.com', cdp_url='ws://localhost:9222')
```
Instead of launching a browser locally (Chromium/Google Chrome), you can connect to a remote browser through the [Chrome DevTools Protocol](https://chromedevtools.github.io/devtools-protocol/).
This fetcher takes it even a step further. You can use [NSTBrowser](https://app.nstbrowser.io/r/1vO5e5)'s [docker browserless](https://hub.docker.com/r/nstbrowser/browserless) option by passing the CDP URL and enabling `nstbrowser_mode` option like below
```python
PlayWrightFetcher.fetch('https://example.com', cdp_url='ws://localhost:9222', nstbrowser_mode=True)
```
There's also a `nstbrowser_config` argument to send the config you want to send with the requests to the NSTBrowser. If you leave it empty, Scrapling defaults to an optimized NSTBrowser's docker browserless config.
## Full list of arguments
Scrapling provides many options with this fetcher, which works in all modes except the [NSTBrowser](https://app.nstbrowser.io/r/1vO5e5) mode. To make it as simple as possible, we will list the options here and give examples of using most of them.
| Argument | Description | Optional |
|:-------------------:|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:--------:|
| url | Target url | ❌ |
| headless | Pass `True` to run the browser in headless/hidden (**default**) or `False` for headful/visible mode. | ✔️ |
| disable_resources | Drop requests of unnecessary resources for a speed boost. It depends, but it made requests ~25% faster in my tests for some websites.<br/>Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`. _This can help save your proxy usage, but be careful with this option as it makes some websites never finish loading._ | ✔️ |
| useragent | Pass a useragent string to be used. **Otherwise, the fetcher will generate and use a real Useragent of the same browser.** | ✔️ |
| network_idle | Wait for the page until there are no network connections for at least 500 ms. | ✔️ |
| timeout | The timeout (milliseconds) used in all operations and waits through the page. The default is 30000. | ✔️ |
| wait | The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the `Response` object. | ✔️ |
| page_action | Added for automation. Pass a function that takes the `page` object and does the necessary automation, then returns `page` again. | ✔️ |
| wait_selector | Wait for a specific css selector to be in a specific state. | ✔️ |
| wait_selector_state | Scrapling will wait for the given state to be fulfilled for the selector given with `wait_selector`. _Default state is `attached`._ | ✔️ |
| google_search | Enabled by default, Scrapling will set the referer header as if this request came from a Google search for this website's domain name. | ✔️ |
| extra_headers | A dictionary of extra headers to add to the request. The referer set by the `google_search` argument takes priority over the referer set here if used together. | ✔️ |
| proxy | The proxy to be used with requests. It can be a string or a dictionary with the keys 'server', 'username', and 'password' only. | ✔️ |
| hide_canvas | Add random noise to canvas operations to prevent fingerprinting. | ✔️ |
| disable_webgl | Disables WebGL and WebGL 2.0 support entirely. | ✔️ |
| stealth | Enables stealth mode; you should always check the documentation to see what stealth mode does currently. | ✔️ |
| real_chrome | If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch and use an instance of your browser and use it. | ✔️ |
| locale | Set the locale for the browser if wanted. The default value is `en-US`. | ✔️ |
| cdp_url | Instead of launching a new browser instance, connect to this CDP URL to control real browsers/NSTBrowser through CDP. | ✔️ |
| nstbrowser_mode | Enables NSTBrowser mode, **it have to be used with `cdp_url` argument or it will get completely ignored.** | ✔️ |
| nstbrowser_config | The config you want to send with requests to the NSTBrowser. _Scrapling defaults to an optimized NSTBrowser's docker browserless config if you leave this argument empty._ | ✔️ |
Scrapling provides many options with this fetcher. To make it as simple as possible, we will list the options here and give examples of using most of them.
| Argument | Description | Optional |
|:-------------------:|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:--------:|
| url | Target url | ❌ |
| headless | Pass `True` to run the browser in headless/hidden (**default**) or `False` for headful/visible mode. | ✔️ |
| disable_resources | Drop requests of unnecessary resources for a speed boost. It depends, but it made requests ~25% faster in my tests for some websites.<br/>Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`. _This can help save your proxy usage, but be cautious with this option, as it may cause some websites to never finish loading._ | ✔️ |
| cookies | Set cookies for the next request. | ✔️ |
| useragent | Pass a useragent string to be used. **Otherwise, the fetcher will generate and use a real Useragent of the same browser.** | ✔️ |
| network_idle | Wait for the page until there are no network connections for at least 500 ms. | ✔️ |
| timeout | The timeout (milliseconds) used in all operations and waits through the page. The default is 30,000 ms (30 seconds). | ✔️ |
| wait | The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the `Response` object. | ✔️ |
| page_action | Added for automation. Pass a function that takes the `page` object and does the necessary automation, then returns `page` again. | ✔️ |
| wait_selector | Wait for a specific css selector to be in a specific state. | ✔️ |
| wait_selector_state | Scrapling will wait for the given state to be fulfilled for the selector given with `wait_selector`. _Default state is `attached`._ | ✔️ |
| google_search | Enabled by default, Scrapling will set the referer header as if this request came from a Google search of this website's domain name. | ✔️ |
| extra_headers | A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ | ✔️ |
| proxy | The proxy to be used with requests. It can be a string or a dictionary with the keys 'server', 'username', and 'password' only. | ✔️ |
| hide_canvas | Add random noise to canvas operations to prevent fingerprinting. | ✔️ |
| disable_webgl | Disables WebGL and WebGL 2.0 support entirely. | ✔️ |
| stealth | Enables stealth mode; you should always check the documentation to see what the stealth mode does currently. | ✔️ |
| real_chrome | If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch and use an instance of your browser. | ✔️ |
| locale | Set the locale for the browser if wanted. The default value is `en-US`. | ✔️ |
| cdp_url | Instead of launching a new browser instance, connect to this CDP URL to control real browsers through CDP. | ✔️ |
| selector_config | A dictionary of custom parsing arguments to be used when creating the final `Selector`/`Response` class. | ✔️ |
## Examples
It's easier to understand with examples, so let's look at it.
It's easier to understand with examples, so let's take a look.
### Resource Control
```python
# Disable unnecessary resources
page = PlayWrightFetcher.fetch(
page = DynamicFetcher.fetch(
'https://example.com',
disable_resources=True # Blocks fonts, images, media, etc...
)
@@ -115,22 +108,22 @@ page = PlayWrightFetcher.fetch(
```python
# Wait for network idle (Consider fetch to be finished when there are no network connections for at least 500 ms)
page = PlayWrightFetcher.fetch('https://example.com', network_idle=True)
page = DynamicFetcher.fetch('https://example.com', network_idle=True)
# Custom timeout (in milliseconds)
page = PlayWrightFetcher.fetch('https://example.com', timeout=30000) # 30 seconds
page = DynamicFetcher.fetch('https://example.com', timeout=30000) # 30 seconds
# Proxy support
page = PlayWrightFetcher.fetch(
page = DynamicFetcher.fetch(
'https://example.com',
proxy='http://username:password@host:port' # Or it can be a dictionary with the keys 'server', 'username', and 'password' only
)
```
### Browser Automation
This is where your knowledge about [PlayWright's Page API](https://playwright.dev/python/docs/api/class-page) comes into play. The function you pass here takes the page object from Playwright's API, does what you want, and then returns it again for the current fetcher to continue working on it.
This is where your knowledge about [Playwright's Page API](https://playwright.dev/python/docs/api/class-page) comes into play. The function you pass here takes the page object from Playwright's API, performs the desired action, and then returns it for the current fetcher to continue processing.
This function is executed right after waiting for network_idle (if enabled) and before waiting for the `wait_selector` argument, so it can be used for many things, not just automation. You can alter the page as you want.
This function is executed immediately after waiting for `network_idle` (if enabled) and before waiting for the `wait_selector` argument, allowing it to be used for various purposes, not just automation. You can alter the page as you want.
In the example below, I used page [mouse events](https://playwright.dev/python/docs/api/class-mouse) to move the mouse wheel to scroll the page and then move the mouse.
```python
@@ -142,7 +135,7 @@ def scroll_page(page: Page):
page.mouse.up()
return page
page = PlayWrightFetcher.fetch(
page = DynamicFetcher.fetch(
'https://example.com',
page_action=scroll_page
)
@@ -157,7 +150,7 @@ async def scroll_page(page: Page):
await page.mouse.up()
return page
page = await PlayWrightFetcher.async_fetch(
page = await DynamicFetcher.async_fetch(
'https://example.com',
page_action=scroll_page
)
@@ -167,7 +160,7 @@ page = await PlayWrightFetcher.async_fetch(
```python
# Wait for the selector
page = PlayWrightFetcher.fetch(
page = DynamicFetcher.fetch(
'https://example.com',
wait_selector='h1',
wait_selector_state='visible'
@@ -175,20 +168,20 @@ page = PlayWrightFetcher.fetch(
```
This is the last wait the fetcher will do before returning the response (if enabled). You pass a CSS selector to the `wait_selector` argument, and the fetcher will wait for the state you passed in the `wait_selector_state` argument to be fulfilled. If you didn't pass a state, the default would be `attached`, which means it will wait for the element to be present in the DOM.
After that, the fetcher will check again to see if all JS files are loaded and executed (the `domcontentloaded` state) and wait for them to be. If you have enabled `network_idle` with this, the fetcher will wait for `network_idle` to be fulfilled again, as explained above.
After that, the fetcher will check again to see if all JS files are loaded and executed (the `domcontentloaded` state) or continue waiting. If you have enabled `network_idle` with this, the fetcher will wait for `network_idle` to be fulfilled again, as explained above.
The states the fetcher can wait for can be either ([source](https://playwright.dev/python/docs/api/class-page#page-wait-for-selector)):
The states the fetcher can wait for can be any of the following ([source](https://playwright.dev/python/docs/api/class-page#page-wait-for-selector)):
- `attached`: Wait for an element to be present in DOM.
- `detached`: Wait for an element to not be present in DOM.
- `attached`: Wait for an element to be present in the DOM.
- `detached`: Wait for an element to not be present in the DOM.
- `visible`: wait for an element to have a non-empty bounding box and no `visibility:hidden`. Note that an element without any content or with `display:none` has an empty bounding box and is not considered visible.
- `hidden`: wait for an element to be either detached from DOM, or have an empty bounding box or `visibility:hidden`. This is opposite to the `'visible'` option.
- `hidden`: wait for an element to be either detached from the DOM, or have an empty bounding box, or `visibility:hidden`. This is opposite to the `'visible'` option.
### Some Stealth Features
```python
# Full stealth mode
page = PlayWrightFetcher.fetch(
page = DynamicFetcher.fetch(
'https://example.com',
stealth=True,
hide_canvas=True,
@@ -197,28 +190,28 @@ page = PlayWrightFetcher.fetch(
)
# Custom user agent
page = PlayWrightFetcher.fetch(
page = DynamicFetcher.fetch(
'https://example.com',
useragent='Mozilla/5.0...'
)
# Set browser locale
page = PlayWrightFetcher.fetch(
page = DynamicFetcher.fetch(
'https://example.com',
locale='en-US'
)
```
Hence, the `hide_canvas` argument doesn't disable canvas but hides it by adding random noise to canvas operations to prevent fingerprinting. Also, if you didn't set a useragent (preferred), the fetcher will generate a real Useragent of the same browser and use it.
Hence, the `hide_canvas` argument doesn't disable the canvas but instead hides it by adding random noise to canvas operations, preventing fingerprinting. Also, if you didn't set a user agent (preferred), the fetcher will generate a real User Agent of the same browser and use it.
The `google_search` argument is enabled by default, making the request look like it came from Google. So, a request for `https://example.com` will set the referer to `https://www.google.com/search?q=example`. Also, if used together, it takes priority over the referer set by the `extra_headers` argument.
The `google_search` argument is enabled by default, making the request look as if it came from a Google search page. So, a request for `https://example.com` will set the referer to `https://www.google.com/search?q=example`. Also, if used together, it takes priority over the referer set by the `extra_headers` argument.
### General example
```python
from scrapling.fetchers import PlayWrightFetcher
from scrapling.fetchers import DynamicFetcher
def scrape_dynamic_content():
# Use PlayWright for JavaScript content
page = PlayWrightFetcher.fetch(
# Use Playwright for JavaScript content
page = DynamicFetcher.fetch(
'https://example.com/dynamic',
network_idle=True,
wait_selector='.content'
@@ -235,9 +228,64 @@ def scrape_dynamic_content():
}
```
## Session Management
To keep the browser open until you make multiple requests with the same configuration, use `DynamicSession`/`AsyncDynamicSession` classes. Those classes can accept all the arguments that the `fetch` function can take, which enables you to specify a config for the entire session.
```python
from scrapling.fetchers import DynamicSession
# Create a session with default configuration
with DynamicSession(
headless=True,
stealth=True,
disable_resources=True,
real_chrome=True
) as session:
# Make multiple requests with the same browser instance
page1 = session.fetch('https://example1.com')
page2 = session.fetch('https://example2.com')
page3 = session.fetch('https://dynamic-site.com')
# All requests reuse the same tab on the same browser instance
```
### Async Session Usage
```python
import asyncio
from scrapling.fetchers import AsyncDynamicSession
async def scrape_multiple_sites():
async with AsyncDynamicSession(
stealth=True,
network_idle=True,
timeout=30000,
max_pages=3
) as session:
# Make async requests with shared browser configuration
pages = await asyncio.gather(
session.fetch('https://spa-app1.com'),
session.fetch('https://spa-app2.com'),
session.fetch('https://dynamic-content.com')
)
return pages
```
You may have noticed the `max_pages` argument. This is a new argument that enables the fetcher to create a **pool of Browser tabs** that will be rotated automatically. Instead of waiting for one browser tab to become ready, it checks if the next tab in the pool is ready to be used and uses it. This allows for multiple websites to be fetched at the same time in the same browser, which saves a lot of resources, but most importantly, is so fast :)
When all tabs inside the pool are busy, the fetcher checks every subsecond if a tab becomes ready. If none become free within a 30-second interval, it raises a `TimeoutError` error. This can happen when the website you are fetching becomes unresponsive for some reason.
### Session Benefits
- **Browser reuse**: Much faster subsequent requests by reusing the same browser instance.
- **Cookie persistence**: Automatic cookie and session state handling as any browser does automatically.
- **Consistent fingerprint**: Same browser fingerprint across all requests.
- **Memory efficiency**: Better resource usage compared to launching new browsers with each fetch.
## When to Use
Use PlayWrightFetcher when:
Use DynamicFetcher when:
- Need browser automation
- Want multiple browser options
docs/fetching/static.md
+127 -39
View File
@@ -1,6 +1,6 @@
# Introduction
The `Fetcher` class provides fast and lightweight HTTP requests with some stealth capabilities. This class uses [httpx](https://www.python-httpx.org/) as an engine for making requests. For advanced usages, you will need some knowledge about [httpx](https://www.python-httpx.org/), but it becomes simpler and simpler with user feedback and updates.
The `Fetcher` class provides rapid and lightweight HTTP requests using the high-performance `curl_cffi` library with a lot of stealth capabilities.
## Basic Usage
You have one primary way to import this Fetcher, which is the same for all fetchers.
@@ -13,17 +13,34 @@ Check out how to configure the parsing options [here](choosing.md#parser-configu
### Shared arguments
All methods for making requests here share some arguments, so let's discuss them first.
- **url**: The URL you want to request, of course :)
- **url**: The targeted URL
- **stealthy_headers**: If enabled (default), it creates and adds real browser headers. It also sets the referer header as if this request came from a Google search of the URL's domain.
- **follow_redirects**: As the name implies, tell the fetcher to follow redirections. **Enabled by default**
- **timeout**: The number of seconds to wait for each request to be finished. **Defaults to 30 seconds**.
- **retries**: The number of retries that the fetcher will do for failed requests. **Defaults to three retries**.
- **retry_delay**: Number of seconds to wait between retry attempts. **Defaults to 1 second**.
- **impersonate**: Impersonate specific browsers' TLS fingerprints. Accepts browser strings like `"chrome110"`, `"firefox102"`, `"safari15_5"` to use specific versions or `"chrome"`, `"firefox"`, `"safari"`, `"edge"` to automatically use the latest version available. This makes your requests appear as if they're coming from real browsers at the TLS level. **Defaults to the latest available Chrome version.**
- **http3**: Use HTTP/3 protocol for requests. **Defaults to False**. It might be problematic if used with `impersonate`.
- **cookies**: Cookies to use in the request. Can be a dictionary of `name→value` or a list of dictionaries.
- **proxy**: As the name implies, the proxy for this request is used to route all traffic (HTTP and HTTPS). The format accepted here is `http://username:password@localhost:8030`.
- **stealthy_headers**: Generate and use real browser's headers, then create a referer header as if this request came from a Google search page of this URL's domain. Enabled by default, all headers generated can be overwritten by you through the `headers` argument.
- **follow_redirects**: As the name implies, tell the fetcher to follow redirections. Enabled by default
- **timeout**: The timeout to wait for each request to be finished in milliseconds. The default is 30000ms (30 seconds).
- **retries**: The number of retries that [httpx](https://www.python-httpx.org/) will do for failed requests. The default number of retries is 3.
- **proxy_auth**: HTTP basic auth for proxy, tuple of (username, password).
- **proxies**: Dict of proxies to use. Format: `{"http": proxy_url, "https": proxy_url}`.
- **headers**: Headers to include in the request. Can override any header generated by the `stealthy_headers` argument
- **max_redirects**: Maximum number of redirects. **Defaults to 30**, use -1 for unlimited.
- **verify**: Whether to verify HTTPS certificates. **Defaults to True**.
- **cert**: Tuple of (cert, key) filenames for the client certificate.
- **selector_config**: A dictionary of custom parsing arguments to be used when creating the final `Selector`/`Response` class.
Other than this, you can pass any arguments that `httpx.<method_name>` takes, and that's why I said, in the beginning, you need a bit of knowledge about [httpx](https://www.python-httpx.org/), but in the following examples, we will try to cover most cases.
> Note: <br/>
> 1. The currently available browsers to impersonate are (`"edge"`, `"chrome"`, `"chrome_android"`, `"safari"`, `"safari_beta"`, `"safari_ios"`, `"safari_ios_beta"`, `"firefox"`, `"tor"`)<br/>
> 2. The available browsers to impersonate and their corresponding versions are automatically displayed in the argument autocompletion and updated automatically with each `curl_cffi` update.
Other than this, for further customization, you can pass any arguments that `curl_cffi` supports for any method if that method doesn't already support it.
### HTTP Methods
Examples are the best way to explain this
There are additional arguments for each method, depending on the method, such as `params` for GET requests and `data`/`json` for POST/PUT/DELETE requests.
Examples are the best way to explain this, as follows.
> Hence: `OPTIONS` and `HEAD` methods are not supported.
#### GET
@@ -31,8 +48,8 @@ Examples are the best way to explain this
>>> from scrapling.fetchers import Fetcher
>>> # Basic GET
>>> page = Fetcher.get('https://example.com')
>>> page = Fetcher.get('https://httpbin.org/get', stealthy_headers=True, follow_redirects=True)
>>> page = Fetcher.get('https://httpbin.org/get', proxy='http://username:password@localhost:8030')
>>> page = Fetcher.get('https://scrapling.requestcatcher.com/get', stealthy_headers=True, follow_redirects=True)
>>> page = Fetcher.get('https://scrapling.requestcatcher.com/get', proxy='http://username:password@localhost:8030')
>>> # With parameters
>>> page = Fetcher.get('https://example.com/search', params={'q': 'query'})
>>>
@@ -40,14 +57,18 @@ Examples are the best way to explain this
>>> page = Fetcher.get('https://example.com', headers={'User-Agent': 'Custom/1.0'})
>>> # Basic HTTP authentication
>>> page = Fetcher.get("https://example.com", auth=("my_user", "password123"))
>>> # Browser impersonation
>>> page = Fetcher.get('https://example.com', impersonate='chrome')
>>> # HTTP/3 support
>>> page = Fetcher.get('https://example.com', http3=True)
```
And for asynchronous requests, it's a small adjustment
```python
>>> from scrapling.fetchers import AsyncFetcher
>>> # Basic GET
>>> page = await AsyncFetcher.get('https://example.com')
>>> page = await AsyncFetcher.get('https://httpbin.org/get', stealthy_headers=True, follow_redirects=True)
>>> page = await AsyncFetcher.get('https://httpbin.org/get', proxy='http://username:password@localhost:8030')
>>> page = await AsyncFetcher.get('https://scrapling.requestcatcher.com/get', stealthy_headers=True, follow_redirects=True)
>>> page = await AsyncFetcher.get('https://scrapling.requestcatcher.com/get', proxy='http://username:password@localhost:8030')
>>> # With parameters
>>> page = await AsyncFetcher.get('https://example.com/search', params={'q': 'query'})
>>>
@@ -55,8 +76,12 @@ And for asynchronous requests, it's a small adjustment
>>> page = await AsyncFetcher.get('https://example.com', headers={'User-Agent': 'Custom/1.0'})
>>> # Basic HTTP authentication
>>> page = await AsyncFetcher.get("https://example.com", auth=("my_user", "password123"))
>>> # Browser impersonation
>>> page = await AsyncFetcher.get('https://example.com', impersonate='chrome110')
>>> # HTTP/3 support
>>> page = await AsyncFetcher.get('https://example.com', http3=True)
```
Needless to say, the `page` object in all cases is [Response](choosing.md#response-object) object, which is an `Adaptor` as we said, so you will use it directly
Needless to say, the `page` object in all cases is [Response](choosing.md#response-object) object, which is a [Selector](../parsing/main_classes.md#selector) as we said, so you can use it directly
```python
>>> page.css('.something.something')
@@ -77,66 +102,122 @@ Needless to say, the `page` object in all cases is [Response](choosing.md#respon
```python
>>> from scrapling.fetchers import Fetcher
>>> # Basic POST
>>> page = Fetcher.post('https://httpbin.org/post', data={'key': 'value'})
>>> page = Fetcher.post('https://httpbin.org/post', data={'key': 'value'}, stealthy_headers=True, follow_redirects=True)
>>> page = Fetcher.post('https://httpbin.org/post', data={'key': 'value'}, proxy='http://username:password@localhost:8030')
>>> page = Fetcher.post('https://scrapling.requestcatcher.com/post', data={'key': 'value'}, params={'q': 'query'})
>>> page = Fetcher.post('https://scrapling.requestcatcher.com/post', data={'key': 'value'}, stealthy_headers=True, follow_redirects=True)
>>> page = Fetcher.post('https://scrapling.requestcatcher.com/post', data={'key': 'value'}, proxy='http://username:password@localhost:8030', impersonate="chrome")
>>> # Another example of form-encoded data
>>> page = Fetcher.post('https://example.com/submit', data={'username': 'user', 'password': 'pass'})
>>> page = Fetcher.post('https://example.com/submit', data={'username': 'user', 'password': 'pass'}, http3=True)
>>> # JSON data
>>> page = Fetcher.post('https://example.com/api', json={'key': 'value'})
>>> # Uploading file
>>> r = Fetcher.post("https://httpbin.org/post", files={'upload-file': open('something.xlsx', 'rb')})
```
And for asynchronous requests, it's a small adjustment
```python
>>> from scrapling.fetchers import AsyncFetcher
>>> # Basic POST
>>> page = await AsyncFetcher.post('https://httpbin.org/post', data={'key': 'value'})
>>> page = await AsyncFetcher.post('https://httpbin.org/post', data={'key': 'value'}, stealthy_headers=True, follow_redirects=True)
>>> page = await AsyncFetcher.post('https://httpbin.org/post', data={'key': 'value'}, proxy='http://username:password@localhost:8030')
>>> page = await AsyncFetcher.post('https://scrapling.requestcatcher.com/post', data={'key': 'value'})
>>> page = await AsyncFetcher.post('https://scrapling.requestcatcher.com/post', data={'key': 'value'}, stealthy_headers=True, follow_redirects=True)
>>> page = await AsyncFetcher.post('https://scrapling.requestcatcher.com/post', data={'key': 'value'}, proxy='http://username:password@localhost:8030', impersonate="chrome")
>>> # Another example of form-encoded data
>>> page = await AsyncFetcher.post('https://example.com/submit', data={'username': 'user', 'password': 'pass'})
>>> page = await AsyncFetcher.post('https://example.com/submit', data={'username': 'user', 'password': 'pass'}, http3=True)
>>> # JSON data
>>> page = await AsyncFetcher.post('https://example.com/api', json={'key': 'value'})
>>> # Uploading file
>>> r = await AsyncFetcher.post("https://httpbin.org/post", files={'upload-file': open('something.xlsx', 'rb')})
```
#### PUT
```python
>>> from scrapling.fetchers import Fetcher
>>> # Basic PUT
>>> page = Fetcher.put('https://example.com/update', data={'status': 'updated'})
>>> page = Fetcher.put('https://example.com/update', data={'status': 'updated'}, stealthy_headers=True, follow_redirects=True)
>>> page = Fetcher.put('https://example.com/update', data={'status': 'updated'}, stealthy_headers=True, follow_redirects=True, impersonate="chrome")
>>> page = Fetcher.put('https://example.com/update', data={'status': 'updated'}, proxy='http://username:password@localhost:8030')
>>> # Another example of form-encoded data
>>> page = Fetcher.put("https://httpbin.org/put", data={'key': ['value1', 'value2']})
>>> page = Fetcher.put("https://scrapling.requestcatcher.com/put", data={'key': ['value1', 'value2']})
```
And for asynchronous requests, it's a small adjustment
```python
>>> from scrapling.fetchers import AsyncFetcher
>>> # Basic PUT
>>> page = await AsyncFetcher.put('https://example.com/update', data={'status': 'updated'})
>>> page = await AsyncFetcher.put('https://example.com/update', data={'status': 'updated'}, stealthy_headers=True, follow_redirects=True)
>>> page = await AsyncFetcher.put('https://example.com/update', data={'status': 'updated'}, stealthy_headers=True, follow_redirects=True, impersonate="chrome")
>>> page = await AsyncFetcher.put('https://example.com/update', data={'status': 'updated'}, proxy='http://username:password@localhost:8030')
>>> # Another example of form-encoded data
>>> page = await AsyncFetcher.put("https://httpbin.org/put", data={'key': ['value1', 'value2']})
>>> page = await AsyncFetcher.put("https://scrapling.requestcatcher.com/put", data={'key': ['value1', 'value2']})
```
#### DELETE
```python
>>> from scrapling.fetchers import Fetcher
>>> page = Fetcher.delete('https://example.com/resource/123')
>>> page = Fetcher.delete('https://example.com/resource/123', stealthy_headers=True, follow_redirects=True)
>>> page = Fetcher.delete('https://example.com/resource/123', stealthy_headers=True, follow_redirects=True, impersonate="chrome")
>>> page = Fetcher.delete('https://example.com/resource/123', proxy='http://username:password@localhost:8030')
```
And for asynchronous requests, it's a small adjustment
```python
>>> from scrapling.fetchers import AsyncFetcher
>>> page = await AsyncFetcher.delete('https://example.com/resource/123')
>>> page = await AsyncFetcher.delete('https://example.com/resource/123', stealthy_headers=True, follow_redirects=True)
>>> page = await AsyncFetcher.delete('https://example.com/resource/123', stealthy_headers=True, follow_redirects=True, impersonate="chrome")
>>> page = await AsyncFetcher.delete('https://example.com/resource/123', proxy='http://username:password@localhost:8030')
```
## Session Management
For making multiple requests with the same configuration, use the `FetcherSession` class. It can be used in both synchronous and asynchronous code without issue; the class detects and changes the session type automatically without requiring a different import.
The `FetcherSession` class can accept nearly all the arguments that the methods can take, which enables you to specify a config for the entire session and later choose a different config for one of the requests effortlessly, as you will see in the following examples.
```python
from scrapling.fetchers import FetcherSession
# Create a session with default configuration
with FetcherSession(
impersonate='chrome',
http3=True,
stealthy_headers=True,
timeout=30,
retries=3
) as session:
# Make multiple requests with the same settings
page1 = session.get('https://scrapling.requestcatcher.com/get')
page2 = session.post('https://scrapling.requestcatcher.com/post', data={'key': 'value'})
page3 = session.get('https://api.github.com/events')
# All requests share the same session and connection pool
```
And here's an async example
```python
async with FetcherSession(impersonate='firefox', http3=True) as session:
# All standard HTTP methods available
response = async session.get('https://example.com')
response = async session.post('https://scrapling.requestcatcher.com/post', json={'data': 'value'})
response = async session.put('https://scrapling.requestcatcher.com/put', data={'update': 'info'})
response = async session.delete('https://scrapling.requestcatcher.com/delete')
```
or better
```python
import asyncio
from scrapling.fetchers import FetcherSession
# Async session usage
async with FetcherSession(impersonate="safari") as session:
urls = ['https://example.com/page1', 'https://example.com/page2']
tasks = [
session.get(url) for url in urls
]
pages = await asyncio.gather(*tasks)
```
The `Fetcher` class uses `FetcherSession` to create a temporary session with each request you make.
### Session Benefits
- **A lot faster**: 10 times faster than creating a single session for each request
- **Cookie persistence**: Automatic cookie handling across requests
- **Resource efficiency**: Better memory and CPU usage for multiple requests
- **Centralized configuration**: Single place to manage request settings
## Examples
Some well-rounded examples to aid newcomers to Web Scraping
@@ -276,7 +357,7 @@ def extract_menu():
link = item.css_first('a')
if link:
menu[link.text] = {
'url': link.attrib['href'],
'url': link['href'],
'has_submenu': bool(item.css('.submenu'))
}
@@ -287,14 +368,21 @@ def extract_menu():
Use `Fetcher` when:
- Need fast HTTP requests
- Want minimal overhead
- Don't need JavaScript
- Want simple configuration
- Need basic stealth features
- Need rapid HTTP requests.
- Want minimal overhead.
- Don't need JavaScript execution (the website can be scraped through requests).
- Need some stealth features (ex, the targeted website is using protection but doesn't use JavaScript challenges).
Use `FetcherSession` when:
- Making multiple requests to the same or different sites.
- Need to maintain cookies/authentication between requests.
- Want connection pooling for better performance.
- Require consistent configuration across requests.
- Working with APIs that require a session state.
Use other fetchers when:
- Need browser automation.
- Need advanced anti-bot/stealth.
- Need JavaScript support.
- Need advanced anti-bot/stealth capabilities.
- Need JavaScript support or interacting with dynamic content
docs/fetching/stealthy.md
+130 -42
View File
@@ -1,8 +1,8 @@
# Introduction
Here, we will discuss the `StealthyFetcher` class. This class is similar to [PlayWrightFetcher](dynamic.md#introduction) in many ways, like browser automation and using [PlayWright](https://playwright.dev/python/docs/intro) as an engine for fetching websites. The main difference is that this class provides advanced anti-bot protection bypass capabilities and a modified Firefox browser called [Camoufox](https://github.com/daijro/camoufox), from which most stealth comes.
Here, we will discuss the `StealthyFetcher` class. This class is similar to [DynamicFetcher](dynamic.md#introduction) in many ways, such as browser automation and utilizing [Playwright's API](https://playwright.dev/python/docs/intro). The main difference is that this class provides advanced anti-bot protection bypass capabilities and a custom version of a modified Firefox browser called [Camoufox](https://github.com/daijro/camoufox), from which most stealth comes.
As with [PlayWrightFetcher](dynamic.md#introduction), you will need some knowledge about [PlayWright's Page API](https://playwright.dev/python/docs/api/class-page) to automate the page, as we will explain later.
As with [DynamicFetcher](dynamic.md#introduction), you will need some knowledge about [Playwright's Page API](https://playwright.dev/python/docs/api/class-page) to automate the page, as we will explain later.
## Basic Usage
You have one primary way to import this Fetcher, which is the same for all fetchers.
@@ -14,40 +14,43 @@ Check out how to configure the parsing options [here](choosing.md#parser-configu
> Notes:
>
> 1. Every time you fetch a website with this fetcher, it waits by default for all JavaScript to fully load and execute, so you don't have to (waits for the `domcontentloaded` state).
> 1. Every time you fetch a website with this fetcher, it waits by default for all JavaScript to fully load and execute, so you don't have to (wait for the `domcontentloaded` state).
> 2. Of course, the async version of the `fetch` method is the `async_fetch` method.
## Full list of arguments
Before jumping to [examples](#examples), here's the full list of arguments
| Argument | Description | Optional |
|:--------------------:|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:--------:|
| url | Target url | ❌ |
| headless | Pass `True` to run the browser in headless/hidden (**default**), `virtual` to run it in virtual screen mode, or `False` for headful/visible mode. The `virtual` mode requires having `xvfb` installed. | ✔️ |
| block_images | Prevent the loading of images through Firefox preferences. _This can help save your proxy usage, but be careful with this option as it makes some websites never finish loading._ | ✔️ |
| disable_resources | Drop requests of unnecessary resources for a speed boost. It depends, but it made requests ~25% faster in my tests for some websites.<br/>Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`. _This can help save your proxy usage, but be careful with this option as it makes some websites never finish loading._ | ✔️ |
| google_search | Enabled by default, Scrapling will set the referer header as if this request came from a Google search for this website's domain name. | ✔️ |
| extra_headers | A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ | ✔️ |
| block_webrtc | Blocks WebRTC entirely. | ✔️ |
| page_action | Added for automation. A function that takes the `page` object and does the automation you need, then returns `page` again. | ✔️ |
| addons | List of Firefox addons to use. **Must be paths to extracted addons.** | ✔️ |
| humanize | Humanize the cursor movement. The cursor movement takes either True or the MAX duration in seconds. The cursor typically takes up to 1.5 seconds to move across the window. | ✔️ |
| allow_webgl | Enabled by default. Disabling WebGL is not recommended, as many WAFs now check if WebGL is enabled. | ✔️ |
| geoip | Recommended to use with proxies; Automatically use IP's longitude, latitude, timezone, country, locale, & spoof the WebRTC IP address. It will also calculate and spoof the browser's language based on the distribution of language speakers in the target region. | ✔️ |
| os_randomize | If enabled, Scrapling will randomize the OS fingerprints used. The default is matching the fingerprints with the current OS. | ✔️ |
| disable_ads | Disabled by default; this installs the `uBlock Origin` addon on the browser if enabled. | ✔️ |
| network_idle | Wait for the page until there are no network connections for at least 500 ms. | ✔️ |
| timeout | The timeout used in all operations and waits through the page. It's in milliseconds, and the default is 30000. | ✔️ |
| wait | The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the `Response` object. | ✔️ |
| wait_selector | Wait for a specific css selector to be in a specific state. | ✔️ |
| wait_selector_state | Scrapling will wait for the given state to be fulfilled for the selector given with `wait_selector`. _Default state is `attached`._ | ✔️ |
| proxy | The proxy to be used with requests. It can be a string or a dictionary with the keys 'server', 'username', and 'password' only. | ✔️ |
| additional_arguments | Arguments passed to Camoufox as additional settings that take higher priority than Scrapling's. | ✔️ |
| Argument | Description | Optional |
|:-------------------:|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:--------:|
| url | Target url | ❌ |
| headless | Pass `True` to run the browser in headless/hidden (**default**) or `False` for headful/visible mode. | ✔️ |
| block_images | Prevent the loading of images through Firefox preferences. _This can help save your proxy usage, but be cautious with this option, as it may cause some websites to never finish loading._ | ✔️ |
| disable_resources | Drop requests of unnecessary resources for a speed boost. It depends, but it made requests ~25% faster in my tests for some websites.<br/>Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`. _This can help save your proxy usage, but be cautious with this option, as it may cause some websites to never finish loading._ | ✔️ |
| cookies | Set cookies for the next request. | ✔️ |
| google_search | Enabled by default, Scrapling will set the referer header as if this request came from a Google search of this website's domain name. | ✔️ |
| extra_headers | A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ | ✔️ |
| block_webrtc | Blocks WebRTC entirely. | ✔️ |
| page_action | Added for automation. Pass a function that takes the `page` object and does the necessary automation, then returns `page` again. | ✔️ |
| addons | List of Firefox addons to use. **Must be paths to extracted addons.** | ✔️ |
| humanize | Humanize the cursor movement. The cursor movement takes either True or the maximum duration in seconds. The cursor typically takes up to 1.5 seconds to move across the window. | ✔️ |
| allow_webgl | Enabled by default. Disabling WebGL is not recommended, as many WAFs now check if WebGL is enabled. | ✔️ |
| geoip | Recommended to use with proxies; Automatically use IP's longitude, latitude, timezone, country, locale, & spoof the WebRTC IP address. It will also calculate and spoof the browser's language based on the distribution of language speakers in the target region. | ✔️ |
| os_randomize | If enabled, Scrapling will randomize the OS fingerprints used. The default is matching the fingerprints with the current OS. | ✔️ |
| disable_ads | Disabled by default; this installs the `uBlock Origin` addon on the browser if enabled. | ✔️ |
| solve_cloudflare | When enabled, fetcher solves all three types of Cloudflare's Turnstile wait/captcha page before returning the response to you. | ✔️ |
| network_idle | Wait for the page until there are no network connections for at least 500 ms. | ✔️ |
| timeout | The timeout used in all operations and waits through the page. It's in milliseconds, and the default is 30000. | ✔️ |
| wait | The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the `Response` object. | ✔️ |
| wait_selector | Wait for a specific css selector to be in a specific state. | ✔️ |
| wait_selector_state | Scrapling will wait for the given state to be fulfilled for the selector given with `wait_selector`. _Default state is `attached`._ | ✔️ |
| proxy | The proxy to be used with requests. It can be a string or a dictionary with the keys 'server', 'username', and 'password' only. | ✔️ |
| additional_args | Additional arguments to be passed to Camoufox as additional settings, and they take higher priority than Scrapling's settings. | ✔️ |
| selector_config | A dictionary of custom parsing arguments to be used when creating the final `Selector`/`Response` class. | ✔️ |
## Examples
It's easier to understand with examples, so now we will go over most of the arguments individually with examples.
It's easier to understand with examples, so we will now review most of the arguments individually with examples.
### Browser Modes
@@ -55,9 +58,6 @@ It's easier to understand with examples, so now we will go over most of the argu
# Headless/hidden mode (default)
page = StealthyFetcher.fetch('https://example.com', headless=True)
# Virtual display mode (requires having `xvfb` installed)
page = StealthyFetcher.fetch('https://example.com', headless='virtual')
# Visible browser mode
page = StealthyFetcher.fetch('https://example.com', headless=False)
```
@@ -72,6 +72,37 @@ page = StealthyFetcher.fetch('https://example.com', block_images=True)
page = StealthyFetcher.fetch('https://example.com', disable_resources=True) # Blocks fonts, images, media, etc.
```
### Cloudflare Protection Bypass
```python
# Automatic Cloudflare solver
page = StealthyFetcher.fetch(
'https://nopecha.com/demo/cloudflare',
solve_cloudflare=True # Automatically solve Cloudflare challenges
)
# Works with other stealth options
page = StealthyFetcher.fetch(
'https://protected-site.com',
solve_cloudflare=True,
humanize=True,
geoip=True,
os_randomize=True
)
```
The `solve_cloudflare` parameter enables automatic detection and solving all three types of Cloudflare's Turnstile challenges:
- JavaScript challenges (managed)
- Interactive challenges (clicking verification boxes)
- Invisible challenges (automatic background verification)
**Important notes:**
- When `solve_cloudflare=True` is enabled, `humanize=True` is automatically activated for more realistic behavior
- The timeout should be at least 60 seconds when using Cloudflare solver for sufficient challenge-solving time
- This feature works seamlessly with proxies and other stealth options
### Additional stealth options
```python
@@ -79,7 +110,7 @@ page = StealthyFetcher.fetch(
'https://example.com',
block_webrtc=True, # Block WebRTC
allow_webgl=False, # Disable WebGL
humanize=True, # Make the mouse move as how a human would move it
humanize=True, # Make the mouse move as a human would move it
geoip=True, # Use IP's longitude, latitude, timezone, country, and locale, then spoof the WebRTC IP address...
os_randomize=True, # Randomize the OS fingerprints used. The default is matching the fingerprints with the current OS.
disable_ads=True, # Block ads with uBlock Origin addon (enabled by default)
@@ -93,7 +124,7 @@ page = StealthyFetcher.fetch(
)
```
The `google_search` argument is enabled by default. It makes the request as if it came from Google, so for a request for `https://example.com`, it will set the referer to `https://www.google.com/search?q=example`. Also, if used together, it takes priority over the referer set by the `extra_headers` argument.
The `google_search` argument is enabled by default, making the request look as if it came from a Google search page. So, a request for `https://example.com` will set the referer to `https://www.google.com/search?q=example`. Also, if used together, it takes priority over the referer set by the `extra_headers` argument.
### Network Control
@@ -112,9 +143,9 @@ page = StealthyFetcher.fetch(
```
### Browser Automation
This is where your knowledge about [PlayWright's Page API](https://playwright.dev/python/docs/api/class-page) comes into play. The function you pass here takes the page object from Playwright's API, does what you want, and then returns it again for the current fetcher to continue working on it.
This is where your knowledge about [Playwright's Page API](https://playwright.dev/python/docs/api/class-page) comes into play. The function you pass here takes the page object from Playwright's API, performs the desired action, and then returns it for the current fetcher to continue processing.
This function is executed right after waiting for `network_idle` (if enabled) and before waiting for the `wait_selector` argument, so it can be used for many things, not just automation. You can alter the page as you want.
This function is executed immediately after waiting for `network_idle` (if enabled) and before waiting for the `wait_selector` argument, allowing it to be used for various purposes, not just automation. You can alter the page as you want.
In the example below, I used page [mouse events](https://playwright.dev/python/docs/api/class-mouse) to move the mouse wheel to scroll the page and then move the mouse.
```python
@@ -158,14 +189,14 @@ page = StealthyFetcher.fetch(
```
This is the last wait the fetcher will do before returning the response (if enabled). You pass a CSS selector to the `wait_selector` argument, and the fetcher will wait for the state you passed in the `wait_selector_state` argument to be fulfilled. If you didn't pass a state, the default would be `attached`, which means it will wait for the element to be present in the DOM.
After that, the fetcher will check again to see if all JS files are loaded and executed (the `domcontentloaded` state) and wait for them to be. If you have enabled `network_idle` with this, the fetcher will wait for `network_idle` to be fulfilled again, as explained above.
After that, the fetcher will check again to see if all JS files are loaded and executed (the `domcontentloaded` state) or continue waiting. If you have enabled `network_idle`, the fetcher will wait for `network_idle` to be fulfilled again, as explained above.
The states the fetcher can wait for can be either ([source](https://playwright.dev/python/docs/api/class-page#page-wait-for-selector)):
The states the fetcher can wait for can be any of the following ([source](https://playwright.dev/python/docs/api/class-page#page-wait-for-selector)):
- `attached`: wait for the element to be present in DOM.
- `detached`: wait for the element to not be present in DOM.
- `visible`: wait for the element to have a non-empty bounding box and no `visibility:hidden`. Note that an element without any content or with `display:none` has an empty bounding box and is not considered visible.
- `hidden`: Wait for the element to be detached from DOM, have an empty bounding box, or have `visibility:hidden`. This is opposite to the `'visible'` option.
- `attached`: Wait for an element to be present in the DOM.
- `detached`: Wait for an element to not be present in the DOM.
- `visible`: wait for an element to have a non-empty bounding box and no `visibility:hidden`. Note that an element without any content or with `display:none` has an empty bounding box and is not considered visible.
- `hidden`: wait for an element to be either detached from the DOM, or have an empty bounding box, or `visibility:hidden`. This is opposite to the `'visible'` option.
### Firefox Addons
@@ -179,7 +210,7 @@ page = StealthyFetcher.fetch(
The paths here must be paths of extracted addons, which will be installed automatically upon browser launch.
### Real-world example (Amazon)
This is for educational purposes only; this example was generated by AI, which shows too how easy it is to work with Scrapling through AI
This is for educational purposes only; this example was generated by AI, which shows how easy it is to work with Scrapling through AI
```python
def scrape_amazon_product(url):
# Use StealthyFetcher to bypass protection
@@ -201,6 +232,62 @@ def scrape_amazon_product(url):
}
```
## Session Management
To keep the browser open until you make multiple requests with the same configuration, use `StealthySession`/`AsyncStealthySession` classes. Those classes can accept all the arguments that the `fetch` function can take, which enables you to specify a config for the entire session.
```python
from scrapling.fetchers import StealthySession
# Create a session with default configuration
with StealthySession(
headless=True,
geoip=True,
humanize=True,
solve_cloudflare=True
) as session:
# Make multiple requests with the same browser instance
page1 = session.fetch('https://example1.com')
page2 = session.fetch('https://example2.com')
page3 = session.fetch('https://nopecha.com/demo/cloudflare')
# All requests reuse the same tab on the same browser instance
```
### Async Session Usage
```python
import asyncio
from scrapling.fetchers import AsyncStealthySession
async def scrape_multiple_sites():
async with AsyncStealthySession(
geoip=True,
os_randomize=True,
solve_cloudflare=True,
timeout=60000, # 60 seconds for Cloudflare challenges
max_pages=3
) as session:
# Make async requests with shared browser configuration
pages = await asyncio.gather(
session.fetch('https://site1.com'),
session.fetch('https://site2.com'),
session.fetch('https://protected-site.com')
)
return pages
```
You may have noticed the `max_pages` argument. This is a new argument that enables the fetcher to create a **pool of Browser tabs** that will be rotated automatically. Instead of waiting for one browser tab to become ready, it checks if the next tab in the pool is ready to be used and uses it. This allows for multiple websites to be fetched at the same time in the same browser, which saves a lot of resources, but most importantly, is so fast :)
When all tabs inside the pool are busy, the fetcher checks every subsecond if a tab becomes ready. If none become free within a 30-second interval, it raises a `TimeoutError` error. This can happen when the website you are fetching becomes unresponsive for some reason.
### Session Benefits
- **Browser reuse**: Much faster subsequent requests by reusing the same browser instance.
- **Cookie persistence**: Automatic cookie and session state handling as any browser does automatically.
- **Consistent fingerprint**: Same browser fingerprint across all requests.
- **Memory efficiency**: Better resource usage compared to launching new browsers with each fetch.
## When to Use
Use StealthyFetcher when:
@@ -209,4 +296,5 @@ Use StealthyFetcher when:
- Need a reliable browser fingerprint
- Full JavaScript support needed
- Want automatic stealth features
- Need browser automation
- Need browser automation
- Dealing with Cloudflare protection
docs/index.md
+64 -38
View File
@@ -4,29 +4,32 @@
}
</style>
<p align="center">
<div align="center">
<a href="https://scrapling.readthedocs.io/en/latest/" alt="poster">
<img alt="poster" src="assets/poster.png" style="width: 50%; height: 100%;"></a>
</p>
</div>
Scrapling is an Undetectable, high-performance, intelligent Web scraping library for Python 3 to make Web Scraping easy!
<div align="center">
<i><code>Easy, effortless Web Scraping as it should be!</code></i>
<br/><br/>
</div>
Scrapling isn't only about making undetectable requests or fetching pages under the radar!
**Stop fighting anti-bot systems. Stop rewriting selectors after every website update.**
It has its own parser that adapts to website changes and provides many element selection/querying options other than traditional selectors, powerful DOM traversal API, and many other features while significantly outperforming popular parsing alternatives.
Scrapling isn't just another Web Scraping library. It's the first **adaptive** scraping library that learns from website changes and evolves with them. While other libraries break when websites update their structure, Scrapling automatically relocates your elements and keeps your scrapers running.
Scrapling is built from the ground up by Web scraping experts for beginners and experts. The goal is to provide powerful features while maintaining simplicity and minimal boilerplate code.
Built for the modern Web, Scrapling has its own rapid parsing engine and its fetchers to handle all Web Scraping challenges you are facing or will face. Built by Web Scrapers for Web Scrapers and regular users, there's something for everyone.
```python
>> from scrapling.fetchers import Fetcher, AsyncFetcher, StealthyFetcher, PlayWrightFetcher
>> StealthyFetcher.auto_match = True
>> from scrapling.fetchers import Fetcher, AsyncFetcher, StealthyFetcher, DynamicFetcher
>> StealthyFetcher.adaptive = True
# Fetch websites' source under the radar!
>> page = StealthyFetcher.fetch('https://example.com', headless=True, network_idle=True)
>> print(page.status)
200
>> products = page.css('.product', auto_save=True) # Scrape data that survives website design changes!
>> # Later, if the website structure changes, pass `auto_match=True`
>> products = page.css('.product', auto_match=True) # and Scrapling still finds them!
>> # Later, if the website structure changes, pass `adaptive=True`
>> products = page.css('.product', adaptive=True) # and Scrapling still finds them!
```
## Top Sponsors
@@ -38,31 +41,38 @@ Scrapling is built from the ground up by Web scraping experts for beginners and
</div>
<!-- /sponsors -->
<i><sub>Do you want to show your ad here? Click [here](https://github.com/sponsors/D4Vinci) and choose the tier that suites you!</sub></i>
<i><sub>Do you want to show your ad here? Click [here](https://github.com/sponsors/D4Vinci/sponsorships?tier_id=435495) and enjoy the rest of the perks!</sub></i>
## Key Features
### Fetch websites as you prefer with async support
- **HTTP Requests**: Fast and stealthy HTTP requests with the `Fetcher` class.
- **Dynamic Loading & Automation**: Fetch dynamic websites with the `PlayWrightFetcher` class through your real browser, Scrapling's stealth mode, Playwright's Chromium browser, or [NSTbrowser](https://app.nstbrowser.io/r/1vO5e5)'s browserless!
- **Anti-bot Protections Bypass**: Easily bypass protections with the `StealthyFetcher` and `PlayWrightFetcher` classes.
### Easy Scraping
- **Smart Element Tracking**: Relocate elements after website changes using an intelligent similarity system and integrated storage.
- **Flexible Selection**: CSS selectors, XPath selectors, filters-based search, text search, regex search, and more.
- **Find Similar Elements**: Automatically locate elements similar to the element you found!
- **Smart Content Scraping**: Extract data from multiple websites without specific selectors using Scrapling powerful features.
### Advanced Websites Fetching with Session Support
- **HTTP Requests**: Fast and stealthy HTTP requests with the `Fetcher` class. Can impersonate browsers' TLS fingerprint, headers, and use HTTP/3.
- **Dynamic Loading**: Fetch dynamic websites with full browser automation through the `DynamicFetcher` class supporting Playwright's Chromium, real Chrome, and custom stealth mode.
- **Anti-bot Bypass**: Advanced stealth capabilities with `StealthyFetcher` using a modified version of Firefox and fingerprint spoofing. Can bypass all levels of Cloudflare's Turnstile with automation easily.
- **Session Management**: Persistent session support with `FetcherSession`, `StealthySession`, and `DynamicSession` classes for cookie and state management across requests.
- **Async Support**: Complete async support across all fetchers and dedicated async session classes.
### High Performance
- **Lightning Fast**: Built from the ground up with performance in mind, outperforming most popular Python scraping libraries.
- **Memory Efficient**: Optimized data structures for minimal memory footprint.
- **Fast JSON serialization**: 10x faster than standard library.
### Adaptive Scraping & AI Integration
- 🔄 **Smart Element Tracking**: Relocate elements after website changes using intelligent similarity algorithms.
- 🎯 **Smart Flexible Selection**: CSS selectors, XPath selectors, filter-based search, text search, regex search, and more.
- 🔍 **Find Similar Elements**: Automatically locate elements similar to found elements.
- 🤖 **MCP Server to be used with AI**: Built-in MCP server for AI-assisted Web Scraping and data extraction. The MCP server features custom, powerful capabilities that utilize Scrapling to extract targeted content before passing it to the AI (Claude/Cursor/etc), thereby speeding up operations and reducing costs by minimizing token usage.
### High-Performance & battle-tested Architecture
- 🚀 **Lightning Fast**: Optimized performance outperforming most Python scraping libraries.
- 🔋 **Memory Efficient**: Optimized data structures and lazy loading for a minimal memory footprint.
- ⚡ **Fast JSON Serialization**: 10x faster than the standard library.
- 🏗️ **Battle tested**: Not only does Scrapling have 92% test coverage and full type hints coverage, but it has been used daily by hundreds of Web Scrapers over the past year.
### Developer/Web Scraper Friendly Experience
- 🎯 **Interactive Web Scraping Shell**: Optional built-in IPython shell with Scrapling integration, shortcuts, and new tools to speed up Web Scraping scripts development, like converting curl requests to Scrapling requests and viewing requests results in your browser.
- 🚀 **Use it directly from the Terminal**: Optionally, you can use Scrapling to scrape a URL without writing a single code!
- 🛠️ **Rich Navigation API**: Advanced DOM traversal with parent, sibling, and child navigation methods.
- 🧬 **Enhanced Text Processing**: Built-in regex, cleaning methods, and optimized string operations.
- 📝 **Auto Selector Generation**: Generate robust CSS/XPath selectors for any element.
- 🔌 **Familiar API**: Similar to Scrapy/BeautifulSoup with the same pseudo-elements used in Scrapy/Parsel.
- 📘 **Complete Type Coverage**: Full type hints for excellent IDE support and code completion.
### Developer Friendly
- **Powerful Navigation API**: Easy DOM traversal in all directions.
- **Rich Text Processing**: All strings have built-in regex, cleaning methods, and more. All elements' attributes are optimized dictionaries that use less memory than standard dictionaries with added methods.
- **Auto Selectors Generation**: Generate robust short and full CSS/XPath selectors for any element.
- **Familiar API**: Similar to Scrapy/BeautifulSoup and the same CSS pseudo-elements used in Scrapy.
- **Type hints**: Complete type/doc-strings coverage for future-proofing and best autocompletion support.
## Star History
Scrapling’s GitHub stars have grown steadily since its release (see chart below).
@@ -98,19 +108,35 @@ observer.observe(document.body, {
</script>
## Installation
Scrapling is a breeze to get started with!<br/>Starting from version 0.2.9, we require at least Python 3.9 to work.
Scrapling requires Python 3.10 or higher:
Run this command to install it with Python's pip.
```bash
pip3 install scrapling
pip install scrapling
```
You are ready if you plan to use the parser only (the `Adaptor` class).
But if you are going to make requests or fetch pages with Scrapling, then run this command to install browsers' dependencies needed to use the Fetchers
#### Fetchers Setup
If you are going to use any of the fetchers or their session classes, then install browser dependencies with
```bash
scrapling install
```
If you have any installation issues, please open an [issue](https://github.com/D4Vinci/Scrapling/issues/new/choose).
This downloads all browsers with their system dependencies and fingerprint manipulation dependencies.
### Optional Dependencies
- Install the MCP server feature:
```bash
pip install "scrapling[ai]"
```
- Install shell features (Web Scraping shell and the `extract` command):
```bash
pip install "scrapling[shell]"
```
- Install everything:
```bash
pip install "scrapling[all]"
```
## How the documentation is organized
Scrapling has a lot of documentation, so we try to follow a guideline called the [Diátaxis documentation framework](https://diataxis.fr/).
@@ -121,9 +147,9 @@ If you like Scrapling and want to support its development:
- ⭐ Star the [GitHub repository](https://github.com/D4Vinci/Scrapling)
- 🚀 Follow us on [Twitter](https://x.com/Scrapling_dev) and join the [discord server](https://discord.gg/EMgGbDceNQ)
- 💝 Consider [sponsoring the project or buying me a coffe](donate.md) :wink:
- 💝 Consider [sponsoring the project or buying me a coffee](donate.md) :wink:
- 🐛 Report bugs and suggest features through [GitHub Issues](https://github.com/D4Vinci/Scrapling/issues)
## License
This project is licensed under BSD-3 License. See the [LICENSE](https://github.com/D4Vinci/Scrapling/blob/main/LICENSE) file for details.
This project is licensed under the BSD-3 License. See the [LICENSE](https://github.com/D4Vinci/Scrapling/blob/main/LICENSE) file for details.
docs/overview.md
+45 -50
View File
@@ -1,6 +1,6 @@
We will start by quickly reviewing the parsing capabilities. Then, we will fetch websites with custom browsers, make requests, and parse the response.
Here's an HTML document generated by ChatGPT we will be using as an example throughout this page:
Here's an HTML document generated by ChatGPT that we will be using as an example throughout this page:
```html
<html>
<head>
@@ -71,8 +71,8 @@ Here's an HTML document generated by ChatGPT we will be using as an example thro
```
Starting with loading raw HTML above like this
```python
from scrapling.parser import Adaptor
page = Adaptor(html_doc)
from scrapling.parser import Selector
page = Selector(html_doc)
page # <data='<html><head><title>Complex Web Page</tit...'>
```
Get all text content on the page recursively
@@ -101,7 +101,7 @@ section_elements = page.find_all('section', {'id':"products"})
section_elements = page.find_all('section', id="products")
# [<data='<section id="products" schema='{"jsonabl...' parent='<main><section id="products" schema='{"j...'>]
```
Find all `section` elements that its `id` attribute value contains `product`
Find all `section` elements whose `id` attribute value contains `product`
```python
section_elements = page.find_all('section', {'id*':"product"})
```
@@ -110,12 +110,12 @@ Find all `h3` elements whose text content matches this regex `Product \d`
page.find_all('h3', re.compile(r'Product \d'))
# [<data='<h3>Product 1</h3>' parent='<article class="product" data-id="1"><h3...'>, <data='<h3>Product 2</h3>' parent='<article class="product" data-id="2"><h3...'>, <data='<h3>Product 3</h3>' parent='<article class="product" data-id="3"><h3...'>]
```
Find all `h3` and `h2` elements whose text content matches regex `Product` only
Find all `h3` and `h2` elements whose text content matches the regex `Product` only
```python
page.find_all(['h3', 'h2'], re.compile(r'Product'))
# [<data='<h3>Product 1</h3>' parent='<article class="product" data-id="1"><h3...'>, <data='<h3>Product 2</h3>' parent='<article class="product" data-id="2"><h3...'>, <data='<h3>Product 3</h3>' parent='<article class="product" data-id="3"><h3...'>, <data='<h2>Products</h2>' parent='<section id="products" schema='{"jsonabl...'>]
```
Find all elements that its text content matches exactly `Products` (Whitespaces are not taken into consideration)
Find all elements whose text content matches exactly `Products` (Whitespaces are not taken into consideration)
```python
page.find_by_text('Products', first_match=False)
# [<data='<h2>Products</h2>' parent='<section id="products" schema='{"jsonabl...'>]
@@ -225,12 +225,12 @@ Using the elements we found above
>>> page.css_first('[data-id="1"]').has_class('product')
True
```
If your case needs more than the element's parent, you can iterate over the whole ancestors' tree of any element like the one below
If your case needs more than the element's parent, you can iterate over the whole ancestors' tree of any element, like the one below
```python
for ancestor in quote.iterancestors():
# do something with it...
```
You can search for a specific ancestor of an element that satisfies a function; all you need to do is to pass a function that takes an `Adaptor` object as an argument and return `True` if the condition satisfies or `False` otherwise like below:
You can search for a specific ancestor of an element that satisfies a function; all you need to do is pass a function that takes a `Selector` object as an argument and returns `True` if the condition is satisfied or `False` otherwise, like below:
```python
>>> section_element.find_ancestor(lambda ancestor: ancestor.css('nav'))
<data='<body> <header><nav><ul><li> <a href="#h...' parent='<html><head><title>Complex Web Page</tit...'>
@@ -242,86 +242,81 @@ Instead of passing the raw HTML to Scrapling, you can get a website's response d
A fetcher is made for every use case.
### HTTP Requests
For simple HTTP requests, there's a `Fetcher` class that can be imported as below:
For simple HTTP requests, there's a `Fetcher` class that can be imported and used as below:
```python
from scrapling.fetchers import Fetcher
```
But that's class, so you will need to create an instance of the Fetcher first like this:
```python
from scrapling.fetchers import Fetcher
fetcher = Fetcher()
page = fetcher.get('https://httpbin.org/get')
```
This is intended, and you will find it with all fetchers because there are settings you can pass to `Fetcher()` initialization, but more on this later.
If you are going to use the default settings anyway, you can do this instead for a cleaner approach:
```python
from scrapling.fetchers import Fetcher
page = Fetcher.get('https://httpbin.org/get')
page = Fetcher.get('https://scrapling.requestcatcher.com/get', impersonate="chrome")
```
With that out of the way, here's how to do all HTTP methods:
```python
>>> from scrapling.fetchers import Fetcher
>>> page = Fetcher.get('https://httpbin.org/get', stealthy_headers=True, follow_redirects=True)
>>> page = Fetcher.post('https://httpbin.org/post', data={'key': 'value'}, proxy='http://username:password@localhost:8030')
>>> page = Fetcher.put('https://httpbin.org/put', data={'key': 'value'})
>>> page = Fetcher.delete('https://httpbin.org/delete')
>>> page = Fetcher.get('https://scrapling.requestcatcher.com/get', stealthy_headers=True, follow_redirects=True)
>>> page = Fetcher.post('https://scrapling.requestcatcher.com/post', data={'key': 'value'}, proxy='http://username:password@localhost:8030')
>>> page = Fetcher.put('https://scrapling.requestcatcher.com/put', data={'key': 'value'})
>>> page = Fetcher.delete('https://scrapling.requestcatcher.com/delete')
```
For Async requests, you will just replace the import like below:
For Async requests, you will replace the import like below:
```python
>>> from scrapling.fetchers import AsyncFetcher
>>> page = await AsyncFetcher.get('https://httpbin.org/get', stealthy_headers=True, follow_redirects=True)
>>> page = await AsyncFetcher.post('https://httpbin.org/post', data={'key': 'value'}, proxy='http://username:password@localhost:8030')
>>> page = await AsyncFetcher.put('https://httpbin.org/put', data={'key': 'value'})
>>> page = await AsyncFetcher.delete('https://httpbin.org/delete')
>>> page = await AsyncFetcher.get('https://scrapling.requestcatcher.com/get', stealthy_headers=True, follow_redirects=True)
>>> page = await AsyncFetcher.post('https://scrapling.requestcatcher.com/post', data={'key': 'value'}, proxy='http://username:password@localhost:8030')
>>> page = await AsyncFetcher.put('https://scrapling.requestcatcher.com/put', data={'key': 'value'})
>>> page = await AsyncFetcher.delete('https://scrapling.requestcatcher.com/delete')
```
> Note: You have the `stealthy_headers` argument, which, when enabled, makes requests to generate real browser headers and use them, including a referer header, as if this request came from Google's search of this URL's domain. It's enabled by default.
> Notes:
>
> 1. You have the `stealthy_headers` argument, which, when enabled, makes requests to generate real browser headers and use them, including a referer header, as if this request came from a Google search of this domain. It's enabled by default.
> 2. The `impersonate` argument allows you to fake the TLS fingerprint for a specific version of a browser.
> 3. There's also the `http3` argument, which, when enabled, makes the fetcher use HTTP/3 for requests, which makes your requests more authentic
This is just the tip of this fetcher; check the full page from [here](fetching/static.md)
This is just the tip of the iceberg with this fetcher; check out the rest from [here](fetching/static.md)
### Dynamic loading
We have you covered if you deal with dynamic websites like most today!
The `PlayWrightFetcher` class provides many options to fetch/load websites' pages through browsers.
The `DynamicFetcher` class (previously known as `PlayWrightFetcher`) provides many options to fetch/load websites' pages through browsers.
```python
>>> from scrapling.fetchers import PlayWrightFetcher
>>> page = PlayWrightFetcher.fetch('https://www.google.com/search?q=%22Scrapling%22', disable_resources=True) # Vanilla Playwright option
>>> from scrapling.fetchers import DynamicFetcher
>>> page = DynamicFetcher.fetch('https://www.google.com/search?q=%22Scrapling%22', disable_resources=True) # Vanilla Playwright option
>>> page.css_first("#search a::attr(href)")
'https://github.com/D4Vinci/Scrapling'
>>> # The async version of fetch
>>> page = await PlayWrightFetcher.async_fetch('https://www.google.com/search?q=%22Scrapling%22', disable_resources=True)
>>> page = await DynamicFetcher.async_fetch('https://www.google.com/search?q=%22Scrapling%22', disable_resources=True)
>>> page.css_first("#search a::attr(href)")
'https://github.com/D4Vinci/Scrapling'
```
It's named like that because it's built on top of [Playwright](https://playwright.dev/python/), and it currently provides 4 main run options that can be mixed as you want:
It's built on top of [Playwright](https://playwright.dev/python/) and it's currently providing three main run options that can be mixed as you want:
- Vanilla Playwright without any modifications other than the ones you chose.
- Stealthy Playwright with custom stealth mode explicitly written for it. It's not top-tier stealth mode but bypasses many online tests like [Sannysoft's](https://bot.sannysoft.com/). Check out the `StealthyFetcher` class below for more advanced stealth mode.
- Real browsers by passing the `real_chrome` argument or the CDP URL of your browser to be controlled by the Fetcher, and most of the options can be enabled on it.
- [NSTBrowser](https://app.nstbrowser.io/r/1vO5e5)'s [docker browserless](https://hub.docker.com/r/nstbrowser/browserless) option by passing the CDP URL and enabling `nstbrowser_mode` option.
- Vanilla Playwright without any modifications other than the ones you chose. It uses the Chromium browser.
- Stealthy Playwright with custom stealth mode explicitly written for it. It's not top-tier stealth mode, but it bypasses many online tests like [Sannysoft's](https://bot.sannysoft.com/). Check out the `StealthyFetcher` class below for more advanced stealth mode. It uses the Chromium browser.
- Real browsers like your Chrome browser by passing the `real_chrome` argument or the CDP URL of your browser to be controlled by the Fetcher, and most of the options can be enabled on it.
> Note: All requests done by this fetcher are waited by default for all javascript to be fully loaded and executed. In detail, it waits for the `load` and `domcontentloaded` load states to be reached; you can make it wait for the `networkidle` load state by passing 'network_idle=True', as you will see later.
> Note: All requests done by this fetcher are waiting by default for all JavaScript to be fully loaded and executed. In detail, it waits for the `load` and `domcontentloaded` load states to be reached; you can make it wait for the `networkidle` load state by passing `network_idle=True`, as you will see later.
Again, this is just the tip of this fetcher. Check out the full page from [here](fetching/dynamic.md) for all details and the complete list of arguments.
Again, this is just the tip of the iceberg with this fetcher. Check out the rest from [here](fetching/dynamic.md) for all details and the complete list of arguments.
### Dynamic anti-protection loading
We also have you covered if you deal with dynamic websites with annoying anti-protections!
The `StealthyFetcher` class uses a modified Firefox browser called [Camoufox](https://github.com/daijro/camoufox), bypassing most anti-bot protections by default. Scrapling adds extra layers of flavors and configurations to further increase performance and undetectability.
The `StealthyFetcher` class uses a custom version of a modified Firefox browser called [Camoufox](https://github.com/daijro/camoufox), bypassing most bot detections by default. Scrapling offers a faster custom version, includes extra tools, and features easy configurations to further increase undetectability.
```python
>>> page = StealthyFetcher().fetch('https://www.browserscan.net/bot-detection') # Running headless by default
>>> from scrapling.fetchers import StealthyFetcher
>>> page = StealthyFetcher.fetch('https://www.browserscan.net/bot-detection') # Running headless by default
>>> page.status == 200
True
>>> page = StealthyFetcher().fetch('https://www.browserscan.net/bot-detection', humanize=True, os_randomize=True) # and the rest of arguments...
>>> page = StealthyFetcher.fetch('https://nopecha.com/demo/cloudflare', solve_cloudflare=True) # Solve Cloudflare captcha automatically if presented
>>> page.status == 200
True
>>> page = StealthyFetcher.fetch('https://www.browserscan.net/bot-detection', humanize=True, os_randomize=True) # and the rest of arguments...
>>> # The async version of fetch
>>> page = await StealthyFetcher().async_fetch('https://www.browserscan.net/bot-detection')
>>> page = await StealthyFetcher.async_fetch('https://www.browserscan.net/bot-detection')
>>> page.status == 200
True
```
> Note: All requests done by this fetcher are waited by default for all javascript to be fully loaded and executed. In detail, it waits for the `load` and `domcontentloaded` load states to be reached; you can make it wait for the `networkidle` load state by passing 'network_idle=True', as you will see later.
> Note: All requests done by this fetcher are waiting by default for all JavaScript to be fully loaded and executed. In detail, it waits for the `load` and `domcontentloaded` load states to be reached; you can make it wait for the `networkidle` load state by passing `network_idle=True`, as you will see later.
Again, this is just the tip of this fetcher. Check out the full page from [here](fetching/dynamic.md) for all details and the complete list of arguments.
Again, this is just the tip of the iceberg with this fetcher. Check out the rest from [here](fetching/dynamic.md) for all details and the complete list of arguments.
---
docs/parsing/automatch.md → docs/parsing/adaptive.md
+48 -48
View File
@@ -1,5 +1,5 @@
## Introduction
Auto-matching is one of Scrapling's most powerful features. It allows your scraper to survive website changes by intelligently tracking and relocating elements.
Adaptive scraping (previously known as automatch) is one of Scrapling's most powerful features. It allows your scraper to survive website changes by intelligently tracking and relocating elements.
Let's say you are scraping a page with a structure like this:
```html
@@ -41,31 +41,31 @@ When website owners implement structural changes like
</div>
</div>
```
The selector will no longer function, and your code needs maintenance. That's where Scrapling's auto-matching feature comes into play.
The selector will no longer function, and your code needs maintenance. That's where Scrapling's `adaptive` feature comes into play.
With Scrapling, you can enable the `automatch` feature the first time you select an element, and the next time you select that element and it doesn't exist, Scrapling will remember its properties and search on the website for the element with the highest percentage of similarity to that element and without AI :)
With Scrapling, you can enable the `adaptive` feature the first time you select an element, and the next time you select that element and it doesn't exist, Scrapling will remember its properties and search on the website for the element with the highest percentage of similarity to that element and without AI :)
```python
from scrapling import Adaptor, Fetcher
from scrapling import Selector, Fetcher
# Before the change
page = Adaptor(page_source, auto_match=True, url='example.com')
page = Selector(page_source, adaptive=True, url='example.com')
# or
Fetcher.auto_match = True
Fetcher.adaptive = True
page = Fetcher.get('https://example.com')
# then
element = page.css('#p1' auto_save=True)
element = page.css('#p1', auto_save=True)
if not element: # One day website changes?
element = page.css('#p1', auto_match=True) # Scrapling still finds it!
element = page.css('#p1', adaptive=True) # Scrapling still finds it!
# the rest of your code...
```
Below, I will show you one usage example for this feature. Then, we will dive deep into how to use it and provide details about this feature.
Below, I will show you one usage example for this feature. Then, we will dive deep into how to use it and provide details about this feature. Note that it works with all selection methods, not just CSS/XPATH selection.
## Real-World Scenario
Let's use a real website as an example and use one of the fetchers to fetch its source. To do this, we need to find a website that will soon change its design/structure, take a copy of its source, and then wait for the website to make the change. Of course, that's nearly impossible to know unless I know the website's owner, but that will make it a staged test, haha.
Let's use a real website as an example and use one of the fetchers to fetch its source. To achieve this, we need to identify a website that is about to update its design/structure, copy its source, and then wait for the website to change. Of course, that's nearly impossible to know unless I know the website's owner, but that will make it a staged test, haha.
To solve this issue, I will use [The Web Archive](https://archive.org/)'s [Wayback Machine](https://web.archive.org/). Here is a copy of [StackOverFlow's website in 2010](https://web.archive.org/web/20100102003420/http://stackoverflow.com/); pretty old, eh?</br>Let's test if the automatch feature can extract the same button in the old design from 2010 and the current design using the same selector :)
To solve this issue, I will use [The Web Archive](https://archive.org/)'s [Wayback Machine](https://web.archive.org/). Here is a copy of [StackOverFlow's website in 2010](https://web.archive.org/web/20100102003420/http://stackoverflow.com/); pretty old, eh?</br>Let's see if the adaptive feature can extract the same button in the old design from 2010 and the current design using the same selector :)
If I want to extract the Questions button from the old design, I can use a selector like this: `#hmenus > div:nth-child(1) > ul > li:nth-child(1) > a` This selector is too specific because it was generated by Google Chrome.
If I want to extract the Questions button from the old design, I can use a selector like this: `#hmenus > div:nth-child(1) > ul > li:nth-child(1) > a`. This selector is too specific because it was generated by Google Chrome.
Now, let's test the same selector in both versions
@@ -74,48 +74,48 @@ Now, let's test the same selector in both versions
>> selector = '#hmenus > div:nth-child(1) > ul > li:nth-child(1) > a'
>> old_url = "https://web.archive.org/web/20100102003420/http://stackoverflow.com/"
>> new_url = "https://stackoverflow.com/"
>> Fetcher.configure(auto_match = True, automatch_domain='stackoverflow.com')
>> Fetcher.configure(adaptive = True, adaptive_domain='stackoverflow.com')
>>
>> page = Fetcher.get(old_url, timeout=30)
>> element1 = page.css_first(selector, auto_save=True)
>>
>> # Same selector but used in the updated website
>> page = Fetcher.get(new_url)
>> element2 = page.css_first(selector, auto_match=True)
>> element2 = page.css_first(selector, adaptive=True)
>>
>> if element1.text == element2.text:
... print('Scrapling found the same element in the old and new designs!')
'Scrapling found the same element in the old and new designs!'
```
Note that I used a new argument called `automatch_domain`; this is because, for Scrapling, these are two different domains(`archive.org` and `stackoverflow.com`), so scrapling will isolate their `auto_match` data. To tell Scrapling they are the same website, we need to pass the custom domain we want to use while saving auto-match data for them both so Scrapling doesn't isolate them.
Note that I introduced a new argument called `adaptive_domain`. This is because, for Scrapling, these are two different domains (`archive.org` and `stackoverflow.com`), so Scrapling will isolate their `adaptive` data. To inform Scrapling that they are the same website, we must pass the custom domain we wish to use while saving `adaptive` data for both, ensuring Scrapling doesn't isolate them.
The code will be the same in a real-world scenario, except it will use the same URL for both requests, so you won't need to use the `automatch_domain` argument. This is the closest example I can give to real-world cases, so I hope it didn't confuse you :)
The code will be the same in a real-world scenario, except it will use the same URL for both requests, so you won't need to use the `adaptive_domain` argument. This is the closest example I can give to real-world cases, so I hope it didn't confuse you :)
Hence, in the two examples above, I used both the `Adaptor` class and the `Fetcher` class to show you that the logic for automatch is the same.
Hence, in the two examples above, I used both the `Selector` class and the `Fetcher` class to show you that the logic for adaptive is the same.
## How the automatch feature works
Auto-matching works in two phases:
## How the adaptive scraping feature works
Adaptive scraping works in two phases:
1. **Save Phase**: Store unique properties of elements
2. **Match Phase**: Find elements with similar properties later
Let's say you have an element you got through selection or any method and want the library to find it the next time you scrape this website, even if it had structural/design changes.
Let's say you've got an element through selection or any method and want the library to find it the next time you scrape this website, even if it undergoes structural/design changes.
As little technical details as possible, the general logic goes as the following:
With as few technical details as possible, the general logic goes as follows:
1. You tell Scrapling to save that element's unique properties in one of the ways we will show below.
2. Scrapling uses its configured database (SQLite by default) and saves each element's unique properties.
3. Now, because everything about the element can be changed or removed from the website's owner(s), nothing from the element can be used as a unique identifier for the database. To solve this issue, I made the storage system rely on two things:
1. The domain of the current website. If you are using the `Adaptor` class, you should pass it while initializing the class, or if you are using one of the fetchers, the domain will be taken from the URL automatically.
1. The domain of the current website. If you are using the `Selector` class, you should pass it while initializing the class, or if you are using one of the fetchers, the domain will be taken from the URL automatically.
2. An `identifier` to query that element's properties from the database. You don't always have to set the identifier yourself, as you will see later when we discuss this.
Together, they will be used to retrieve the element's unique properties from the database later.
4. Later, when the website structural changes, you tell Scrapling to automatch the element. Scrapling retrieves the element's unique properties and matches all elements on the page against the unique properties we already have for this element. A score is calculated for their similarity to the element we want. In that comparison, everything is taken into consideration, as you will see later
4. Later, when the website's structure changes, you tell Scrapling to find the element by enabling `adaptive`. Scrapling retrieves the element's unique properties and matches all elements on the page against the unique properties we already have for this element. A score is calculated for their similarity with the desired element. In that comparison, everything is taken into consideration, as you will see later
5. The element(s) with the highest similarity score to the wanted element are returned.
### The unique properties
You might wonder, if all aspects of an element can be removed or changed, what unique properties we are talking about.
You might wonder what unique properties we are referring to when discussing the removal or alteration of all element properties.
For Scrapling, the unique elements we are relying on are:
@@ -124,64 +124,64 @@ For Scrapling, the unique elements we are relying on are:
But you need to understand that the comparison between elements is not exact; it's more about finding how similar these values are. So everything is considered, even the values' order, like the order in which the element class names were written before and the order in which the same element class names are written now.
## How to use automatch feature
The automatch feature can be used on any element you have, and it's added as arguments to CSS/XPath Selection methods, as you saw above, but we will get back to that later.
## How to use adaptive feature
The adaptive feature can be applied to any found element, and it's added as arguments to CSS/XPath Selection methods, as you saw above, but we will get back to that later.
First, you must enable the automatch feature by passing `auto_match=True` to the [Adaptor](main_classes.md#adaptor) class when you initialize it or enable it in the fetcher you are using of the available fetchers, as we will show.
First, you must enable the `adaptive` feature by passing `adaptive=True` to the [Selector](main_classes.md#selector) class when you initialize it or enable it in the fetcher you are using of the available fetchers, as we will show.
Examples:
```python
>>> from scrapling import Adaptor, Fetcher
>>> page = Adaptor(html_doc, auto_match=True)
>>> from scrapling import Selector, Fetcher
>>> page = Selector(html_doc, adaptive=True)
# OR
>>> Fetcher.auto_match = True
>>> Fetcher.adaptive = True
>>> page = Fetcher.fetch('https://example.com')
```
If you are using the [Adaptor](main_classes.md#adaptor) class, you need to pass the url of the website you are using with the argument `url` so Scrapling can separate the properties saved for each element by domain.
If you are using the [Selector](main_classes.md#selector) class, you need to pass the url of the website you are using with the argument `url` so Scrapling can separate the properties saved for each element by domain.
If you didn't pass a URL, the word `default` will be used in place of the URL field while saving the element's unique properties. So, this will only be an issue if you used the same identifier later for a different website and didn't pass the URL parameter while initializing it. The save process will overwrite the previous data, and auto-matching only uses the latest saved properties.
If you didn't pass a URL, the word `default` will be used in place of the URL field while saving the element's unique properties. So, this will only be an issue if you used the same identifier later for a different website and didn't pass the URL parameter while initializing it. The save process will overwrite the previous data, and the `adaptive` feature only uses the latest saved properties.
Besides those arguments, we have `storage` and `storage_args`. Both are for the class to be used to connect to the database; by default, it's set to the SQLite class that the library is using. Those arguments shouldn't matter unless you want to write your own storage system, which we will cover on a [separate page in the development section](../development/automatch_storage_system.md).
Besides those arguments, we have `storage` and `storage_args`. Both are for the class to be used to connect to the database; by default, it's set to the SQLite class that the library is using. Those arguments shouldn't matter unless you want to write your own storage system, which we will cover on a [separate page in the development section](../development/adaptive_storage_system.md).
Now, after enabling the automatch feature globally, you have two main ways to use it.
Now, after enabling the `adaptive` feature globally, you have two main ways to use it.
### The CSS/XPath Selection way
As you have seen in the example above, first, you have to use the `auto_save` argument while selecting an element that exists on the page like below
As you have seen in the example above, first, you have to use the `auto_save` argument while selecting an element that exists on the page, like below
```python
element = page.css('#p1' auto_save=True)
```
and when the element doesn't exist, you can use the same selector and the `auto_match` argument, and the library will find it for you
And when the element doesn't exist, you can use the same selector and the `adaptive` argument, and the library will find it for you
```python
element = page.css('#p1', auto_match=True)
element = page.css('#p1', adaptive=True)
```
Pretty simple, eh?
Well, a lot happened under the hood here. Remember the identifier part we mentioned before that you need to set so you can retrieve the element you want? Here, with the `css`/`css_first`/`xpath`/`xpath_first` methods, the identifier is set automatically as the selector you passed here to make things easier :)
Also, that's why here, for all these methods, you can pass the `identifier` argument to set it yourself, and there are cases for this, or you can use it to save the properties with the `auto_save` argument.
Additionally, for all these methods, you can pass the `identifier` argument to set it yourself. This is useful in some instances, or you can use it to save properties with the `auto_save` argument.
### The manual way
You manually save and retrieve an element, then relocate it, which all happens within the automatch feature, as shown below. This allows you to automatch any element you have by any way or any selection method!
You manually save and retrieve an element, then relocate it, which all happens within the `adaptive` feature, as shown below. This allows you to relocate any element using any method or selection!
First, let's say you got an element like this by text:
```python
>>> element = page.find_by_text('Tipping the Velvet', first_match=True)
```
You can save its unique properties with the `save` method like below, but you must set the identifier yourself. For this example, I chose `my_special_element` as an identifier, but it's best to use a meaningful identifier in your code for the same reason you use meaningful variable names :)
You can save its unique properties with the `save` method, like below, but you must set the identifier yourself. For this example, I chose `my_special_element` as an identifier, but it's best to use a meaningful identifier in your code for the same reason you use meaningful variable names :)
```python
>>> page.save(element, 'my_special_element')
```
Now, later, when you want to retrieve it and relocate it inside the page with auto-matching, it would be like this
Now, later, when you want to retrieve it and relocate it inside the page with `adaptive`, it would be like this
```python
>>> element_dict = page.retrieve('my_special_element')
>>> page.relocate(element_dict, adaptor_type=True)
>>> page.relocate(element_dict, selector_type=True)
[<data='<a href="catalogue/tipping-the-velvet_99...' parent='<h3><a href="catalogue/tipping-the-velve...'>]
>>> page.relocate(element_dict, adaptor_type=True).css('::text')
>>> page.relocate(element_dict, selector_type=True).css('::text')
['Tipping the Velvet']
```
Hence, the `retrieve` and relocate` methods are used.
Hence, the `retrieve` and `relocate` methods are used.
if you want to keep it as `lxml.etree` object, leave the `adaptor_type` argument
If you want to keep it as a `lxml.etree` object, leave the `selector_type` argument
```python
>>> page.relocate(element_dict)
[<Element a at 0x105a2a7b0>]
@@ -197,7 +197,7 @@ if not element_data:
print("No data saved for this identifier")
# 2. Try with different identifier
products = page.css('.product', auto_match=True, identifier='old_selector')
products = page.css('.product', adaptive=True, identifier='old_selector')
# 3. Save again with new identifier
products = page.css('.new-product', auto_save=True, identifier='new_identifier')
@@ -214,7 +214,7 @@ page.save(product, 'specific_product')
```
## Known Issues
In the auto-matching save process, the unique properties of the first element from the selection results are the only ones that get saved. So if the selector you are using selects different elements on the page in different locations, auto-matching will return the first element to you only when you relocate it later. This doesn't include combined CSS selectors (Using commas to combine more than one selector, for example), as these selectors get separated, and each selector gets executed alone.
In the `adaptive` save process, the unique properties of the first element from the selection results are the only ones that get saved. So if the selector you are using selects different elements on the page in other locations, `adaptive` will return the first element to you only when you relocate it later. This doesn't include combined CSS selectors (Using commas to combine more than one selector, for example), as these selectors get separated, and each selector gets executed alone.
## Final thoughts
Explaining this feature in detail without complications turned out to be challenging, but still, if there's something left unclear, you can head out to the [discussions section](https://github.com/D4Vinci/Scrapling/discussions), and I will reply to you ASAP or reach out to me privately and have a chat :)
Explaining this feature in detail without complications turned out to be challenging. However, still, if there's something left unclear, you can head out to the [discussions section](https://github.com/D4Vinci/Scrapling/discussions), and I will reply to you ASAP, or the Discord server, or reach out to me privately and have a chat :)
docs/parsing/main_classes.md
+92 -72
View File
@@ -1,41 +1,41 @@
## Introduction
After exploring the various ways to select elements with Scrapling and related features, Let's take a step back and examine the [Adaptor](#adaptor) class generally and other objects to better understand the parsing engine.
After exploring the various ways to select elements with Scrapling and related features, let's take a step back and examine the [Selector](#selector) class generally and other objects to better understand the parsing engine.
The [Adaptor](#adaptor) class is the core parsing engine in Scrapling that provides HTML parsing and element selection capabilities. You can always import it with any of the following imports
The [Selector](#selector) class is the core parsing engine in Scrapling that provides HTML parsing and element selection capabilities. You can always import it with any of the following imports
```python
from scrapling import Adaptor
from scrapling.parser import Adaptor
from scrapling import Selector
from scrapling.parser import Selector
```
then use it directly as you already learned in the [overview](../overview.md) page
Then use it directly as you already learned in the [overview](../overview.md) page
```python
adaptor = Adaptor(
text='<html>...</html>',
page = Selector(
'<html>...</html>',
url='https://example.com'
)
# Then select elements as you like
elements = adaptor.css('.product')
elements = page.css('.product')
```
In Scrapling, the main object you deal with after passing an HTML source or fetching a website is, of course, an [Adaptor](#adaptor) object. Any operation you do, like selection, navigation, etc., will return either an [Adaptor](#adaptor) object or an [Adaptors](#adaptors) object, given that the result is element/elements from the page, not text or similar.
In Scrapling, the main object you deal with after passing an HTML source or fetching a website is, of course, a [Selector](#selector) object. Any operation you do, like selection, navigation, etc., will return either a [Selector](#selector) object or a [Selectors](#selectors) object, given that the result is element/elements from the page, not text or similar.
In other words, the main page is a [Adaptor](#adaptor) object, and the elements within are [Adaptor](#adaptor) objects, and so on. Any text, such as the text content inside elements or the text inside element attributes, is a [TextHandler](#texthandler) object, and the attributes of each element are stored as [AttributesHandler](#attributeshandler). We will return to both objects later, so let's focus on the [Adaptor](#adaptor) object.
In other words, the main page is a [Selector](#selector) object, and the elements within are [Selector](#selector) objects, and so on. Any text, such as the text content inside elements or the text inside element attributes, is a [TextHandler](#texthandler) object, and the attributes of each element are stored as [AttributesHandler](#attributeshandler). We will return to both objects later, so let's focus on the [Selector](#selector) object.
## Adaptor
## Selector
### Arguments explained
The most important ones are `text` and `body`. Both are used to pass the HTML code you want to parse, but the first one accepts `str`, and the latter accepts `bytes` like how you used to do with `parsel` :)
The most important one is `content`, it's used to pass the HTML code you want to parse, and it accepts the HTML content as `str` or `bytes`.
Otherwise, you have the arguments `url`, `auto_match`, `storage`, and `storage_args`. All these arguments are settings used with the `auto_match` feature, and they don't make a difference if you are not going to use that feature, so just ignore them for now, and we will explain them in the [automatch](automatch.md) feature page.
Otherwise, you have the arguments `url`, `adaptive`, `storage`, and `storage_args`. All these arguments are settings used with the `adaptive` feature, and they don't make a difference if you are not going to use that feature, so just ignore them for now, and we will explain them in the [adaptive](adaptive.md) feature page.
Then you have the arguments for adjustments for parsing or adjusting/manipulating the HTML while the library parsing it:
Then you have the arguments for parsing adjustments or adjusting/manipulating the HTML content while the library is parsing it:
- **encoding**: This is the encoding that will be used while parsing the HTML. The default is `UTF-8`.
- **keep_comments**: This tells the library whether to keep HTML comments while parsing the page. It's disabled by default, as it can mess up your scraping in many ways.
- **keep_cdata**: Same logic as the HTML comments. [cdata](https://stackoverflow.com/questions/7092236/what-is-cdata-in-html) is removed by default for cleaner HTML. This also means when you check for the raw html content, you will find it doesn't have the cdata.
- **keep_comments**: This tells the library whether to keep HTML comments while parsing the page. It's disabled by default because it can cause issues with your scraping in various ways.
- **keep_cdata**: Same logic as the HTML comments. [cdata](https://stackoverflow.com/questions/7092236/what-is-cdata-in-html) is removed by default for cleaner HTML.
I have intended to ignore the arguments `huge_tree` and `root` to avoid making this page more complicated than needed.
You may notice that I'm doing that a lot, and that's because it's something you don't need to know to use the library. The development section will cover these missing parts if you are that interested.
You may notice that I'm doing that a lot because it involves advanced features that you don't need to know to use the library. The development section will cover these missing parts if you are very invested.
After that, for the main page and elements within, most properties don't get initialized until you use it like the text content of a page/element, and this is one of the reasons for Scrapling speed :)
After that, for the main page and elements within, most properties are lazily loaded. This means they don't get initialized until you use them like the text content of a page/element, and this is one of the reasons for Scrapling speed :)
### Properties
You have already seen much of this on the [overview](../overview.md) page, but don't worry if you didn't. We will review it more thoroughly using more advanced methods/usages. For clarity, the properties for traversal are separated below in the [traversal](#traversal) section.
@@ -81,15 +81,15 @@ Let's say we are parsing this HTML page for simplicity:
```
Load the page directly as shown before:
```python
from scrapling import Adaptor
page = Adaptor(html_doc)
from scrapling import Selector
page = Selector(html_doc)
```
Get all text content on the page recursively
```python
>>> page.get_all_text()
'Some page\n\n \n\n \nProduct 1\nThis is product 1\n$10.99\nIn stock: 5\nProduct 2\nThis is product 2\n$20.99\nIn stock: 3\nProduct 3\nThis is product 3\n$15.99\nOut of stock'
```
Get the first article as explained before; we will use it as an example
Get the first article, as explained before; we will use it as an example
```python
article = page.find('article')
```
@@ -98,7 +98,7 @@ With the same logic, get all text content on the element recursively
>>> article.get_all_text()
'Product 1\nThis is product 1\n$10.99\nIn stock: 5'
```
But if you try to get the direct text content, it will be empty; notice the logic difference
But if you try to get the direct text content, it will be empty because it doesn't have direct text in the HTML code above
```python
>>> article.text
''
@@ -107,10 +107,10 @@ The `get_all_text` method has the following optional arguments:
1. **separator**: All strings collected will be concatenated using this separator. The default is '\n'
2. **strip**: If enabled, strings will be stripped before concatenation. Disabled by default.
3. **ignore_tags**: A tuple of all tag names you want to ignore in the final results. The default is `('script', 'style',)`.
3. **ignore_tags**: A tuple of all tag names you want to ignore in the final results and ignore any elements nested within them. The default is `('script', 'style',)`.
4. **valid_values**: If enabled, the method will only collect elements with real values, so all elements with empty text content or only whitespaces will be ignored. It's enabled by default
By the way, the text returned here is not a standard string but a [TextHandler](#texthandler); we will get to this in detail later, so if the text content can be serialized to JSON, then use `.json()` on it
By the way, the text returned here is not a standard string but a [TextHandler](#texthandler); we will get to this in detail later, so if the text content can be serialized to JSON, use `.json()` on it
```python
>>> script = page.find('script')
>>> script.json()
@@ -121,7 +121,7 @@ Let's continue to get the element tag
>>> article.tag
'article'
```
If you used it on the page directly, you will find you are operating on the root `html` element
If you use it on the page directly, you will find that you are operating on the root `html` element
```python
>>> page.tag
'html'
@@ -133,6 +133,17 @@ Getting the attributes of the element
>>> print(article.attrib)
{'class': 'product', 'data-id': '1'}
```
Access a specific attribute with any method of the following
```python
>>> article.attrib['class']
>>> article.attrib.get('class')
>>> article['class'] # new in v0.3
```
Check if the attributes contain a specific attribute with any of the methods below
```python
>>> 'class' in article.attrib
>>> 'class' in article # new in v0.3
```
Get the HTML content of the element
```python
>>> article.html_content
@@ -143,7 +154,7 @@ It's the same if you used the `.body` property
>>> article.body
'<article class="product" data-id="1"><h3>Product 1</h3>\n <p class="description">This is product 1</p>\n <span class="price">$10.99</span>\n <div class="hidden stock">In stock: 5</div>\n </article>'
```
Get the prettified version of the HTML content of the element
Get the prettified version of the element's HTML content
```python
>>> print(article.prettify())
<article class="product" data-id="1"><h3>Product 1</h3>
@@ -175,12 +186,12 @@ Same case with XPath
```
### Traversal
Using the elements we found above, we will go over the properties/methods for moving in the page in detail.
Using the elements we found above, we will go over the properties/methods for moving on the page in detail.
If you are unfamiliar with the DOM tree or the tree data structure in general, the following traversal part can be confusing. I recommend you look up these concepts online for a better understanding.
If you are too lazy to search about it, here's a quick explanation to give you a good idea.<br/>
Simply put, the `html` element is the root of the website's tree, as every page starts with an `html` element.<br/>
In simple words, the `html` element is the root of the website's tree, as every page starts with an `html` element.<br/>
This element will be directly above elements like `head` and `body`. These are considered "children" of the `html` element, and the `html` element is considered their "parent." The element `body` is a "sibling" of the element `head` and vice versa.
Accessing the parent of an element
@@ -238,7 +249,7 @@ Get the siblings of an element
```
Get the next element of the current element
```python
>>> article.next # gets the next element, the same logic applies to `quote.previous`
>>> article.next
<data='<article class="product" data-id="2"><h3...' parent='<div class="product-list"> <article clas...'>
```
The same logic applies to the `previous` property
@@ -258,7 +269,7 @@ If your case needs more than the element's parent, you can iterate over the whol
for ancestor in article.iterancestors():
# do something with it...
```
You can search for a specific ancestor of an element that satisfies a function; all you need to do is to pass a function that takes an [Adaptor](#adaptor) object as an argument and return `True` if the condition satisfies or `False` otherwise like below:
You can search for a specific ancestor of an element that satisfies a search function; all you need to do is to pass a function that takes a [Selector](#selector) object as an argument and return `True` if the condition satisfies or `False` otherwise, like below:
```python
>>> article.find_ancestor(lambda ancestor: ancestor.has_class('product-list'))
<data='<div class="product-list"> <article clas...' parent='<body> <div class="product-list"> <artic...'>
@@ -266,10 +277,10 @@ You can search for a specific ancestor of an element that satisfies a function;
>>> article.find_ancestor(lambda ancestor: ancestor.css('.product-list')) # Same result, different approach
<data='<div class="product-list"> <article clas...' parent='<body> <div class="product-list"> <artic...'>
```
## Adaptors
The class `Adaptors` is the "List" version of the [Adaptor](#adaptor) class. It inherits from the Python standard `List` type, so it shares all `List` properties and methods while adding more methods to make the operations you want to execute on the [Adaptor](#adaptor) instances within more straightforward.
## Selectors
The class `Selectors` is the "List" version of the [Selector](#selector) class. It inherits from the Python standard `List` type, so it shares all `List` properties and methods while adding more methods to make the operations you want to execute on the [Selector](#selector) instances within more straightforward.
In the [Adaptor](#adaptor) class, all methods/properties that should return a group of elements return them as an [Adaptors](#adaptors) class instance. The only exceptions are when you use the CSS/XPath methods as follows:
In the [Selector](#selector) class, all methods/properties that should return a group of elements return them as a [Selectors](#selectors) class instance. The only exceptions are when you use the CSS/XPath methods as follows:
- If you selected a text node with the selector, then the return type will be [TextHandler](#texthandler)/[TextHandlers](#texthandlers). <br/>Examples:
```python
@@ -284,18 +295,18 @@ In the [Adaptor](#adaptor) class, all methods/properties that should return a gr
```
- If you used a combined selector that returns mixed types, the result will be a Python standard `List`. <br/>Examples:
```python
>>> page.css('.price_color') # -> Adaptors
>>> page.css('.price_color') # -> Selectors
>>> page.css('.product_pod a::attr(href)') # -> TextHandlers
>>> page.css('.price_color, .product_pod a::attr(href)') # -> List
```
Let's see what [Adaptors](#adaptors) class adds to the table with that out of the way.
Let's see what [Selectors](#selectors) class adds to the table with that out of the way.
### Properties
Apart from the normal operations on Python lists like iteration, slicing, etc...
You can do the following:
Execute CSS and XPath selectors directly on the [Adaptor](#adaptor) instances it has while the arguments and the return types are the same as [Adaptor](#adaptor)'s `css` and `xpath` methods. This, of course, makes chaining methods very straightforward.
Execute CSS and XPath selectors directly on the [Selector](#selector) instances it has, while the arguments and the return types are the same as [Selector](#selector)'s `css` and `xpath` methods. This, of course, makes chaining methods very straightforward.
```python
>>> page.css('.product_pod a')
[<data='<a href="catalogue/a-light-in-the-attic_...' parent='<div class="image_container"> <a href="c...'>,
@@ -315,9 +326,9 @@ Execute CSS and XPath selectors directly on the [Adaptor](#adaptor) instances it
<data='<a href="catalogue/soumission_998/index....' parent='<h3><a href="catalogue/soumission_998/in...'>,
...]
```
Run the `re` and `re_first` methods directly. They take the same arguments passed as the [Adaptor](#adaptor) class. I'm still leaving these methods to be explained in the [TextHandler](#texthandler) section below.
Run the `re` and `re_first` methods directly. They take the same arguments passed to the [Selector](#selector) class. I'm still leaving these methods to be explained in the [TextHandler](#texthandler) section below.
However, in this class, the `re_first` behaves differently as it runs `re` on each [Adaptor](#adaptor) within and returns the first one with a result. The `re` method will return a [TextHandlers](#texthandlers) object as normal that has all the results combined in one [TextHandlers](#texthandlers) instance.
However, in this class, the `re_first` behaves differently as it runs `re` on each [Selector](#selector) within and returns the first one with a result. The `re` method will return a [TextHandlers](#texthandlers) object as normal, that has all the [TextHandler](#texthandler) instances combined in one [TextHandlers](#texthandlers) instance.
```python
>>> page.css('.price_color').re(r'[\d\.]+')
['51.77',
@@ -334,14 +345,14 @@ However, in this class, the `re_first` behaves differently as it runs `re` on ea
'sharp-objects_997',
...]
```
With the `search` method, you can search quickly in the available [Adaptor](#adaptor) classes. The function you pass must accept an [Adaptor](#adaptor) instance as the first argument and return True/False. The method will return the first [Adaptor](#adaptor) instance that satisfies the function; otherwise, it will return `None`.
With the `search` method, you can search quickly in the available [Selector](#selector) instances. The function you pass must accept a [Selector](#selector) instance as the first argument and return True/False. The method will return the first [Selector](#selector) instance that satisfies the function; otherwise, it will return `None`.
```python
# Find all the products with price '53.23'
>>> search_function = lambda p: float(p.css('.price_color').re_first(r'[\d\.]+')) == 54.23
>>> page.css('.product_pod').search(search_function)
<data='<article class="product_pod"><div class=...' parent='<li class="col-xs-6 col-sm-4 col-md-3 co...'>
```
You can use the `filter` method, too, which takes a function like the `search` method but returns an `Adaptors` instance of all the [Adaptor](#adaptor) classes that satisfy the function
You can use the `filter` method, too, which takes a function like the `search` method but returns an `Selectors` instance of all the [Selector](#selector) instances that satisfy the function
```python
# Find all products with prices over $50
>>> filtering_function = lambda p: float(p.css('.price_color').re_first(r'[\d\.]+')) > 50
@@ -351,26 +362,35 @@ You can use the `filter` method, too, which takes a function like the `search` m
<data='<article class="product_pod"><div class=...' parent='<li class="col-xs-6 col-sm-4 col-md-3 co...'>,
...]
```
If you are too lazy like me and want to know the number of [Selector](#selector) instances in a [Selectors](#selectors) instance. You can do this:
```python
page.css('.product_pod').length
```
instead of this
```python
len(page.css('.product_pod'))
```
Yup, like JavaScript :)
## TextHandler
This class is mandatory to understand, as all methods/properties that should return a string for you will return `TextHandler`, and the ones that should return a list of strings will return [TextHandlers](#texthandlers) instead.
TextHandler is a subclass of the standard Python string, so you can do anything with it. So, what is the difference that requires a different naming?
TextHandler is a subclass of the standard Python string, so you can do anything with it that you can do with a Python string. So, what is the difference that requires a different naming?
Of course, TextHandler provides extra methods and properties that the standard Python strings can't do. We will review them now, but remember that all methods and properties in all classes that return string(s) are returning TextHandler, which opens the door for creativity and makes the code shorter and cleaner, as you will see. Also, you can import it directly and use it on any string, which we will explain later.
Of course, TextHandler provides extra methods and properties that standard Python strings can't do. We will review them now, but remember that all methods and properties in all classes that return string(s) return TextHandler, which opens the door for creativity and makes the code shorter and cleaner, as you will see. Also, you can import it directly and use it on any string, which we will explain [later](../development/scrapling_custom_types.md).
### Usage
First, before discussing the added methods, you need to know that all operations on it, like slicing, accessing by index, etc., and methods like `split`, `replace`, `strip`, etc., all return a TextHandler again, so you can chain them as you want. If you find a method or property that returns a standard string instead of TextHandler, please open an issue, and we will override it as well.
First, before discussing the added methods, you need to know that all operations on it, like slicing, accessing by index, etc., and methods like `split`, `replace`, `strip`, etc., all return a `TextHandler` again, so you can chain them as you want. If you find a method or property that returns a standard string instead of `TextHandler`, please open an issue, and we will override it as well.
First, we start with the `re` and `re_first` methods. These are the same methods that exist in the rest of the classes ([Adaptor](#adaptor), [Adaptors](#adaptors), and [TextHandlers](#texthandlers)), so they will take the same arguments as well.
First, we start with the `re` and `re_first` methods. These are the same methods that exist in the rest of the classes ([Selector](#selector), [Selectors](#selectors), and [TextHandlers](#texthandlers)), so they will take the same arguments as well.
The `re` method takes a string/compiled regex pattern as the first argument. It searches the data for all strings matching the regex and returns them as a [TextHandlers](#texthandlers) instance. The `re_first` method takes the same arguments and behaves similarly, but as you probably figured out from the naming, it returns the first result only as a `TextHandler` instance.
- The `re` method takes a string/compiled regex pattern as the first argument. It searches the data for all strings matching the regex and returns them as a [TextHandlers](#texthandlers) instance. The `re_first` method takes the same arguments and behaves similarly, but as you probably figured out from the naming, it returns the first result only as a `TextHandler` instance.
Also, it takes other helpful arguments, which are:
- **replace_entities**: This is enabled by default. It replaces character entity references with their corresponding characters.
- **clean_match**: It's disabled by default. This makes the method ignore all whitespaces and consecutive spaces while matching.
- **case_sensitive**: It's enabled by default. As the name implies, disabling it will make the regex ignore letters case while compiling it.
- **clean_match**: It's disabled by default. This makes the method ignore all whitespaces and consecutive spaces while matching.
- **case_sensitive**: It's enabled by default. As the name implies, disabling it will make the regex ignore the case of letters while compiling it.
You have seen these examples before; the return result is [TextHandlers](#texthandlers) because we used the `re` method.
```python
>>> page.css('.price_color').re(r'[\d\.]+')
@@ -405,25 +425,25 @@ First, we start with the `re` and `re_first` methods. These are the same methods
>>> test_string.re('hi there', clean_match=True, case_sensitive=False)
['hi There']
```
Another use of the idea of replacing strings with `TextHandler` everywhere is a property like `html_content` returns `TextHandler` so you can do regex on the HTML content if you want:
Another use of the idea of replacing strings with `TextHandler` everywhere is that a property like `html_content` returns `TextHandler`, so you can do regex on the HTML content if you want:
```python
>>> page.html_content.re('div class=".*">(.*)</div')
['In stock: 5', 'In stock: 3', 'Out of stock']
```
- You also have the `.json()` method, which tries to convert the content to a json object quickly if possible; otherwise, it throws an error
- You also have the `.json()` method, which tries to convert the content to a JSON object quickly if possible; otherwise, it throws an error
```python
>>> page.css_first('#page-data::text')
'\n {\n "lastUpdated": "2024-09-22T10:30:00Z",\n "totalProducts": 3\n }\n '
>>> page.css_first('#page-data::text').json()
{'lastUpdated': '2024-09-22T10:30:00Z', 'totalProducts': 3}
```
Hence, if you didn't specify a text node while selecting an element (like the text content or an attribute text content), the text content will be selected automatically like this
Hence, if you didn't specify a text node while selecting an element (like the text content or an attribute text content), the text content will be selected automatically, like this
```python
>>> page.css_first('#page-data').json()
{'lastUpdated': '2024-09-22T10:30:00Z', 'totalProducts': 3}
```
The [Adaptor](#adaptor) class adds one thing here, too; let's say this is the page we are working with:
The [Selector](#selector) class adds one thing here, too; let's say this is the page we are working with:
```html
<html>
<body>
@@ -438,42 +458,42 @@ First, we start with the `re` and `re_first` methods. These are the same methods
</body>
</html>
```
The [Adaptor](#adaptor) class has the `get_all_text` method, which you should be aware of by now. This method returns a `TextHandler`, of course.<br/><br/>
The [Selector](#selector) class has the `get_all_text` method, which you should be aware of by now. This method returns a `TextHandler`, of course.<br/><br/>
So, as you know here, if you did something like this
```python
>>> page.css_first('div::text').json()
```
You will get an error because the `div` tag doesn't have direct text content that can be serialized to JSON; it actually doesn't have text content at all.<br/><br/>
You will get an error because the `div` tag doesn't have direct text content that can be serialized to JSON; it actually doesn't have direct text content at all.<br/><br/>
In this case, the `get_all_text` method comes to the rescue, so you can do something like that
```python
>>> page.css_first('div').get_all_text(ignore_tags=[]).json()
{'lastUpdated': '2024-09-22T10:30:00Z', 'totalProducts': 3}
```
I used the `ignore_tags` argument here because the default value of it is `('script', 'style',)`, as you are aware.<br/><br/>
Another related behavior you should be aware of is the case while using any of the fetchers, which we will explain later. If you have a JSON response like this example:
Another related behavior to be aware of occurs when using any of the fetchers, which we will explain later. If you have a JSON response like this example:
```python
>>> page = Adaptor("""{"some_key": "some_value"}""")
>>> page = Selector("""{"some_key": "some_value"}""")
```
Because the [Adaptor](#adaptor) class is optimized to deal with HTML pages, it will deal with it as a broken HTML response and fix it, so if you used the `html_content` property, you get this
Because the [Selector](#selector) class is optimized to deal with HTML pages, it will deal with it as a broken HTML response and fix it, so if you used the `html_content` property, you get this
```python
>>> page.html_content
'<html><body><p>{"some_key": "some_value"}</p></body></html>'
```
Here, you can use `json` method directly, and it will work
Here, you can use the `json` method directly, and it will work
```python
>>> page.json()
{'some_key': 'some_value'}
```
You might wonder how this happened while the `html` tag lacks direct text?<br/>
Well, for these cases like JSON responses, I made the `.json()` method inside the [Adaptor](#adaptor) class to check if the current element doesn't have text content; it will use the `get_all_text` method directly.<br/><br/>It might sound hacky a bit but remember, Scrapling is currently optimized to work with HTML pages only so that's the best way till now to handle JSON responses currently without sacrificing speed. This will be changed in the upcoming versions.
You might wonder how this happened while the `html` tag doesn't have direct text?<br/>
Well, for cases like JSON responses, I made the [Selector](#selector) class maintain a raw copy of the content passed to it. This way, when you use the `.json()` method, it checks for that raw copy and then converts it to JSON. If the raw copy is not available like the case with the elements, it checks for the current element text content, or otherwise it used the `get_all_text` method directly.<br/><br/>This might sound hacky a bit but remember, Scrapling is currently optimized to work with HTML pages only so that's the best way till now to handle JSON responses currently without sacrificing speed. This will be changed in the upcoming versions.
- Another handy method is `.clean()`, this will remove all white spaces and consecutive spaces for you and return a new `TextHandler`, wonderful
- Another handy method is `.clean()`, which will remove all white spaces and consecutive spaces for you and return a new `TextHandler` instance
```python
>>> TextHandler('\n wonderful idea, \reh?').clean()
'wonderful idea, eh?'
```
- Another method that might be helpful in some cases is the `.sort()` method to sort the string for you as you do with lists
- Another method that might be helpful in some cases is the `.sort()` method to sort the string for you, as you do with lists
```python
>>> TextHandler('acb').sort()
'abc'
@@ -487,19 +507,19 @@ Or do it in reverse:
Other methods and properties will be added over time, but remember that this class is returned in place of strings nearly everywhere in the library.
## TextHandlers
You probably guessed it: This class is similar to [Adaptors](#adaptors) and [Adaptor](#adaptor), but here it inherits the same logic and method as standard lists, with only `re` and `re_first` as new methods.
You probably guessed it: This class is similar to [Selectors](#selectors) and [Selector](#selector), but here it inherits the same logic and method as standard lists, with only `re` and `re_first` as new methods.
The only difference is that the `re_first` method logic here does `re` on each [TextHandler](#texthandler) within and returns the first result it has or `None`. Nothing is new to explain here, but new methods will be added here with time.
The only difference is that the `re_first` method logic here does `re` on each [TextHandler](#texthandler) within and returns the first result it has or `None`. Nothing is new to explain here, but new methods will be added over time.
## AttributesHandler
This is a read-only version of Python's standard dictionary or `dict` that's only used to store the attributes of each element or each [Adaptor](#adaptor) instance, in other words.
This is a read-only version of Python's standard dictionary or `dict` that's only used to store the attributes of each element or each [Selector](#selector) instance, in other words.
```python
>>> print(page.find('script').attrib)
{'id': 'page-data', 'type': 'application/json'}
>>> type(page.find('script').attrib).__name__
'AttributesHandler'
```
Because it's read-only, it will use fewer resources than the standard dictionary. Still, it has the same dictionary method/properties other than those allowing you to modify/override the data.
Because it's read-only, it will use fewer resources than the standard dictionary. Still, it has the same dictionary method and properties, except those that allow you to modify/override the data.
It currently adds two extra simple methods:
@@ -530,10 +550,10 @@ It currently adds two extra simple methods:
Hence, I used the `list` function here because `search_values` returns a generator, so it would be `True` for all elements.
- The `json_string` property
- The `json_string` property
This property converts current attributes to JSON string if the attributes are JSON serializable; otherwise, it throws an error
```python
>>> page.find('script').attrib.json_string
b'{"id":"page-data","type":"application/json"}'
```
This property converts current attributes to a JSON string if the attributes are JSON serializable; otherwise, it throws an error
```python
>>>page.find('script').attrib.json_string
b'{"id":"page-data","type":"application/json"}'
```
docs/parsing/selection.md
+40 -40
View File
@@ -1,13 +1,13 @@
## Introduction
Scrapling currently supports parsing HTML pages exclusively, so it doesn't support XML feeds. This decision was made because the automatch feature won't work with XML, but that might change soon, so stay tuned :)
Scrapling currently supports parsing HTML pages exclusively, so it doesn't support XML feeds. This decision was made because the adaptive feature won't work with XML, but that might change soon, so stay tuned :)
In Scrapling, there are 5 main ways to find elements:
In Scrapling, there are five main ways to find elements:
1. CSS3 Selectors
2. XPath Selectors
3. Finding elements based on filters/conditions.
4. Finding elements whose content contains specific text
5. Finding elements whose content matches specific regex
4. Finding elements whose content contains a specific text
5. Finding elements whose content matches a specific regex
Of course, there are other indirect ways to find elements with Scrapling, but here we will discuss the main ways in detail. We will also bring up one of the most remarkable features of Scrapling: the ability to find elements that are similar to the element you have; you can jump to that section directly from [here](#finding-similar-elements).
@@ -18,7 +18,7 @@ If you are new to Web Scraping, have little to no experience writing selectors,
### What are CSS selectors?
[CSS](https://en.wikipedia.org/wiki/CSS) is a language for applying styles to HTML documents. It defines selectors to associate those styles with specific HTML elements.
Scrapling implements CSS3 selectors as described in the [W3C specification](http://www.w3.org/TR/2011/REC-css3-selectors-20110929/). CSS selectors support comes from cssselect, so it's better to read about which [selectors are supported from cssselect](https://cssselect.readthedocs.io/en/latest/#supported-selectors) and pseudo-functions/elements.
Scrapling implements CSS3 selectors as described in the [W3C specification](http://www.w3.org/TR/2011/REC-css3-selectors-20110929/). CSS selectors support comes from `cssselect`, so it's better to read about which [selectors are supported from cssselect](https://cssselect.readthedocs.io/en/latest/#supported-selectors) and pseudo-functions/elements.
Also, Scrapling implements some non-standard pseudo-elements like:
@@ -27,16 +27,16 @@ Also, Scrapling implements some non-standard pseudo-elements like:
In short, if you come from Scrapy/Parsel, you will find the same logic for selectors here to make it easier. No need to implement a stranger logic to the one that most of us are used to :)
To select elements with CSS selectors, you have the `css` and `css_first` methods. The latter is useful when you are interested in the first element it finds only, or if it's one element, etc., and the first when it's more than one, as it returns `Adaptors`.
To select elements with CSS selectors, you have the `css` and `css_first` methods. The latter is ~10% faster and more valuable when you are interested in the first element it finds, or if it's just one element, etc. It's beneficial when there's more than one, as it returns `Selectors`.
### What are XPath selectors?
[XPath](https://en.wikipedia.org/wiki/XPath) is a language for selecting nodes in XML documents, which can also be used with HTML. This [cheatsheet] (https://devhints.io/xpath) is a good resource for learning about [XPath](https://en.wikipedia.org/wiki/XPath). Scrapling adds XPath selectors directly through LXML.
[XPath](https://en.wikipedia.org/wiki/XPath) is a language for selecting nodes in XML documents, which can also be used with HTML. This [cheatsheet](https://devhints.io/xpath) is a good resource for learning about [XPath](https://en.wikipedia.org/wiki/XPath). Scrapling adds XPath selectors directly through [lxml](https://lxml.de/).
In short, it is the same situation as CSS Selectors; if you come from Scrapy/Parsel, you will find the same logic for selectors here. BUT Scrapling doesn't implement the XPath extension function `has-class` as Scrapy/Parsel—instead, there's the `has_class` method that you can use on elements returned for the same purpose.
In short, it is the same situation as CSS Selectors; if you come from Scrapy/Parsel, you will find the same logic for selectors here. However, Scrapling doesn't implement the XPath extension function `has-class` as Scrapy/Parsel does. Instead, it provides the `has_class` method, which can be used on elements returned for the same purpose.
To select elements with XPath selectors, you have the `xpath` and `xpath_first` methods. Again, these methods follow the same logic as the CSS selectors methods above.
To select elements with XPath selectors, you have the `xpath` and `xpath_first` methods. Again, these methods follow the same logic as the CSS selectors methods above, and `xpath_first` is faster.
> Note that each method of `css`, `css_first`, `xpath`, and `xpath_first` have additional arguments, but we didn't explain them here as they are all about the automatch feature. The automatch feature will have its page later to be described in detail.
> Note that each method of `css`, `css_first`, `xpath`, and `xpath_first` has additional arguments, but we didn't explain them here as they are all about the adaptive feature. The adaptive feature will have its own page later to be described in detail.
### Selectors examples
Let's see some shared examples of using CSS and XPath Selectors.
@@ -46,14 +46,14 @@ Select all elements with the class `product`
products = page.css('.product')
products = page.xpath('//*[@class="product"]')
```
Note: The XPath one won't be accurate if there's another class; better rely on CSS for selecting by class
Note: The XPath one won't be accurate if there's another class; **it's always better to rely on CSS for selecting by class**
Select the first element with the class `product`
```python
product = page.css_first('.product')
product = page.xpath_first('//*[@class="product"]')
```
Which would be the same as doing
Which would be the same as doing (but a bit slower)
```python
product = page.css('.product')[0]
product = page.xpath('//*[@class="product"]')[0]
@@ -68,12 +68,12 @@ Which is again the same as doing
title = page.css_first('h1').text
title = page.xpath_first('//h1').text
```
Get the `href` attribute of the first element with `a` tag name
Get the `href` attribute of the first element with the `a` tag name
```python
link = page.css_first('a::attr(href)')
link = page.xpath_first('//a/@href')
```
Select the text of the first element with the `h1` tag name, which contains 'Phone' and under an element with class 'product'
Select the text of the first element with the `h1` tag name, which contains 'Phone', and under an element with class 'product'
```python
title = page.css_first('.product h1:contains("Phone")::text')
title = page.page.xpath_first('//*[@class="product"]//h1[contains(text(),"Phone")]/text()')
@@ -99,46 +99,46 @@ for index, link in enumerate(links):
## Text-content selection
Scrapling provides the ability to select elements based on their direct text content, and you have two ways to do this:
1. Elements whose direct text content contains given text with many options through the `find_by_text` method.
1. Elements whose direct text content contains the given text with many options through the `find_by_text` method.
2. Elements whose direct text content matches the given regex pattern with many options through the `find_by_regex` method.
What you can do with `find_by_text` can be done with `find_by_regex` if you are good enough with regular expressions (regex), but we are providing more options to make them easier for all users to access.
With `find_by_text`, you will pass the text as the first argument; with the `find_by_regex` method, the regex pattern is the first. Both methods share the following arguments:
With `find_by_text`, you will pass the text as the first argument; with the `find_by_regex` method, the regex pattern is the first argument. Both methods share the following arguments:
* **first_match**: If `True` (the default), the method used will return the first result it finds.
* **case_sensitive**: If `True`, the case of the letters will be considered.
* **clean_match**: If `True`, all whitespaces and consecutive spaces will be ignored while matching.
* **clean_match**: If `True`, all whitespaces and consecutive spaces will be replaced with a single space before matching.
By default, Scrapling search for exact matching for the text you pass to `find_by_text`, so the text content of the wanted element have to be ONLY the text you inputted, but that's why it also has one extra argument, which is:
By default, Scrapling searches for the exact matching of the text/pattern you pass to `find_by_text`, so the text content of the wanted element has to be ONLY the text you input, but that's why it also has one extra argument, which is:
* **partial**: If enabled, `find_by_text` will return elements that contain the input text. So it's not an exact match anymore
Note: The method `find_by_regex` can accept both regular strings and a compiled regex pattern as its first argument, as you will see in the upcoming examples.
### Finding Similar Elements
One of the most remarkable new features that Scrapling puts on the table is the feature that allows the user to tell Scrapling to find elements similar to the element at hand. This feature inspiration came from the AutoScraper library, but here, it can be used on elements found by any method. Most likely, most of its usage would be after finding elements through text content like how AutoScraper works, so it would also be convenient to explain it here.
One of the most remarkable new features that Scrapling puts on the table is the feature that allows the user to tell Scrapling to find elements similar to the element at hand. This feature's inspiration came from the AutoScraper library, but in Scrapling, it can be used on elements found by any method. Most of its usage would likely occur after finding elements through text content, similar to how AutoScraper works, making it convenient to explain here.
So, how does it work?
Imagine a scenario where you found a product by its title, for example, and you want to extract other products listed in the same table/container. With the element you have, you can simply call the method `.find_similar()` on it, and Scrapling will:
Imagine a scenario where you found a product by its title, for example, and you want to extract other products listed in the same table/container. With the element you have, you can call the method `.find_similar()` on it, and Scrapling will:
1. Find all page elements with the same tree depth as this element.
1. Find all page elements with the same DOM tree depth as this element.
2. All found elements will be checked, and those without the same tag name, parent tag name, and grandparent tag name will be dropped.
3. Now we are sure (like 99% sure) that these elements are the ones we want, but as a last check, Scrapling will use fuzzy matching to drop the elements whose attributes don't look like the attributes of our element. There's a percentage to control this step, and I recommend you not play with it unless the default settings don't get the elements you want.
That's a lot of talking, I know, but I had to go deep, I will give examples of using this method in the next section, but first, these are the arguments that can be passed to this method:
That's a lot of talking, I know, but I had to go deep. I will give examples of using this method in the next section, but first, these are the arguments that can be passed to this method:
* **similarity_threshold**: This is the percentage we discussed in step 3 for comparing elements' attributes. The default value is 0.2. In Simpler words, the attributes' values of both elements should be at least 20% similar. If you want to turn off this check (Step 3, basically), you can set this attribute to 0, but I recommend you read what other arguments do first.
* **similarity_threshold**: This is the percentage we discussed in step 3 for comparing elements' attributes. The default value is 0.2. In Simpler words, the values of the attributes of both elements should be at least 20% similar. If you want to turn off this check (Step 3, basically), you can set this attribute to 0, but I recommend you read what the other arguments do first.
* **ignore_attributes**: The attribute names passed will be ignored while matching the attributes in the last step. The default value is `('href', 'src',)` because URLs can change a lot between elements, making them unreliable.
* **match_text**: If `True`, the element's text content will be considered when matching. Using this in normal cases is not recommended, but it depends.
* **match_text**: If `True`, the element's text content will be considered when matching (Step 3). Using this argument in typical cases is not recommended, but it depends.
Now, let's check out the examples below.
### Examples
Let's see some shared examples of finding elements with raw text and regex.
I will use the `Fetcher` to clarify these examples, but it will be explained in detail later.
I will use the `Fetcher` class with these examples, but it will be explained in detail later.
```python
from scrapling.fetchers import Fetcher
page = Fetcher.get('https://books.toscrape.com/index.html')
@@ -155,7 +155,7 @@ Combining it with `page.urljoin` to return the full URL from the relative `href`
>>> page.urljoin(page.find_by_text('Tipping the Velvet').attrib['href'])
'https://books.toscrape.com/catalogue/tipping-the-velvet_999/index.html'
```
Get all matches if there are more (hence, it returned a list)
Get all matches if there are more (notice it returns a list)
```python
>>> page.find_by_text('Tipping the Velvet', first_match=False)
[<data='<a href="catalogue/tipping-the-velvet_99...' parent='<h3><a href="catalogue/tipping-the-velve...'>]
@@ -174,7 +174,7 @@ Get all elements that contain the word `the` (Partial matching)
'Mesaerion: The Best Science ...',
"It's Only the Himalayas"]
```
The search is case insensitive, so those results have `The`, not only the lowercase one `the`; let's limit the search to the elements with `the` only.
The search is case-insensitive, so those results have `The`, not only the lowercase one `the`; let's limit the search to the elements with `the` only.
```python
>>> results = page.find_by_text('the', partial=True, first_match=False, case_sensitive=True)
>>> [i.text for i in results]
@@ -183,7 +183,7 @@ The search is case insensitive, so those results have `The`, not only the lowerc
'The Boys in the ...',
"It's Only the Himalayas"]
```
Get the first element that its text content matches my price regex
Get the first element whose text content matches my price regex
```python
>>> page.find_by_regex(r'£[\d\.]+')
<data='<p class="price_color">£51.77</p>' parent='<div class="product_price"> <p class="pr...'>
@@ -235,7 +235,7 @@ Get the `href` attribute from all similar elements
'catalogue/sharp-objects_997/index.html',
...]
```
To increase the complexity a little bit, let's say we want to get all books' data using that element as a starting point for some reason
To increase the complexity a little bit, let's say we want to get all the books' data using that element as a starting point for some reason
```python
>>> for product in element.parent.parent.find_similar():
print({
@@ -332,16 +332,16 @@ def extract_reviews(page):
]
```
## Filters-based searching
This search method might be arguably the best way to find elements in Scrapling because it is powerful and easier to learn for newcomers to Web Scraping than learning to write selectors.
This search method is arguably the best way to find elements in Scrapling, as it is powerful and easier to learn for newcomers to Web Scraping than writing selectors.
Inspired by BeautifulSoup's `find_all` function, you can find elements using the `find_all` and `find` methods. Both methods can take multiple types of filters and return all elements in the pages that all these filters apply to.
To be more specific:
* Any string passed is considered a tag name.
* Any iterable passed like List/Tuple/Set is considered an iterable of tag names.
* Any dictionary is considered a mapping of HTML element(s) attribute names and attribute values.
* Any regex patterns passed are used to filter elements by content like the `find_by_regex` method
* Any iterable passed, like List/Tuple/Set, is considered an iterable of tag names.
* Any dictionary is considered a mapping of HTML element(s), attribute names, and attribute values.
* Any regex patterns passed are used to filter elements by content, like the `find_by_regex` method
* Any functions passed are used to filter elements
* Any keyword argument passed is considered as an HTML element attribute with its value.
@@ -356,8 +356,8 @@ It filters all elements in the current page/element in the following order:
Notes:
1. As you probably understood, the filtering process always starts from the first filter it finds in the filtering order above. So, if no tag name(s) are passed but attributes are passed, the process starts from that layer, and so on.
2. The order in which you pass the arguments doesn't matter. The only order that's taken into consideration is the order explained above.
1. As you probably understood, the filtering process always starts from the first filter it finds in the filtering order above. So, if no tag name(s) are passed but attributes are passed, the process starts from that step (number 2), and so on.
2. The order in which you pass the arguments doesn't matter. The only order taken into consideration is the order explained above.
Check examples to clear any confusion :)
@@ -407,7 +407,7 @@ Find all elements that don't have children.
<data='<body> <div class="container"> <div clas...' parent='<html lang="en"><head><meta charset="UTF...'>,
...]
```
Find all elements that contain the word 'world' in its content.
Find all elements that contain the word 'world' in their content.
```python
>>> page.find_all(lambda element: "world" in element.text)
[<data='<span class="text" itemprop="text">“The...' parent='<div class="quote" itemscope itemtype="h...'>,
@@ -439,7 +439,7 @@ A bonus pro tip: Find all elements whose `href` attribute's value ends with the
<data='<a href="/author/Albert-Einstein">(about...' parent='<span>by <small class="author" itemprop=...'>,
<data='<a href="/author/Albert-Einstein">(about...' parent='<span>by <small class="author" itemprop=...'>]
```
Another pro tip: Find all elements that its `href` attribute's value has '/author/' in it
Another pro tip: Find all elements whose `href` attribute's value has '/author/' in it
```python
>>> page.find_all({'href*': '/author/'})
[<data='<a href="/author/Albert-Einstein">(about...' parent='<span>by <small class="author" itemprop=...'>,
@@ -474,12 +474,12 @@ Generate a full XPath selector for the `url_element` element from the start of t
'//body/div/div[2]/div/div/span[2]/a'
```
> Note: <br>
> When you tell Scrapling to create a short selector, it tries to find a unique element to use in generation as a stop point, like an element with an `id` attribute, but in our case, there wasn't any so that's why the short and the full selector will be the same.
> When you tell Scrapling to create a short selector, it tries to find a unique element to use in generation as a stop point, like an element with an `id` attribute, but in our case, there wasn't any, so that's why the short and the full selector will be the same.
## Using selectors with regular expressions
Like in `parsel`/`scrapy`, you have the methods `re` and `re_first` for extracting data using regular expressions. However, unlike the former, these methods are in nearly all classes like `Adaptor`/`Adaptors`/`TextHandler` and `TextHandlers`, which means you can use them directly on the element even if you didn't select a text node.
Similar to `parsel`/`scrapy`, `re` and `re_first` methods are available for extracting data using regular expressions. However, unlike the former libraries, these methods are in nearly all classes like `Selector`/`Selectors`/`TextHandler` and `TextHandlers`, which means you can use them directly on the element even if you didn't select a text node.
We will have a deep look at it while explaining the [TextHandler](main_classes.md#texthandler) class, but in general, it works like the below examples:
We will have a deep look at it while explaining the [TextHandler](main_classes.md#texthandler) class, but in general, it works like the examples below:
```python
>>> page.css_first('.price_color').re_first(r'[\d\.]+')
'51.77'
docs/stylesheets/extra.css
+17
View File
@@ -1,3 +1,20 @@
.md-grid {
max-width: 90%;
}
@font-face {
font-family: 'Maple Mono';
font-style: normal;
font-display: swap;
font-weight: 400;
src: url(https://cdn.jsdelivr.net/fontsource/fonts/maple-mono@latest/latin-400-normal.woff2) format('woff2'), url(https://cdn.jsdelivr.net/fontsource/fonts/maple-mono@latest/latin-400-normal.woff) format('woff');
}
:root {
--md-code-font: 'Maple Mono';
}
[align="center"] code {
font-family: 'Maple Mono';
font-style: italic;
font-weight: 800;
}
docs/tutorials/migrating_from_beautifulsoup.md
+12 -12
View File
@@ -1,16 +1,16 @@
# Migrating from BeautifulSoup to Scrapling
If you're already familiar with BeautifulSoup, you're in for a treat. Scrapling is faster, provides similar parsing capabilities, and adds powerful new features for fetching and handling modern web pages. This guide will help you quickly adapt your existing BeautifulSoup code to take advantage of Scrapling's capabilities.
If you're already familiar with BeautifulSoup, you're in for a treat. Scrapling is incredibly faster, provides the same parsing capabilities, adds more parsing capabilities not found in BS, and introduces powerful new features for fetching and handling modern web pages. This guide will help you quickly adapt your existing BeautifulSoup code to leverage Scrapling's capabilities.
Below is a table that covers the most common operations you'll perform when scraping web pages. Each row shows how to accomplish a specific task in BeautifulSoup and the corresponding way to do it in Scrapling.
Below is a table that covers the most common operations you'll perform when scraping web pages. Each row illustrates how to accomplish a specific task using BeautifulSoup and the corresponding method in Scrapling.
You will notice some shortcuts in BeautifulSoup are missing in Scrapling, but that's one of the reasons that makes BeautifulSoup slower than Scrapling. The point is: If the same feature can be used in a short oneliner, there is no need to sacrifice performance to make that short line shorter :)
You will notice that some shortcuts in BeautifulSoup are missing in Scrapling, but that's one of the reasons why BeautifulSoup is slower than Scrapling. The point is: If the same feature can be used in a short oneliner, there is no need to sacrifice performance to shorten that short line :)
| Task | BeautifulSoup Code | Scrapling Code |
|-----------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------|
| Parser import | `from bs4 import BeautifulSoup` | `from scrapling.parser import Adaptor` |
| Parsing HTML from string | `soup = BeautifulSoup(html, 'html.parser')` | `page = Adaptor(html)` |
| Parser import | `from bs4 import BeautifulSoup` | `from scrapling.parser import Selector` |
| Parsing HTML from string | `soup = BeautifulSoup(html, 'html.parser')` | `page = Selector(html)` |
| Finding a single element | `element = soup.find('div', class_='example')` | `element = page.find('div', class_='example')` |
| Finding multiple elements | `elements = soup.find_all('div', class_='example')` | `elements = page.find_all('div', class_='example')` |
| Finding a single element (Example 2) | `element = soup.find('div', attrs={"class": "example"})` | `element = page.find('div', {"class": "example"})` |
@@ -26,7 +26,7 @@ You will notice some shortcuts in BeautifulSoup are missing in Scrapling, but th
| Extracting text content of an element | `string = element.string` | `string = element.text` |
| Extracting all the text in a document or beneath a tag | `text = soup.get_text(strip=True)` | `text = page.get_all_text(strip=True)` |
| Access the dictionary of attributes | `attrs = element.attrs` | `attrs = element.attrib` |
| Extracting attributes | `attr = element['href']` | `attr = element.attrib['href']` |
| Extracting attributes | `attr = element['href']` | `attr = element['href']` |
| Navigating to parent | `parent = element.parent` | `parent = element.parent` |
| Get all parents of an element | `parents = list(element.parents)` | `parents = list(element.iterancestors())` |
| Searching for an element in the parents of an element | `target_parent = element.find_parent("a")` | `target_parent = element.find_ancestor(lambda p: p.tag == 'a')` |
@@ -44,7 +44,7 @@ You will notice some shortcuts in BeautifulSoup are missing in Scrapling, but th
| Filtering a group of elements that satisfies a condition | `group = soup.find('p', 'story').css.filter('a')` | `group = page.find_all('p', 'story').filter(lambda p: p.tag == 'a')` |
One point to remember: BeautifulSoup provides features for modifying and manipulating the page after parsing it. Scrapling focuses more on Scraping the page faster for you, and then you can do what you want with the extracted information. So, two different tools can be used in Web SScraping, but one of them specializes in Web Scraping :)
**One key point to remember**: BeautifulSoup offers features for modifying and manipulating the page after it has been parsed. Scrapling focuses more on scraping the page faster for you, and then you can do what you want with the extracted information. So, two different tools can be used in Web Scraping, but one of them specializes in Web Scraping :)
### Putting It All Together
@@ -71,7 +71,7 @@ for link in links:
from scrapling import Fetcher
url = 'http://example.com'
page = Fetcher.get(url=url)
page = Fetcher.get(url)
links = page.css('a::attr(href)')
for link in links:
@@ -83,10 +83,10 @@ As you can see, Scrapling simplifies the process by handling the fetching and pa
**Additional Notes:**
- **Different parsers**: BeautifulSoup allows you to set the parser engine to use, and one of them is `lxml`. Scrapling doesn't do that and uses the `lxml` library by default for performance reasons.
- **Element Types**: In BeautifulSoup, elements are `Tag` objects, while in Scrapling, they are `Adaptor` objects. However, they provide similar methods and properties for navigation and data extraction.
- **Error Handling**: Both libraries return `None` when an element is not found (e.g., `soup.find()` or `page.css_first()`). To avoid errors, Check for `None` before accessing properties.
- **Text Extraction**: Scrapling provides additional methods for handling text through `TextHandler`, such as `clean()`, which can be helpful for removing extra whitespace or unwanted characters. Please check out the documentation for the complete list.
- **Element Types**: In BeautifulSoup, elements are `Tag` objects, while in Scrapling, they are `Selector` objects. However, they provide similar methods and properties for navigation and data extraction.
- **Error Handling**: Both libraries return `None` when an element is not found (e.g., `soup.find()` or `page.css_first()`). To avoid errors, check for `None` before accessing properties.
- **Text Extraction**: Scrapling provides additional methods for handling text through `TextHandler`, such as `clean()`, which can help remove extra whitespace, consecutive spaces, or unwanted characters. Please check out the documentation for the complete list.
The documentation provides more details on Scrapling's features and the full list of arguments that can be passed to all methods.
The documentation provides more details on Scrapling's features and the complete list of arguments that can be passed to all methods.
This guide should make your transition from BeautifulSoup to Scrapling smooth and straightforward. Happy scraping!
docs/tutorials/replacing_ai.md
+38 -38
View File
@@ -1,12 +1,12 @@
# Scrapling: A Free Alternative to AI for Robust Web Scraping
Web scraping has long been a vital tool for data extraction, but experienced users often encounter persistent issues that can hinder effectiveness. Recently, there's been a noticeable shift toward AI-based web scraping, driven by its potential to address these challenges.
Web scraping has long been a vital tool for data extraction, indexing, and preparing datasets, among other purposes. But experienced users often encounter persistent issues that can hinder effectiveness. Recently, there's been a noticeable shift toward AI-based web scraping, driven by its potential to address these challenges.
In this article, we will discuss these common issues, why companies are shifting toward that approach, the problems with that approach, and how scrapling solves them for you without the cost of using AI.
## Common issues and challenging goals
If you have been doing Web Scraping for a long time, you probably noticed that there are repeating problems with Web Scraping like:
If you have been doing Web Scraping for a long time, you probably noticed that there are repeating problems with Web Scraping, like:
1. **Rapidly changing website structures** — Sites frequently update their DOM structures, breaking static XPath/CSS selectors.
2. **Unstable selectors** — Class names and IDs often change or use randomly generated values that break scrapers or make scraping these websites difficult.
@@ -15,55 +15,54 @@ and others
But that's only if you are doing targeted Web Scraping for known websites, in which case you can write specific code for every website.
If you start thinking about bigger goals like Broad Scraping or Generic Web Scraping or what you like to call it, then the above issues intensify, and you will face new issues like:
If you start thinking about bigger goals like Broad Scraping or Generic Web Scraping, or what you like to call it, then the above issues intensify, and you will face new issues like:
1. **Extreme Website Diversity** — Generic scraping must handle countless variations in HTML structures, CSS usage, JavaScript frameworks, and backend technologies.
2. **Identifying Relevant Data** — How does the scraper know what data is important on a page it has never seen before?
3. **Pagination variations** — Infinite scroll, traditional pagination, "load more" buttons all requiring different approaches
3. **Pagination variations** — Infinite scroll, traditional pagination, "load more" buttons, all requiring different approaches
and more
How are you going to solve that manually? I'm talking about generic web scraping of different websites that don't share any technologies.
How will you solve that manually? I'm referring to generic web scraping of various websites that don't share any common technologies.
## AI to the rescue but at a high cost
## AI to the rescue, but at a high cost
Of course, the AI can solve most of these issues easily because it will understand the page source and tell you where are the fields you want or create selectors for them for you.<br/>
That's, of course, if you already solved the anti-bot measures through other tools :)
Of course, the AI can easily solve most of these issues because it can understand the page source and identify the fields you want or create selectors for them. That's, of course, if you already solved the anti-bot measures through other tools :)
This approach is beautiful, of course. I love AI and find it very interesting to keep learning about it, especially GenAI. You will probably spend a lot of time on prompt engineering and tweaking the prompts, but if that's cool with you, you will soon hit the real issue with using AI here.
This approach is, of course, beautiful. I love AI and find it very fascinating, especially Generative AI. You will probably spend a lot of time on prompt engineering and tweaking the prompts, but if that's cool with you, you will soon hit the real issue with using AI here.
Most websites have huge content per page, which you will need to pass to the AI somehow so it can do its magic. This will burn through tokens like fire in a haystack, quickly building up high costs!
Most websites have vast amounts of content per page, which you will need to pass to the AI somehow so it can do its magic. This will burn through tokens like fire in a haystack, quickly accumulating high costs.
Unless money is irrelevant to you, you will try to find cheaper approaches, and that's why I made Scrapling :smile:
Unless money is irrelevant to you, you will try to find less expensive approaches, and that's why I made Scrapling :smile:
## Scrapling got you covered
Scrapling can deal with almost all issues you will face during Web Scraping, and the following updates will cover the rest carefully.
Scrapling can handle almost all issues you will face during Web Scraping, and the following updates will cover the rest carefully.
### Solving issue T1: Rapidly changing website structures
That's why the [automatch](https://scrapling.readthedocs.io/en/latest/parsing/automatch/) feature was made. You knew I would talk about it, and here we are :)
That's why the [adaptive](https://scrapling.readthedocs.io/en/latest/parsing/adaptive/) feature was made. You knew I would talk about it, and here we are :)
While Web Scraping, if you have automatch enabled, you can save any element's unique properties for it to find it again later if the website's structure changes. The most frustrating thing about changes is that anything about an element can change, so there's nothing to rely on.
While Web Scraping, if you have the `adaptive` feature enabled, you can save any element's unique properties to find it again later when the website's structure changes. The most frustrating thing about changes is that anything about an element can change, so there's nothing to rely on.
That's how the automatch feature works: it stores everything unique about an element. When the website structure changes, it returns the element with the highest similarity score with the saved properties.
That's how the adaptive feature works: it stores everything unique about an element. When the website structure changes, it returns the element with the highest similarity score of the previous element.
I have already explained that in more detail and with many examples. Read more from [here](https://scrapling.readthedocs.io/en/latest/parsing/automatch/#how-the-automatch-feature-works).
I have already explained that in more detail and with many examples. Read more from [here](https://scrapling.readthedocs.io/en/latest/parsing/adaptive/#how-the-adaptive-feature-works).
### Solving issue T2: Unstable selectors
If you have been doing Web scraping for a long enough time, you have likely experienced this once. I'm talking about a website that uses poor design patterns, is built on pure html without any IDs/classes, uses random class names that change a lot with no identifiers or attributes to rely on, and the list goes on!
If you have been doing Web scraping for a long enough time, you have likely experienced this once. I'm referring to a website that employs poor design patterns, built on raw HTML without any IDs/classes, or uses random class names with nothing else to rely on, etc...
In these cases, standard selection methods with CSS/XPath selectors won't be optimal, and that's why Scrapling provides 3 more methods for Selection:
In these cases, standard selection methods with CSS/XPath selectors won't be optimal, and that's why Scrapling provides three more methods for Selection:
1. [Selection by element content](https://scrapling.readthedocs.io/en/latest/parsing/selection/#text-content-selection) - Through text content (`find_by_text`) or regex that match a text content (`find_by_regex`)
2. [Selecting elements similar to another element](https://scrapling.readthedocs.io/en/latest/parsing/selection/#finding-similar-elements) - You find an element, and we will do the rest!
3. [Selecting elements by filters](https://scrapling.readthedocs.io/en/latest/parsing/selection/#filters-based-searching) - You just specify conditions that this element must fulfill
1. [Selection by element content](https://scrapling.readthedocs.io/en/latest/parsing/selection/#text-content-selection): Through text content (`find_by_text`) or regex that matches text content (`find_by_regex`)
2. [Selecting elements similar to another element](https://scrapling.readthedocs.io/en/latest/parsing/selection/#finding-similar-elements): You find an element, and we will do the rest!
3. [Selecting elements by filters](https://scrapling.readthedocs.io/en/latest/parsing/selection/#filters-based-searching): You specify conditions/filters that this element must fulfill, we find it!
There is no need to explain any of these; just click on the links, and it will be clear how Scrapling solves this.
There is no need to explain any of these; click on the links, and it will be clear how Scrapling solves this.
### Solving issue T3: Increasingly complex anti-bot measures
It's known that making an undetectable spider takes more than residential/mobile proxies and human-like behavior. It also needs a hard-to-detect browser, which Scrapling provides two main options to solve:
1. [PlayWrightFetcher](https://scrapling.readthedocs.io/en/latest/fetching/dynamic/) — This fetcher provides not only stealth mode suitable for small-medium protections but also more flexible options, like using your real browser.
2. [StealthyFetcher](https://scrapling.readthedocs.io/en/latest/fetching/stealthy/) — Because we live in a harsh world and you need to take [full measure instead of half measures](https://www.youtube.com/watch?v=7BE4QcwX4dU), `StealthyFetcher` was born. This fetcher uses a modified Firefox browser called [Camoufox](https://camoufox.com/stealth/) that almost passes all known tests and adds more tricks.
1. [DynamicFetcher](https://scrapling.readthedocs.io/en/latest/fetching/dynamic/) — This fetcher provides many flexible options, like stealth mode suitable for small to medium protections and using your real browser.
2. [StealthyFetcher](https://scrapling.readthedocs.io/en/latest/fetching/stealthy/) — Because we live in a harsh world and you need to take [full measure instead of half-measures](https://www.youtube.com/watch?v=7BE4QcwX4dU), `StealthyFetcher` was born. This fetcher utilizes our version of a modified Firefox browser, called [Camoufox](https://camoufox.com/stealth/), which nearly passes all known tests and incorporates additional tricks. **With v0.3, this fetcher can bypass Cloudflare for you automatically as well!**
These two will be improved a lot with the upcoming updates, so stay tuned :)
@@ -80,7 +79,7 @@ price_element = page.find_by_regex(r'£[\d\.,]+', first_match=True) # Get the f
price_element_container = price_element.parent or price_element.find_ancestor(lambda ancestor: ancestor.has_class('product')) # or other methods...
target_element_selector = price_element_container.generate_css_selector or price_element_container.generate_full_css_selector # or xpath
```
Then he said what about cases like this:
Then he said What about cases like this:
```html
<span class='currency'> $ </span> <span class='a-price'> 45,000 </span>
```
@@ -89,29 +88,30 @@ So, I updated the code like this
price_element_container = page.find_by_regex(r'[\d,]+', first_match=True).parent # Adjusted the regex for this example
full_price_data = price_element_container.get_all_text(strip=True) # Returns '$45,000' in this case
```
This was enough for his use case. You can use the first regex, and if it doesn't find anything, use the following regex, and so on. Try to cover the most common patterns first, then the lesser common ones, and so on.
It will be a bit boring, but it's definitely cheaper than AI.
This was enough for his use case. You can use the first regex, and if it doesn't find anything, use the following regex, and so on. Try to cover the most common patterns first, then the less common ones, and so on.
It will be a bit boring, but it's definitely less expensive than AI.
This example demonstrates the idea I want to deliver here. Not every challenge will need AI only to be solved, but sometimes you need to be creative, and that might save you a lot of money :)
This example illustrates the point I aim to convey here. Not every challenge will need AI to be solved, but sometimes you need to be creative, and that might save you a lot of money.
### Solving issue B3: Pagination variations
This issue Scrapling currently doesn't have a direct method to automatically extract pagination's URLs for you, but it will be added with the following updates :)
This issue, Scrapling currently doesn't have a direct method to automatically extract pagination's URLs for you, but it will be added with the following updates :)
But you can handle most websites if you search for the most common patterns with `page.find_by_text('Next').attrib['href']` or `page.find_by_text('load more').attrib['href']` or selectors like `"a[href*="?page="]""` or `"a[href*="/page/"]""`—you get the idea.
But you can handle most websites if you search for the most common patterns with `page.find_by_text('Next')['href']` or `page.find_by_text('load more')['href']` or selectors like `'a[href*="?page="]'` or `'a[href*="/page/"]'`—you get the idea.
## Cost Comparison and Savings
For a quick comparison.
| Aspect | Scrapling | AI-Based Tools (e.g., Browse AI, Oxylabs) |
|----------------|--------------------------------------------|---------------------------------------------------------------------------|
| Cost Structure | Likely free or low-cost, no per-use fees | Starts at $19/month (Browse AI) to $49/month (Oxylabs), scales with usage |
| Setup Effort | Requires technical expertise, manual setup | Often no-code, easier for non-technical users |
| Scalability | Depends on user implementation | Built-in support for large-scale, managed services |
| Adaptability | High with features like automatch | High, automatic with AI, but costly for frequent changes |
| Aspect | Scrapling | AI-Based Tools (e.g., Browse AI, Oxylabs) |
|----------------|----------------------------------------------------------------------------|----------------------------------------------------------------------------|
| Cost Structure | Likely free or low-cost, no per-use fees | Starts at $19/month (Browse AI) to $49/month (Oxylabs), scales with usage |
| Setup Effort | Requires little technical expertise, manual setup | Often no-code, easier for non-technical users |
| Usage options | Through code, terminal, or MCP server. | Often through GUI or API, depending on the option the company is providing |
| Scalability | Depends on user implementation | Built-in support for large-scale, managed services |
| Adaptability | High with features like `adaptive` and the non-selectors selection methods | High, automatic with AI, but costly for frequent changes |
This table is based on pricing from [Browse AI Pricing](https://www.browse.ai/pricing) and [Oxylabs Web Scraper API Pricing](https://oxylabs.io/products/scraper-api/web/pricing)
## Conclusion
While AI offers powerful capabilities, its cost can be prohibitive for many Web scraping tasks. Scrapling provides a robust, flexible, and cost-effective toolkit designed to tackle the real-world challenges of both targeted and broad scraping, often eliminating the need for expensive AI solutions. You can build resilient scrapers more efficiently by leveraging features like automatch, diverse selection methods, and advanced fetchers.
While AI offers powerful capabilities, its cost can be prohibitive for many Web scraping tasks. Scrapling provides a robust, flexible, and cost-effective toolkit designed to tackle the real-world challenges of both targeted and broad scraping, often eliminating the need for expensive AI solutions. You can build resilient scrapers more efficiently by leveraging features like `adaptive`, diverse selection methods, and advanced fetchers.
Explore the documentation further and see how Scrapling can simplify your next scraping project.
Explore the documentation further and see how Scrapling can simplify your future Web Scraping projects!
mkdocs.yml
+15 -7
View File
@@ -1,5 +1,5 @@
site_name: Scrapling
site_description: Scrapling - a Python library to make Web Scraping easy again!
site_description: Scrapling - Easy, effortless Web Scraping as it should be!
site_author: Karim Shoair
repo_url: https://github.com/D4Vinci/Scrapling
site_url: https://scrapling.readthedocs.io/en/latest/
@@ -31,8 +31,8 @@ theme:
icon: material/toggle-switch-off
name: Switch to system preference
font:
text: Roboto
code: Roboto Mono
text: Open Sans
code: JetBrains Mono
icon:
repo: fontawesome/brands/github-alt
features:
@@ -62,27 +62,35 @@ theme:
nav:
- Introduction: index.md
- Overview: overview.md
- Parsing Performance: benchmarks.md
- What's New in v0.3: 'https://github.com/D4Vinci/Scrapling/releases/tag/v0.3'
- Performance Benchmarks: benchmarks.md
- User Guide:
- Parsing:
- Querying elements: parsing/selection.md
- Main classes: parsing/main_classes.md
- Using automatch feature: parsing/automatch.md
- Adaptive scraping: parsing/adaptive.md
- Fetching:
- Choosing a fetcher: fetching/choosing.md
- Static requests: fetching/static.md
- Dynamically loaded websites: fetching/dynamic.md
- Fully bypass protections while fetching: fetching/stealthy.md
- Command Line Interface:
- Overview: cli/overview.md
- Interactive shell: cli/interactive-shell.md
- Extract commands: cli/extract-commands.md
- Integrations:
- AI MCP server: ai/mcp-server.md
- Tutorials:
- A Free Alternative to AI for Robust Web Scraping: tutorials/replacing_ai.md
- Migrating from BeautifulSoup: tutorials/migrating_from_beautifulsoup.md
# - Migrating from AutoScraper: tutorials/migrating_from_autoscraper.md
- Development:
- API Reference:
- Adaptor: api-reference/adaptor.md
- Selector: api-reference/selector.md
- Fetchers: api-reference/fetchers.md
- MCP Server: api-reference/mcp-server.md
- Custom Types: api-reference/custom-types.md
- Writing your retrieval system: development/automatch_storage_system.md
- Writing your retrieval system: development/adaptive_storage_system.md
- Using Scrapling's custom types: development/scrapling_custom_types.md
- Support and Advertisement: donate.md
- Contributing: contributing.md
pyproject.toml
+103
View File
@@ -0,0 +1,103 @@
[build-system]
requires = ["setuptools>=61.0", "wheel"]
build-backend = "setuptools.build_meta"
[project]
name = "scrapling"
dynamic = ["version"]
description = "Scrapling is an undetectable, powerful, flexible, high-performance Python library that makes Web Scraping easy and effortless as it should be!"
readme = {file = "README.md", content-type = "text/markdown"}
license = {file = "LICENSE"}
authors = [
{name = "Karim Shoair", email = "karim.shoair@pm.me"}
]
maintainers = [
{name = "Karim Shoair", email = "karim.shoair@pm.me"}
]
keywords = [
"web-scraping",
"scraping",
"automation",
"browser-automation",
"data-extraction",
"html-parsing",
"undetectable",
"playwright",
"selenium-alternative",
"web-crawler",
"browser",
"crawling",
]
requires-python = ">=3.10"
classifiers = [
"Operating System :: OS Independent",
"Development Status :: 4 - Beta",
# "Development Status :: 5 - Production/Stable",
# "Development Status :: 6 - Mature",
# "Development Status :: 7 - Inactive",
"Intended Audience :: Developers",
"Intended Audience :: Information Technology",
"License :: OSI Approved :: BSD License",
"Natural Language :: English",
"Topic :: Internet :: WWW/HTTP",
"Topic :: Internet :: WWW/HTTP :: Browsers",
"Topic :: Text Processing :: Markup",
"Topic :: Text Processing :: Markup :: HTML",
"Topic :: Scientific/Engineering :: Artificial Intelligence",
"Topic :: Software Development :: Libraries",
"Topic :: Software Development :: Libraries :: Python Modules",
"Programming Language :: Python :: 3",
"Programming Language :: Python :: 3 :: Only",
"Programming Language :: Python :: 3.10",
"Programming Language :: Python :: 3.11",
"Programming Language :: Python :: 3.12",
"Programming Language :: Python :: 3.13",
"Programming Language :: Python :: Implementation :: CPython",
"Typing :: Typed",
]
dependencies = [
"lxml>=6.0.0",
"cssselect>=1.3.0",
"click>=8.2.1",
"orjson>=3.11.2",
"tldextract>=5.3.0",
"curl_cffi>=0.13.0",
"playwright>=1.52.0",
"rebrowser-playwright>=1.52.0",
"camoufox>=0.4.11",
"geoip2>=5.1.0",
"msgspec>=0.19.0",
]
[project.optional-dependencies]
ai = [
"mcp>=1.13.0",
"markdownify>=1.2.0",
]
shell = [
"IPython>=8.37", # The last version that supports Python 3.10
"markdownify>=1.2.0",
]
all = [
"scrapling[ai,shell]",
]
[project.urls]
Homepage = "https://github.com/D4Vinci/Scrapling"
Documentation = "https://scrapling.readthedocs.io/en/latest/"
Repository = "https://github.com/D4Vinci/Scrapling"
"Bug Tracker" = "https://github.com/D4Vinci/Scrapling/issues"
[project.scripts]
scrapling = "scrapling.cli:main"
[tool.setuptools]
zip-safe = false
include-package-data = true
[tool.setuptools.dynamic]
version = {attr = "scrapling.__version__"}
[tool.setuptools.packages.find]
where = ["."]
include = ["scrapling*"]
pytest.ini
+4 -1
View File
@@ -1,4 +1,7 @@
[pytest]
asyncio_mode = auto
asyncio_default_fixture_loop_scope = function
addopts = -p no:warnings --doctest-modules --ignore=setup.py --verbose
addopts = -p no:warnings --doctest-modules --ignore=setup.py --verbose
markers =
asyncio: marks tests as async
asyncio_fixture_scope = function
ruff.toml
+22
View File
@@ -0,0 +1,22 @@
exclude = [
".git",
".venv",
"__pycache__",
"docs",
".github",
"build",
"dist",
"tests",
"benchmarks.py",
]
# Assume Python 3.9
target-version = "py39"
[lint]
select = ["E", "F", "W"]
ignore = ["E501", "F401", "F811"]
[format]
# Like Black, use double quotes for strings.
quote-style = "double"
scrapling/__init__.py
+18 -31
View File
@@ -1,41 +1,28 @@
__author__ = "Karim Shoair (karim.shoair@pm.me)"
__version__ = "0.2.99"
__version__ = "0.3"
__copyright__ = "Copyright (c) 2024 Karim Shoair"
# A lightweight approach to create lazy loader for each import for backward compatibility
# A lightweight approach to create a lazy loader for each import for backward compatibility
# This will reduces initial memory footprint significantly (only loads what's used)
def __getattr__(name):
if name == 'Fetcher':
from scrapling.fetchers import Fetcher as cls
return cls
elif name == 'Adaptor':
from scrapling.parser import Adaptor as cls
return cls
elif name == 'Adaptors':
from scrapling.parser import Adaptors as cls
return cls
elif name == 'AttributesHandler':
from scrapling.core.custom_types import AttributesHandler as cls
return cls
elif name == 'TextHandler':
from scrapling.core.custom_types import TextHandler as cls
return cls
elif name == 'AsyncFetcher':
from scrapling.fetchers import AsyncFetcher as cls
return cls
elif name == 'StealthyFetcher':
from scrapling.fetchers import StealthyFetcher as cls
return cls
elif name == 'PlayWrightFetcher':
from scrapling.fetchers import PlayWrightFetcher as cls
return cls
elif name == 'CustomFetcher':
from scrapling.fetchers import CustomFetcher as cls
return cls
lazy_imports = {
"Fetcher": ("scrapling.fetchers", "Fetcher"),
"Selector": ("scrapling.parser", "Selector"),
"Selectors": ("scrapling.parser", "Selectors"),
"AttributesHandler": ("scrapling.core.custom_types", "AttributesHandler"),
"TextHandler": ("scrapling.core.custom_types", "TextHandler"),
"AsyncFetcher": ("scrapling.fetchers", "AsyncFetcher"),
"StealthyFetcher": ("scrapling.fetchers", "StealthyFetcher"),
"DynamicFetcher": ("scrapling.fetchers", "DynamicFetcher"),
}
if name in lazy_imports:
module_path, class_name = lazy_imports[name]
module = __import__(module_path, fromlist=[class_name])
return getattr(module, class_name)
else:
raise AttributeError(f"module 'scrapling' has no attribute '{name}'")
__all__ = ['Adaptor', 'Fetcher', 'AsyncFetcher', 'StealthyFetcher', 'PlayWrightFetcher']
__all__ = ["Selector", "Fetcher", "AsyncFetcher", "StealthyFetcher", "DynamicFetcher"]
scrapling/cli.py
+820 -22
View File
@@ -1,38 +1,836 @@
import os
import subprocess
import sys
from pathlib import Path
from subprocess import check_output
from sys import executable as python_executable
import click
from scrapling.core.utils import log
from scrapling.engines.toolbelt import Response
from scrapling.core._types import List, Optional, Dict, Tuple, Any, Callable
from scrapling.fetchers import Fetcher, DynamicFetcher, StealthyFetcher
from scrapling.core.shell import Convertor, _CookieParser, _ParseHeaders
from orjson import loads as json_loads, JSONDecodeError
from click import command, option, Choice, group, argument
__OUTPUT_FILE_HELP__ = "The output file path can be an HTML file, a Markdown file of the HTML content, or the text content itself. Use file extensions (`.html`/`.md`/`.txt`) respectively."
__PACKAGE_DIR__ = Path(__file__).parent
def get_package_dir():
return Path(os.path.dirname(__file__))
def run_command(command, line):
print(f"Installing {line}...")
_ = subprocess.check_call(' '.join(command), shell=True)
def __Execute(cmd: List[str], help_line: str) -> None: # pragma: no cover
print(f"Installing {help_line}...")
_ = check_output(cmd, shell=False) # nosec B603
# I meant to not use try except here
@click.command(help="Install all Scrapling's Fetchers dependencies")
@click.option('-f', '--force', 'force', is_flag=True, default=False, type=bool, help="Force Scrapling to reinstall all Fetchers dependencies")
def install(force):
if force or not get_package_dir().joinpath(".scrapling_dependencies_installed").exists():
run_command([sys.executable, "-m", "playwright", "install", 'chromium'], 'Playwright browsers')
run_command([sys.executable, "-m", "playwright", "install-deps", 'chromium', 'firefox'], 'Playwright dependencies')
run_command([sys.executable, "-m", "camoufox", "fetch", '--browserforge'], 'Camoufox browser and databases')
# if no errors raised by above commands, then we add below file
get_package_dir().joinpath(".scrapling_dependencies_installed").touch()
def __ParseJSONData(json_string: Optional[str] = None) -> Optional[Dict[str, Any]]:
"""Parse JSON string into a Python object"""
if not json_string:
return None
try:
return json_loads(json_string)
except JSONDecodeError as e: # pragma: no cover
raise ValueError(f"Invalid JSON data '{json_string}': {e}")
def __Request_and_Save(
fetcher_func: Callable[..., Response],
url: str,
output_file: str,
css_selector: Optional[str] = None,
**kwargs,
) -> None:
"""Make a request using the specified fetcher function and save the result"""
# Handle relative paths - convert to an absolute path based on the current working directory
output_path = Path(output_file)
if not output_path.is_absolute():
output_path = Path.cwd() / output_file
response = fetcher_func(url, **kwargs)
Convertor.write_content_to_file(response, str(output_path), css_selector)
log.info(f"Content successfully saved to '{output_path}'")
def __ParseExtractArguments(
headers: List[str], cookies: str, params: str, json: Optional[str] = None
) -> Tuple[Dict[str, str], Dict[str, str], Dict[str, str], Optional[Dict[str, str]]]:
"""Parse arguments for extract command"""
parsed_headers, parsed_cookies = _ParseHeaders(headers)
if cookies:
for key, value in _CookieParser(cookies):
try:
parsed_cookies[key] = value
except Exception as e:
raise ValueError(f"Could not parse cookies '{cookies}': {e}")
parsed_json = __ParseJSONData(json)
parsed_params = {}
for param in params:
if "=" in param:
key, value = param.split("=", 1)
parsed_params[key] = value
return parsed_headers, parsed_cookies, parsed_params, parsed_json
def __BuildRequest(
headers: List[str], cookies: str, params: str, json: Optional[str] = None, **kwargs
) -> Dict:
"""Build a request object using the specified arguments"""
# Parse parameters
parsed_headers, parsed_cookies, parsed_params, parsed_json = (
__ParseExtractArguments(headers, cookies, params, json)
)
# Build request arguments
request_kwargs = {
"headers": parsed_headers if parsed_headers else None,
"cookies": parsed_cookies if parsed_cookies else None,
}
if parsed_json:
request_kwargs["json"] = parsed_json
if parsed_params:
request_kwargs["params"] = parsed_params
if "proxy" in kwargs:
request_kwargs["proxy"] = kwargs.pop("proxy")
return {**request_kwargs, **kwargs}
@command(help="Install all Scrapling's Fetchers dependencies")
@option(
"-f",
"--force",
"force",
is_flag=True,
default=False,
type=bool,
help="Force Scrapling to reinstall all Fetchers dependencies",
)
def install(force): # pragma: no cover
if (
force
or not __PACKAGE_DIR__.joinpath(".scrapling_dependencies_installed").exists()
):
__Execute(
[python_executable, "-m", "playwright", "install", "chromium"],
"Playwright browsers",
)
__Execute(
[
python_executable,
"-m",
"playwright",
"install-deps",
"chromium",
"firefox",
],
"Playwright dependencies",
)
__Execute(
[python_executable, "-m", "camoufox", "fetch", "--browserforge"],
"Camoufox browser and databases",
)
# if no errors raised by the above commands, then we add the below file
__PACKAGE_DIR__.joinpath(".scrapling_dependencies_installed").touch()
else:
print('The dependencies are already installed')
print("The dependencies are already installed")
@click.group()
@command(help="Run Scrapling's MCP server (Check the docs for more info).")
def mcp():
from scrapling.core.ai import ScraplingMCPServer
ScraplingMCPServer().serve()
@command(help="Interactive scraping console")
@option(
"-c",
"--code",
"code",
is_flag=False,
default="",
type=str,
help="Evaluate the code in the shell, print the result and exit",
)
@option(
"-L",
"--loglevel",
"level",
is_flag=False,
default="debug",
type=Choice(
["debug", "info", "warning", "error", "critical", "fatal"], case_sensitive=False
),
help="Log level (default: DEBUG)",
)
def shell(code, level):
from scrapling.core.shell import CustomShell
console = CustomShell(code=code, log_level=level)
console.start()
@group(
help="Fetch web pages using various fetchers and extract full/selected HTML content as HTML, Markdown, or extract text content."
)
def extract():
"""Extract content from web pages and save to files"""
pass
@extract.command(
help=f"Perform a GET request and save the content to a file.\n\n{__OUTPUT_FILE_HELP__}"
)
@argument("url", required=True)
@argument("output_file", required=True)
@option(
"--headers",
"-H",
multiple=True,
help='HTTP headers in format "Key: Value" (can be used multiple times)',
)
@option("--cookies", help='Cookies string in format "name1=value1; name2=value2"')
@option(
"--timeout", type=int, default=30, help="Request timeout in seconds (default: 30)"
)
@option("--proxy", help='Proxy URL in format "http://username:password@host:port"')
@option(
"--css-selector",
"-s",
help="CSS selector to extract specific content from the page. It returns all matches.",
)
@option(
"--params",
"-p",
multiple=True,
help='Query parameters in format "key=value" (can be used multiple times)',
)
@option(
"--follow-redirects/--no-follow-redirects",
default=True,
help="Whether to follow redirects (default: True)",
)
@option(
"--verify/--no-verify",
default=True,
help="Whether to verify SSL certificates (default: True)",
)
@option("--impersonate", help="Browser to impersonate (e.g., chrome, firefox).")
@option(
"--stealthy-headers/--no-stealthy-headers",
default=True,
help="Use stealthy browser headers (default: True)",
)
def get(
url,
output_file,
headers,
cookies,
timeout,
proxy,
css_selector,
params,
follow_redirects,
verify,
impersonate,
stealthy_headers,
):
"""
Perform a GET request and save the content to a file.
:param url: Target URL for the request.
:param output_file: Output file path (.md for Markdown, .html for HTML).
:param headers: HTTP headers to include in the request.
:param cookies: Cookies to use in the request.
:param timeout: Number of seconds to wait before timing out.
:param proxy: Proxy URL to use. (Format: "http://username:password@localhost:8030")
:param css_selector: CSS selector to extract specific content.
:param params: Query string parameters for the request.
:param follow_redirects: Whether to follow redirects.
:param verify: Whether to verify HTTPS certificates.
:param impersonate: Browser version to impersonate.
:param stealthy_headers: If enabled, creates and adds real browser headers.
"""
kwargs = __BuildRequest(
headers,
cookies,
params,
None,
timeout=timeout,
follow_redirects=follow_redirects,
verify=verify,
stealthy_headers=stealthy_headers,
impersonate=impersonate,
proxy=proxy,
)
__Request_and_Save(Fetcher.get, url, output_file, css_selector, **kwargs)
@extract.command(
help=f"Perform a POST request and save the content to a file.\n\n{__OUTPUT_FILE_HELP__}"
)
@argument("url", required=True)
@argument("output_file", required=True)
@option(
"--data",
"-d",
help='Form data to include in the request body (as string, ex: "param1=value1&param2=value2")',
)
@option("--json", "-j", help="JSON data to include in the request body (as string)")
@option(
"--headers",
"-H",
multiple=True,
help='HTTP headers in format "Key: Value" (can be used multiple times)',
)
@option("--cookies", help='Cookies string in format "name1=value1; name2=value2"')
@option(
"--timeout", type=int, default=30, help="Request timeout in seconds (default: 30)"
)
@option("--proxy", help='Proxy URL in format "http://username:password@host:port"')
@option(
"--css-selector",
"-s",
help="CSS selector to extract specific content from the page. It returns all matches.",
)
@option(
"--params",
"-p",
multiple=True,
help='Query parameters in format "key=value" (can be used multiple times)',
)
@option(
"--follow-redirects/--no-follow-redirects",
default=True,
help="Whether to follow redirects (default: True)",
)
@option(
"--verify/--no-verify",
default=True,
help="Whether to verify SSL certificates (default: True)",
)
@option("--impersonate", help="Browser to impersonate (e.g., chrome, firefox).")
@option(
"--stealthy-headers/--no-stealthy-headers",
default=True,
help="Use stealthy browser headers (default: True)",
)
def post(
url,
output_file,
data,
json,
headers,
cookies,
timeout,
proxy,
css_selector,
params,
follow_redirects,
verify,
impersonate,
stealthy_headers,
):
"""
Perform a POST request and save the content to a file.
:param url: Target URL for the request.
:param output_file: Output file path (.md for Markdown, .html for HTML).
:param data: Form data to include in the request body. (as string, ex: "param1=value1&param2=value2")
:param json: A JSON serializable object to include in the body of the request.
:param headers: Headers to include in the request.
:param cookies: Cookies to use in the request.
:param timeout: Number of seconds to wait before timing out.
:param proxy: Proxy URL to use.
:param css_selector: CSS selector to extract specific content.
:param params: Query string parameters for the request.
:param follow_redirects: Whether to follow redirects.
:param verify: Whether to verify HTTPS certificates.
:param impersonate: Browser version to impersonate.
:param stealthy_headers: If enabled, creates and adds real browser headers.
"""
kwargs = __BuildRequest(
headers,
cookies,
params,
json,
timeout=timeout,
follow_redirects=follow_redirects,
verify=verify,
stealthy_headers=stealthy_headers,
impersonate=impersonate,
proxy=proxy,
data=data,
)
__Request_and_Save(Fetcher.post, url, output_file, css_selector, **kwargs)
@extract.command(
help=f"Perform a PUT request and save the content to a file.\n\n{__OUTPUT_FILE_HELP__}"
)
@argument("url", required=True)
@argument("output_file", required=True)
@option("--data", "-d", help="Form data to include in the request body")
@option("--json", "-j", help="JSON data to include in the request body (as string)")
@option(
"--headers",
"-H",
multiple=True,
help='HTTP headers in format "Key: Value" (can be used multiple times)',
)
@option("--cookies", help='Cookies string in format "name1=value1; name2=value2"')
@option(
"--timeout", type=int, default=30, help="Request timeout in seconds (default: 30)"
)
@option("--proxy", help='Proxy URL in format "http://username:password@host:port"')
@option(
"--css-selector",
"-s",
help="CSS selector to extract specific content from the page. It returns all matches.",
)
@option(
"--params",
"-p",
multiple=True,
help='Query parameters in format "key=value" (can be used multiple times)',
)
@option(
"--follow-redirects/--no-follow-redirects",
default=True,
help="Whether to follow redirects (default: True)",
)
@option(
"--verify/--no-verify",
default=True,
help="Whether to verify SSL certificates (default: True)",
)
@option("--impersonate", help="Browser to impersonate (e.g., chrome, firefox).")
@option(
"--stealthy-headers/--no-stealthy-headers",
default=True,
help="Use stealthy browser headers (default: True)",
)
def put(
url,
output_file,
data,
json,
headers,
cookies,
timeout,
proxy,
css_selector,
params,
follow_redirects,
verify,
impersonate,
stealthy_headers,
):
"""
Perform a PUT request and save the content to a file.
:param url: Target URL for the request.
:param output_file: Output file path (.md for Markdown, .html for HTML).
:param data: Form data to include in the request body.
:param json: A JSON serializable object to include in the body of the request.
:param headers: Headers to include in the request.
:param cookies: Cookies to use in the request.
:param timeout: Number of seconds to wait before timing out.
:param proxy: Proxy URL to use.
:param css_selector: CSS selector to extract specific content.
:param params: Query string parameters for the request.
:param follow_redirects: Whether to follow redirects.
:param verify: Whether to verify HTTPS certificates.
:param impersonate: Browser version to impersonate.
:param stealthy_headers: If enabled, creates and adds real browser headers.
"""
kwargs = __BuildRequest(
headers,
cookies,
params,
json,
timeout=timeout,
follow_redirects=follow_redirects,
verify=verify,
stealthy_headers=stealthy_headers,
impersonate=impersonate,
proxy=proxy,
data=data,
)
__Request_and_Save(Fetcher.put, url, output_file, css_selector, **kwargs)
@extract.command(
help=f"Perform a DELETE request and save the content to a file.\n\n{__OUTPUT_FILE_HELP__}"
)
@argument("url", required=True)
@argument("output_file", required=True)
@option(
"--headers",
"-H",
multiple=True,
help='HTTP headers in format "Key: Value" (can be used multiple times)',
)
@option("--cookies", help='Cookies string in format "name1=value1; name2=value2"')
@option(
"--timeout", type=int, default=30, help="Request timeout in seconds (default: 30)"
)
@option("--proxy", help='Proxy URL in format "http://username:password@host:port"')
@option(
"--css-selector",
"-s",
help="CSS selector to extract specific content from the page. It returns all matches.",
)
@option(
"--params",
"-p",
multiple=True,
help='Query parameters in format "key=value" (can be used multiple times)',
)
@option(
"--follow-redirects/--no-follow-redirects",
default=True,
help="Whether to follow redirects (default: True)",
)
@option(
"--verify/--no-verify",
default=True,
help="Whether to verify SSL certificates (default: True)",
)
@option("--impersonate", help="Browser to impersonate (e.g., chrome, firefox).")
@option(
"--stealthy-headers/--no-stealthy-headers",
default=True,
help="Use stealthy browser headers (default: True)",
)
def delete(
url,
output_file,
headers,
cookies,
timeout,
proxy,
css_selector,
params,
follow_redirects,
verify,
impersonate,
stealthy_headers,
):
"""
Perform a DELETE request and save the content to a file.
:param url: Target URL for the request.
:param output_file: Output file path (.md for Markdown, .html for HTML).
:param headers: Headers to include in the request.
:param cookies: Cookies to use in the request.
:param timeout: Number of seconds to wait before timing out.
:param proxy: Proxy URL to use.
:param css_selector: CSS selector to extract specific content.
:param params: Query string parameters for the request.
:param follow_redirects: Whether to follow redirects.
:param verify: Whether to verify HTTPS certificates.
:param impersonate: Browser version to impersonate.
:param stealthy_headers: If enabled, creates and adds real browser headers.
"""
kwargs = __BuildRequest(
headers,
cookies,
params,
None,
timeout=timeout,
follow_redirects=follow_redirects,
verify=verify,
stealthy_headers=stealthy_headers,
impersonate=impersonate,
proxy=proxy,
)
__Request_and_Save(Fetcher.delete, url, output_file, css_selector, **kwargs)
@extract.command(
help=f"Use DynamicFetcher to fetch content with browser automation.\n\n{__OUTPUT_FILE_HELP__}"
)
@argument("url", required=True)
@argument("output_file", required=True)
@option(
"--headless/--no-headless",
default=True,
help="Run browser in headless mode (default: True)",
)
@option(
"--disable-resources/--enable-resources",
default=False,
help="Drop unnecessary resources for speed boost (default: False)",
)
@option(
"--network-idle/--no-network-idle",
default=False,
help="Wait for network idle (default: False)",
)
@option(
"--timeout",
type=int,
default=30000,
help="Timeout in milliseconds (default: 30000)",
)
@option(
"--wait",
type=int,
default=0,
help="Additional wait time in milliseconds after page load (default: 0)",
)
@option(
"--css-selector",
"-s",
help="CSS selector to extract specific content from the page. It returns all matches.",
)
@option("--wait-selector", help="CSS selector to wait for before proceeding")
@option("--locale", default="en-US", help="Browser locale (default: en-US)")
@option(
"--stealth/--no-stealth", default=False, help="Enable stealth mode (default: False)"
)
@option(
"--hide-canvas/--show-canvas",
default=False,
help="Add noise to canvas operations (default: False)",
)
@option(
"--disable-webgl/--enable-webgl",
default=False,
help="Disable WebGL support (default: False)",
)
@option("--proxy", help='Proxy URL in format "http://username:password@host:port"')
@option(
"--extra-headers",
"-H",
multiple=True,
help='Extra headers in format "Key: Value" (can be used multiple times)',
)
def fetch(
url,
output_file,
headless,
disable_resources,
network_idle,
timeout,
wait,
css_selector,
wait_selector,
locale,
stealth,
hide_canvas,
disable_webgl,
proxy,
extra_headers,
):
"""
Opens up a browser and fetch content using DynamicFetcher.
:param url: Target url.
:param output_file: Output file path (.md for Markdown, .html for HTML).
:param headless: Run the browser in headless/hidden or headful/visible mode.
:param disable_resources: Drop requests of unnecessary resources for a speed boost.
:param network_idle: Wait for the page until there are no network connections for at least 500 ms.
:param timeout: The timeout in milliseconds that is used in all operations and waits through the page.
:param wait: The time (milliseconds) the fetcher will wait after everything finishes before returning.
:param css_selector: CSS selector to extract specific content.
:param wait_selector: Wait for a specific CSS selector to be in a specific state.
:param locale: Set the locale for the browser.
:param stealth: Enables stealth mode.
:param hide_canvas: Add random noise to canvas operations to prevent fingerprinting.
:param disable_webgl: Disables WebGL and WebGL 2.0 support entirely.
:param proxy: The proxy to be used with requests.
:param extra_headers: Extra headers to add to the request.
"""
# Parse parameters
parsed_headers, _ = _ParseHeaders(extra_headers, False)
# Build request arguments
kwargs = {
"headless": headless,
"disable_resources": disable_resources,
"network_idle": network_idle,
"timeout": timeout,
"locale": locale,
"stealth": stealth,
"hide_canvas": hide_canvas,
"disable_webgl": disable_webgl,
}
if wait > 0:
kwargs["wait"] = wait
if wait_selector:
kwargs["wait_selector"] = wait_selector
if proxy:
kwargs["proxy"] = proxy
if parsed_headers:
kwargs["extra_headers"] = parsed_headers
__Request_and_Save(DynamicFetcher.fetch, url, output_file, css_selector, **kwargs)
@extract.command(
help=f"Use StealthyFetcher to fetch content with advanced stealth features.\n\n{__OUTPUT_FILE_HELP__}"
)
@argument("url", required=True)
@argument("output_file", required=True)
@option(
"--headless/--no-headless",
default=True,
help="Run browser in headless mode (default: True)",
)
@option(
"--block-images/--allow-images",
default=False,
help="Block image loading (default: False)",
)
@option(
"--disable-resources/--enable-resources",
default=False,
help="Drop unnecessary resources for speed boost (default: False)",
)
@option(
"--block-webrtc/--allow-webrtc",
default=False,
help="Block WebRTC entirely (default: False)",
)
@option(
"--humanize/--no-humanize",
default=False,
help="Humanize cursor movement (default: False)",
)
@option(
"--solve-cloudflare/--no-solve-cloudflare",
default=False,
help="Solve Cloudflare challenges (default: False)",
)
@option("--allow-webgl/--block-webgl", default=True, help="Allow WebGL (default: True)")
@option(
"--network-idle/--no-network-idle",
default=False,
help="Wait for network idle (default: False)",
)
@option(
"--disable-ads/--allow-ads",
default=False,
help="Install uBlock Origin addon (default: False)",
)
@option(
"--timeout",
type=int,
default=30000,
help="Timeout in milliseconds (default: 30000)",
)
@option(
"--wait",
type=int,
default=0,
help="Additional wait time in milliseconds after page load (default: 0)",
)
@option(
"--css-selector",
"-s",
help="CSS selector to extract specific content from the page. It returns all matches.",
)
@option("--wait-selector", help="CSS selector to wait for before proceeding")
@option(
"--geoip/--no-geoip",
default=False,
help="Use IP geolocation for timezone/locale (default: False)",
)
@option("--proxy", help='Proxy URL in format "http://username:password@host:port"')
@option(
"--extra-headers",
"-H",
multiple=True,
help='Extra headers in format "Key: Value" (can be used multiple times)',
)
def stealthy_fetch(
url,
output_file,
headless,
block_images,
disable_resources,
block_webrtc,
humanize,
solve_cloudflare,
allow_webgl,
network_idle,
disable_ads,
timeout,
wait,
css_selector,
wait_selector,
geoip,
proxy,
extra_headers,
):
"""
Opens up a browser with advanced stealth features and fetch content using StealthyFetcher.
:param url: Target url.
:param output_file: Output file path (.md for Markdown, .html for HTML).
:param headless: Run the browser in headless/hidden, or headful/visible mode.
:param block_images: Prevent the loading of images through Firefox preferences.
:param disable_resources: Drop requests of unnecessary resources for a speed boost.
:param block_webrtc: Blocks WebRTC entirely.
:param humanize: Humanize the cursor movement.
:param solve_cloudflare: Solves all 3 types of the Cloudflare's Turnstile wait page.
:param allow_webgl: Allow WebGL (recommended to keep enabled).
:param network_idle: Wait for the page until there are no network connections for at least 500 ms.
:param disable_ads: Install the uBlock Origin addon on the browser.
:param timeout: The timeout in milliseconds that is used in all operations and waits through the page.
:param wait: The time (milliseconds) the fetcher will wait after everything finishes before returning.
:param css_selector: CSS selector to extract specific content.
:param wait_selector: Wait for a specific CSS selector to be in a specific state.
:param geoip: Automatically use IP's longitude, latitude, timezone, country, locale.
:param proxy: The proxy to be used with requests.
:param extra_headers: Extra headers to add to the request.
"""
# Parse parameters
parsed_headers, _ = _ParseHeaders(extra_headers, False)
# Build request arguments
kwargs = {
"headless": headless,
"block_images": block_images,
"disable_resources": disable_resources,
"block_webrtc": block_webrtc,
"humanize": humanize,
"solve_cloudflare": solve_cloudflare,
"allow_webgl": allow_webgl,
"network_idle": network_idle,
"disable_ads": disable_ads,
"timeout": timeout,
"geoip": geoip,
}
if wait > 0:
kwargs["wait"] = wait
if wait_selector:
kwargs["wait_selector"] = wait_selector
if proxy:
kwargs["proxy"] = proxy
if parsed_headers:
kwargs["extra_headers"] = parsed_headers
__Request_and_Save(StealthyFetcher.fetch, url, output_file, css_selector, **kwargs)
@group()
def main():
pass
# Adding commands
main.add_command(install)
main.add_command(shell)
main.add_command(extract)
main.add_command(mcp)
scrapling/core/_html_utils.py
+348
View File
@@ -0,0 +1,348 @@
"""
This file is mostly copied from the submodule `w3lib.html` source code to stop downloading the whole library to use a small part of it.
So the goal of doing this is to minimize the memory footprint and keep the library size relatively smaller.
Repo source code: https://github.com/scrapy/w3lib/blob/master/w3lib/html.py
"""
from re import compile as _re_compile, IGNORECASE
from scrapling.core._types import Iterable, Optional, Match, StrOrBytes
_ent_re = _re_compile(
r"&((?P<named>[a-z\d]+)|#(?P<dec>\d+)|#x(?P<hex>[a-f\d]+))(?P<semicolon>;?)",
IGNORECASE,
)
# maps HTML4 entity name to the Unicode code point
name2codepoint = {
"AElig": 0x00C6, # latin capital letter AE = latin capital ligature AE, U+00C6 ISOlat1
"Aacute": 0x00C1, # latin capital letter A with acute, U+00C1 ISOlat1
"Acirc": 0x00C2, # latin capital letter A with circumflex, U+00C2 ISOlat1
"Agrave": 0x00C0, # latin capital letter A with grave = latin capital letter A grave, U+00C0 ISOlat1
"Alpha": 0x0391, # greek capital letter alpha, U+0391
"Aring": 0x00C5, # latin capital letter A with the ring above = latin capital letter A ring, U+00C5 ISOlat1
"Atilde": 0x00C3, # latin capital letter A with tilde, U+00C3 ISOlat1
"Auml": 0x00C4, # latin capital letter A with diaeresis, U+00C4 ISOlat1
"Beta": 0x0392, # greek capital letter beta, U+0392
"Ccedil": 0x00C7, # latin capital letter C with cedilla, U+00C7 ISOlat1
"Chi": 0x03A7, # greek capital letter chi, U+03A7
"Dagger": 0x2021, # double dagger, U+2021 ISOpub
"Delta": 0x0394, # greek capital letter delta, U+0394 ISOgrk3
"ETH": 0x00D0, # latin capital letter ETH, U+00D0 ISOlat1
"Eacute": 0x00C9, # latin capital letter E with acute, U+00C9 ISOlat1
"Ecirc": 0x00CA, # latin capital letter E with circumflex, U+00CA ISOlat1
"Egrave": 0x00C8, # latin capital letter E with grave, U+00C8 ISOlat1
"Epsilon": 0x0395, # greek capital letter epsilon, U+0395
"Eta": 0x0397, # greek capital letter eta, U+0397
"Euml": 0x00CB, # latin capital letter E with diaeresis, U+00CB ISOlat1
"Gamma": 0x0393, # greek capital letter gamma, U+0393 ISOgrk3
"Iacute": 0x00CD, # latin capital letter I with acute, U+00CD ISOlat1
"Icirc": 0x00CE, # latin capital letter I with circumflex, U+00CE ISOlat1
"Igrave": 0x00CC, # latin capital letter I with grave, U+00CC ISOlat1
"Iota": 0x0399, # greek capital letter iota, U+0399
"Iuml": 0x00CF, # latin capital letter I with diaeresis, U+00CF ISOlat1
"Kappa": 0x039A, # greek capital letter kappa, U+039A
"Lambda": 0x039B, # greek capital letter lambda, U+039B ISOgrk3
"Mu": 0x039C, # greek capital letter mu, U+039C
"Ntilde": 0x00D1, # latin capital letter N with tilde, U+00D1 ISOlat1
"Nu": 0x039D, # greek capital letter nu, U+039D
"OElig": 0x0152, # latin capital ligature OE, U+0152 ISOlat2
"Oacute": 0x00D3, # latin capital letter O with acute, U+00D3 ISOlat1
"Ocirc": 0x00D4, # latin capital letter O with circumflex, U+00D4 ISOlat1
"Ograve": 0x00D2, # latin capital letter O with grave, U+00D2 ISOlat1
"Omega": 0x03A9, # greek capital letter omega, U+03A9 ISOgrk3
"Omicron": 0x039F, # greek capital letter omicron, U+039F
"Oslash": 0x00D8, # latin capital letter O with stroke = latin capital letter O slash, U+00D8 ISOlat1
"Otilde": 0x00D5, # latin capital letter O with tilde, U+00D5 ISOlat1
"Ouml": 0x00D6, # latin capital letter O with diaeresis, U+00D6 ISOlat1
"Phi": 0x03A6, # greek capital letter phi, U+03A6 ISOgrk3
"Pi": 0x03A0, # greek capital letter pi, U+03A0 ISOgrk3
"Prime": 0x2033, # double prime = seconds = inches, U+2033 ISOtech
"Psi": 0x03A8, # greek capital letter psi, U+03A8 ISOgrk3
"Rho": 0x03A1, # greek capital letter rho, U+03A1
"Scaron": 0x0160, # latin capital letter S with caron, U+0160 ISOlat2
"Sigma": 0x03A3, # greek capital letter sigma, U+03A3 ISOgrk3
"THORN": 0x00DE, # latin capital letter THORN, U+00DE ISOlat1
"Tau": 0x03A4, # greek capital letter tau, U+03A4
"Theta": 0x0398, # greek capital letter theta, U+0398 ISOgrk3
"Uacute": 0x00DA, # latin capital letter U with acute, U+00DA ISOlat1
"Ucirc": 0x00DB, # latin capital letter U with circumflex, U+00DB ISOlat1
"Ugrave": 0x00D9, # latin capital letter U with grave, U+00D9 ISOlat1
"Upsilon": 0x03A5, # greek capital letter upsilon, U+03A5 ISOgrk3
"Uuml": 0x00DC, # latin capital letter U with diaeresis, U+00DC ISOlat1
"Xi": 0x039E, # greek capital letter xi, U+039E ISOgrk3
"Yacute": 0x00DD, # latin capital letter Y with acute, U+00DD ISOlat1
"Yuml": 0x0178, # latin capital letter Y with diaeresis, U+0178 ISOlat2
"Zeta": 0x0396, # greek capital letter zeta, U+0396
"aacute": 0x00E1, # latin small letter a with acute, U+00E1 ISOlat1
"acirc": 0x00E2, # latin small letter a with circumflex, U+00E2 ISOlat1
"acute": 0x00B4, # acute accent = spacing acute, U+00B4 ISOdia
"aelig": 0x00E6, # latin small letter ae = latin small ligature ae, U+00E6 ISOlat1
"agrave": 0x00E0, # latin small letter a with grave = latin small letter a grave, U+00E0 ISOlat1
"alefsym": 0x2135, # alef symbol = first transfinite cardinal, U+2135 NEW
"alpha": 0x03B1, # greek small letter alpha, U+03B1 ISOgrk3
"amp": 0x0026, # ampersand, U+0026 ISOnum
"and": 0x2227, # logical and = wedge, U+2227 ISOtech
"ang": 0x2220, # angle, U+2220 ISOamso
"aring": 0x00E5, # latin small letter a with the ring above = latin small letter a ring, U+00E5 ISOlat1
"asymp": 0x2248, # almost equal to = asymptotic to, U+2248 ISOamsr
"atilde": 0x00E3, # latin small letter a with tilde, U+00E3 ISOlat1
"auml": 0x00E4, # latin small letter a with diaeresis, U+00E4 ISOlat1
"bdquo": 0x201E, # double low-9 quotation mark, U+201E NEW
"beta": 0x03B2, # greek small letter beta, U+03B2 ISOgrk3
"brvbar": 0x00A6, # broken bar = broken vertical bar, U+00A6 ISOnum
"bull": 0x2022, # bullet = black small circle, U+2022 ISOpub
"cap": 0x2229, # intersection = cap, U+2229 ISOtech
"ccedil": 0x00E7, # latin small letter c with cedilla, U+00E7 ISOlat1
"cedil": 0x00B8, # cedilla = spacing cedilla, U+00B8 ISOdia
"cent": 0x00A2, # cent sign, U+00A2 ISOnum
"chi": 0x03C7, # greek small letter chi, U+03C7 ISOgrk3
"circ": 0x02C6, # modifier letter circumflex accent, U+02C6 ISOpub
"clubs": 0x2663, # black club suit = shamrock, U+2663 ISOpub
"cong": 0x2245, # approximately equal to, U+2245 ISOtech
"copy": 0x00A9, # copyright sign, U+00A9 ISOnum
"crarr": 0x21B5, # downwards arrow with corner leftwards = carriage return, U+21B5 NEW
"cup": 0x222A, # union = cup, U+222A ISOtech
"curren": 0x00A4, # currency sign, U+00A4 ISOnum
"dArr": 0x21D3, # downwards double arrow, U+21D3 ISOamsa
"dagger": 0x2020, # dagger, U+2020 ISOpub
"darr": 0x2193, # downwards arrow, U+2193 ISOnum
"deg": 0x00B0, # degree sign, U+00B0 ISOnum
"delta": 0x03B4, # greek small letter delta, U+03B4 ISOgrk3
"diams": 0x2666, # black diamond suit, U+2666 ISOpub
"divide": 0x00F7, # division sign, U+00F7 ISOnum
"eacute": 0x00E9, # latin small letter e with acute, U+00E9 ISOlat1
"ecirc": 0x00EA, # latin small letter e with circumflex, U+00EA ISOlat1
"egrave": 0x00E8, # latin small letter e with grave, U+00E8 ISOlat1
"empty": 0x2205, # empty set = null set = diameter, U+2205 ISOamso
"emsp": 0x2003, # em space, U+2003 ISOpub
"ensp": 0x2002, # en space, U+2002 ISOpub
"epsilon": 0x03B5, # greek small letter epsilon, U+03B5 ISOgrk3
"equiv": 0x2261, # identical to, U+2261 ISOtech
"eta": 0x03B7, # greek small letter eta, U+03B7 ISOgrk3
"eth": 0x00F0, # latin small letter eth, U+00F0 ISOlat1
"euml": 0x00EB, # latin small letter e with diaeresis, U+00EB ISOlat1
"euro": 0x20AC, # euro sign, U+20AC NEW
"exist": 0x2203, # there exists, U+2203 ISOtech
"fnof": 0x0192, # latin small f with hook = function = florin, U+0192 ISOtech
"forall": 0x2200, # for all, U+2200 ISOtech
"frac12": 0x00BD, # vulgar fraction one half = fraction one half, U+00BD ISOnum
"frac14": 0x00BC, # vulgar fraction one quarter = fraction one quarter, U+00BC ISOnum
"frac34": 0x00BE, # vulgar fraction three quarters = fraction three quarters, U+00BE ISOnum
"frasl": 0x2044, # fraction slash, U+2044 NEW
"gamma": 0x03B3, # greek small letter gamma, U+03B3 ISOgrk3
"ge": 0x2265, # greater-than or equal to, U+2265 ISOtech
"gt": 0x003E, # greater-than sign, U+003E ISOnum
"hArr": 0x21D4, # left right double arrow, U+21D4 ISOamsa
"harr": 0x2194, # left right arrow, U+2194 ISOamsa
"hearts": 0x2665, # black heart suit = valentine, U+2665 ISOpub
"hellip": 0x2026, # horizontal ellipsis = three dot leader, U+2026 ISOpub
"iacute": 0x00ED, # latin small letter i with acute, U+00ED ISOlat1
"icirc": 0x00EE, # latin small letter i with circumflex, U+00EE ISOlat1
"iexcl": 0x00A1, # inverted exclamation mark, U+00A1 ISOnum
"igrave": 0x00EC, # latin small letter i with grave, U+00EC ISOlat1
"image": 0x2111, # blackletter capital I = imaginary part, U+2111 ISOamso
"infin": 0x221E, # infinity, U+221E ISOtech
"int": 0x222B, # integral, U+222B ISOtech
"iota": 0x03B9, # greek small letter iota, U+03B9 ISOgrk3
"iquest": 0x00BF, # inverted question mark = turned question mark, U+00BF ISOnum
"isin": 0x2208, # element of, U+2208 ISOtech
"iuml": 0x00EF, # latin small letter i with diaeresis, U+00EF ISOlat1
"kappa": 0x03BA, # greek small letter kappa, U+03BA ISOgrk3
"lArr": 0x21D0, # leftwards double arrow, U+21D0 ISOtech
"lambda": 0x03BB, # greek small letter lambda, U+03BB ISOgrk3
"lang": 0x2329, # left-pointing angle bracket = bra, U+2329 ISOtech
"laquo": 0x00AB, # left-pointing double angle quotation mark = left pointing guillemet, U+00AB ISOnum
"larr": 0x2190, # leftwards arrow, U+2190 ISOnum
"lceil": 0x2308, # left ceiling = apl upstile, U+2308 ISOamsc
"ldquo": 0x201C, # left double quotation mark, U+201C ISOnum
"le": 0x2264, # less-than or equal to, U+2264 ISOtech
"lfloor": 0x230A, # left floor = apl downstile, U+230A ISOamsc
"lowast": 0x2217, # asterisk operator, U+2217 ISOtech
"loz": 0x25CA, # lozenge, U+25CA ISOpub
"lrm": 0x200E, # left-to-right mark, U+200E NEW RFC 2070
"lsaquo": 0x2039, # single left-pointing angle quotation mark, U+2039 ISO proposed
"lsquo": 0x2018, # left single quotation mark, U+2018 ISOnum
"lt": 0x003C, # less-than sign, U+003C ISOnum
"macr": 0x00AF, # macron = spacing macron = overline = APL overbar, U+00AF ISOdia
"mdash": 0x2014, # em dash, U+2014 ISOpub
"micro": 0x00B5, # micro sign, U+00B5 ISOnum
"middot": 0x00B7, # middle dot = Georgian comma = Greek middle dot, U+00B7 ISOnum
"minus": 0x2212, # minus sign, U+2212 ISOtech
"mu": 0x03BC, # greek small letter mu, U+03BC ISOgrk3
"nabla": 0x2207, # nabla = backward difference, U+2207 ISOtech
"nbsp": 0x00A0, # no-break space = non-breaking space, U+00A0 ISOnum
"ndash": 0x2013, # en dash, U+2013 ISOpub
"ne": 0x2260, # not equal to, U+2260 ISOtech
"ni": 0x220B, # contains as member, U+220B ISOtech
"not": 0x00AC, # not sign, U+00AC ISOnum
"notin": 0x2209, # not an element of, U+2209 ISOtech
"nsub": 0x2284, # not a subset of, U+2284 ISOamsn
"ntilde": 0x00F1, # latin small letter n with tilde, U+00F1 ISOlat1
"nu": 0x03BD, # greek small letter nu, U+03BD ISOgrk3
"oacute": 0x00F3, # latin small letter o with acute, U+00F3 ISOlat1
"ocirc": 0x00F4, # latin small letter o with circumflex, U+00F4 ISOlat1
"oelig": 0x0153, # latin small ligature oe, U+0153 ISOlat2
"ograve": 0x00F2, # latin small letter o with grave, U+00F2 ISOlat1
"oline": 0x203E, # overline = spacing overscore, U+203E NEW
"omega": 0x03C9, # greek small letter omega, U+03C9 ISOgrk3
"omicron": 0x03BF, # greek small letter omicron, U+03BF NEW
"oplus": 0x2295, # circled plus = direct sum, U+2295 ISOamsb
"or": 0x2228, # logical or = vee, U+2228 ISOtech
"ordf": 0x00AA, # feminine ordinal indicator, U+00AA ISOnum
"ordm": 0x00BA, # masculine ordinal indicator, U+00BA ISOnum
"oslash": 0x00F8, # latin small letter o with stroke, = latin small letter o slash, U+00F8 ISOlat1
"otilde": 0x00F5, # latin small letter o with tilde, U+00F5 ISOlat1
"otimes": 0x2297, # circled times = vector product, U+2297 ISOamsb
"ouml": 0x00F6, # latin small letter o with diaeresis, U+00F6 ISOlat1
"para": 0x00B6, # pilcrow sign = paragraph sign, U+00B6 ISOnum
"part": 0x2202, # partial differential, U+2202 ISOtech
"permil": 0x2030, # per mille sign, U+2030 ISOtech
"perp": 0x22A5, # up tack = orthogonal to = perpendicular, U+22A5 ISOtech
"phi": 0x03C6, # greek small letter phi, U+03C6 ISOgrk3
"pi": 0x03C0, # greek small letter pi, U+03C0 ISOgrk3
"piv": 0x03D6, # greek pi symbol, U+03D6 ISOgrk3
"plusmn": 0x00B1, # plus-minus sign = plus-or-minus sign, U+00B1 ISOnum
"pound": 0x00A3, # pound sign, U+00A3 ISOnum
"prime": 0x2032, # prime = minutes = feet, U+2032 ISOtech
"prod": 0x220F, # n-ary product = product sign, U+220F ISOamsb
"prop": 0x221D, # proportional to, U+221D ISOtech
"psi": 0x03C8, # greek small letter psi, U+03C8 ISOgrk3
"quot": 0x0022, # quotation mark = APL quote, U+0022 ISOnum
"rArr": 0x21D2, # rightwards double arrow, U+21D2 ISOtech
"radic": 0x221A, # square root = radical sign, U+221A ISOtech
"rang": 0x232A, # right-pointing angle bracket = ket, U+232A ISOtech
"raquo": 0x00BB, # right-pointing double angle quotation mark = right pointing guillemet, U+00BB ISOnum
"rarr": 0x2192, # rightwards arrow, U+2192 ISOnum
"rceil": 0x2309, # right ceiling, U+2309 ISOamsc
"rdquo": 0x201D, # right double quotation mark, U+201D ISOnum
"real": 0x211C, # blackletter capital R = real part symbol, U+211C ISOamso
"reg": 0x00AE, # registered sign = registered trade mark sign, U+00AE ISOnum
"rfloor": 0x230B, # right floor, U+230B ISOamsc
"rho": 0x03C1, # greek small letter rho, U+03C1 ISOgrk3
"rlm": 0x200F, # right-to-left mark, U+200F NEW RFC 2070
"rsaquo": 0x203A, # single right-pointing angle quotation mark, U+203A ISO proposed
"rsquo": 0x2019, # right single quotation mark, U+2019 ISOnum
"sbquo": 0x201A, # single low-9 quotation mark, U+201A NEW
"scaron": 0x0161, # latin small letter s with caron, U+0161 ISOlat2
"sdot": 0x22C5, # dot operator, U+22C5 ISOamsb
"sect": 0x00A7, # section sign, U+00A7 ISOnum
"shy": 0x00AD, # soft hyphen = discretionary hyphen, U+00AD ISOnum
"sigma": 0x03C3, # greek small letter sigma, U+03C3 ISOgrk3
"sigmaf": 0x03C2, # greek small letter final sigma, U+03C2 ISOgrk3
"sim": 0x223C, # tilde operator = varies with = similar to, U+223C ISOtech
"spades": 0x2660, # black spade suit, U+2660 ISOpub
"sub": 0x2282, # subset of, U+2282 ISOtech
"sube": 0x2286, # subset of or equal to, U+2286 ISOtech
"sum": 0x2211, # n-ary summation, U+2211 ISOamsb
"sup": 0x2283, # superset of, U+2283 ISOtech
"sup1": 0x00B9, # superscript one = superscript digit one, U+00B9 ISOnum
"sup2": 0x00B2, # superscript two = superscript digit two = squared, U+00B2 ISOnum
"sup3": 0x00B3, # superscript three = superscript digit three = cubed, U+00B3 ISOnum
"supe": 0x2287, # superset of or equal to, U+2287 ISOtech
"szlig": 0x00DF, # latin small letter sharp s = ess-zed, U+00DF ISOlat1
"tau": 0x03C4, # greek small letter tau, U+03C4 ISOgrk3
"there4": 0x2234, # therefore, U+2234 ISOtech
"theta": 0x03B8, # greek small letter theta, U+03B8 ISOgrk3
"thetasym": 0x03D1, # greek small letter theta symbol, U+03D1 NEW
"thinsp": 0x2009, # thin space, U+2009 ISOpub
"thorn": 0x00FE, # latin small letter thorn with, U+00FE ISOlat1
"tilde": 0x02DC, # small tilde, U+02DC ISOdia
"times": 0x00D7, # multiplication sign, U+00D7 ISOnum
"trade": 0x2122, # trade mark sign, U+2122 ISOnum
"uArr": 0x21D1, # upwards double arrow, U+21D1 ISOamsa
"uacute": 0x00FA, # latin small letter u with acute, U+00FA ISOlat1
"uarr": 0x2191, # upwards arrow, U+2191 ISOnum
"ucirc": 0x00FB, # latin small letter u with circumflex, U+00FB ISOlat1
"ugrave": 0x00F9, # latin small letter u with grave, U+00F9 ISOlat1
"uml": 0x00A8, # diaeresis = spacing diaeresis, U+00A8 ISOdia
"upsih": 0x03D2, # greek upsilon with hook symbol, U+03D2 NEW
"upsilon": 0x03C5, # greek small letter upsilon, U+03C5 ISOgrk3
"uuml": 0x00FC, # latin small letter u with diaeresis, U+00FC ISOlat1
"weierp": 0x2118, # script capital P = power set = Weierstrass p, U+2118 ISOamso
"xi": 0x03BE, # greek small letter xi, U+03BE ISOgrk3
"yacute": 0x00FD, # latin small letter y with acute, U+00FD ISOlat1
"yen": 0x00A5, # yen sign = yuan sign, U+00A5 ISOnum
"yuml": 0x00FF, # latin small letter y with diaeresis, U+00FF ISOlat1
"zeta": 0x03B6, # greek small letter zeta, U+03B6 ISOgrk3
"zwj": 0x200D, # zero width joiner, U+200D NEW RFC 2070
"zwnj": 0x200C, # zero width non-joiner, U+200C NEW RFC 2070
}
def to_unicode(
text: StrOrBytes, encoding: Optional[str] = None, errors: str = "strict"
) -> str:
"""Return the Unicode representation of a bytes object `text`. If `text`
is already a Unicode object, return it as-is."""
if isinstance(text, str):
return text
if not isinstance(text, (bytes, str)):
raise TypeError(
f"to_unicode must receive bytes or str, got {type(text).__name__}"
)
if encoding is None:
encoding = "utf-8"
return text.decode(encoding, errors)
def _replace_entities(
text: StrOrBytes,
keep: Iterable[str] = (),
remove_illegal: bool = True,
encoding: str = "utf-8",
) -> str:
"""Remove entities from the given `text` by converting them to their
corresponding Unicode character.
`text` can be a Unicode string or a byte string encoded in the given
`encoding` (which defaults to 'utf-8').
If `keep` is passed (with a list of entity names), those entities will
be kept (they won't be removed).
It supports both numeric entities (``&#nnnn;`` and ``&#hhhh;``)
and named entities (such as ``&nbsp;`` or ``&gt;``).
If `remove_illegal` is ``True``, entities that can't be converted are removed.
If `remove_illegal` is ``False``, entities that can't be converted are kept "as
is". For more information, see the tests.
Always returns a Unicode string (with the entities removed).
>>> _replace_entities(b'Price: &pound;100')
'Price: \\xa3100'
>>> print(_replace_entities(b'Price: &pound;100'))
Price: £100
>>>
"""
def convert_entity(m: Match[str]) -> str:
groups = m.groupdict()
number = None
if groups.get("dec"):
number = int(groups["dec"], 10)
elif groups.get("hex"):
number = int(groups["hex"], 16)
elif groups.get("named"):
entity_name = groups["named"]
if entity_name.lower() in keep:
return m.group(0)
number = name2codepoint.get(entity_name) or name2codepoint.get(
entity_name.lower()
)
if number is not None:
# Browsers typically
# interpret numeric character references in the 80-9F range as representing the characters mapped
# to bytes 80-9F in the Windows-1252 encoding. For more info
# see: http://en.wikipedia.org/wiki/Character_encodings_in_HTML
try:
if 0x80 <= number <= 0x9F:
return bytes((number,)).decode("cp1252")
return chr(number)
except (ValueError, OverflowError): # pragma: no cover
pass
return "" if remove_illegal and groups.get("semicolon") else m.group(0)
return _ent_re.sub(convert_entity, to_unicode(text, encoding))
scrapling/core/_types.py
+35 -18
View File
@@ -2,26 +2,43 @@
Type definitions for type checking purposes.
"""
from typing import (TYPE_CHECKING, Any, Callable, Dict, Generator, Iterable,
List, Literal, Optional, Pattern, Tuple, Type, TypeVar,
Union)
from typing import (
TYPE_CHECKING,
cast,
overload,
Any,
Callable,
Dict,
Generator,
Iterable,
List,
Literal,
Optional,
Pattern,
Tuple,
TypeVar,
Union,
Match,
Mapping,
Awaitable,
Protocol,
SupportsIndex,
)
SUPPORTED_HTTP_METHODS = Literal["GET", "POST", "PUT", "DELETE"]
SelectorWaitStates = Literal["attached", "detached", "hidden", "visible"]
PageLoadStates = Literal["commit", "domcontentloaded", "load", "networkidle"]
extraction_types = Literal["text", "html", "markdown"]
StrOrBytes = Union[str, bytes]
try:
from typing import Protocol
except ImportError:
# Added in Python 3.8
Protocol = object
# Python 3.11+
from typing import Self # novermin
except ImportError: # pragma: no cover
try:
from typing_extensions import Self # Backport
except ImportError:
from typing import TypeVar
try:
from typing import SupportsIndex
except ImportError:
# 'SupportsIndex' got added in Python 3.8
SupportsIndex = None
if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
else:
Self = object
Self = object
scrapling/core/ai.py
+611
View File
@@ -0,0 +1,611 @@
from asyncio import gather
from mcp.server.fastmcp import FastMCP
from pydantic import BaseModel, Field
from scrapling.core.shell import Convertor
from scrapling.engines.toolbelt import Response as _ScraplingResponse
from scrapling.fetchers import (
Fetcher,
FetcherSession,
DynamicFetcher,
AsyncDynamicSession,
StealthyFetcher,
AsyncStealthySession,
)
from scrapling.core._types import (
Optional,
Tuple,
extraction_types,
Mapping,
Dict,
List,
SelectorWaitStates,
Generator,
)
from curl_cffi.requests import (
BrowserTypeLiteral,
)
class ResponseModel(BaseModel):
"""Request's response information structure."""
status: int = Field(description="The status code returned by the website.")
content: list[str] = Field(
description="The content as Markdown/HTML or the text content of the page."
)
url: str = Field(
description="The URL given by the user that resulted in this response."
)
def _ContentTranslator(
content: Generator[str, None, None], page: _ScraplingResponse
) -> ResponseModel:
"""Convert a content generator to a list of ResponseModel objects."""
return ResponseModel(
status=page.status, content=[result for result in content], url=page.url
)
class ScraplingMCPServer:
_server = FastMCP(name="Scrapling")
@staticmethod
@_server.tool()
def get(
url: str,
impersonate: Optional[BrowserTypeLiteral] = "chrome",
extraction_type: extraction_types = "markdown",
css_selector: Optional[str] = None,
main_content_only: bool = True,
params: Optional[Dict | List | Tuple] = None,
headers: Optional[Mapping[str, Optional[str]]] = None,
cookies: Optional[Dict[str, str] | list[tuple[str, str]]] = None,
timeout: Optional[int | float] = 30,
follow_redirects: bool = True,
max_redirects: int = 30,
retries: Optional[int] = 3,
retry_delay: Optional[int] = 1,
proxy: Optional[str] = None,
proxy_auth: Optional[Tuple[str, str]] = None,
auth: Optional[Tuple[str, str]] = None,
verify: Optional[bool] = True,
http3: Optional[bool] = False,
stealthy_headers: Optional[bool] = True,
) -> ResponseModel:
"""Make GET HTTP request to a URL and return a structured output of the result.
Note: This is only suitable for low-mid protection levels. For high-protection levels or websites that require JS loading, use the other tools directly.
Note: If the `css_selector` resolves to more than one element, all the elements will be returned.
:param url: The URL to request.
:param impersonate: Browser version to impersonate its fingerprint. It's using the latest chrome version by default.
:param extraction_type: The type of content to extract from the page. Defaults to "markdown". Options are:
- Markdown will convert the page content to Markdown format.
- HTML will return the raw HTML content of the page.
- Text will return the text content of the page.
:param css_selector: CSS selector to extract the content from the page. If main_content_only is True, then it will be executed on the main content of the page. Defaults to None.
:param main_content_only: Whether to extract only the main content of the page. Defaults to True. The main content here is the data inside the `<body>` tag.
:param params: Query string parameters for the request.
:param headers: Headers to include in the request.
:param cookies: Cookies to use in the request.
:param timeout: Number of seconds to wait before timing out.
:param follow_redirects: Whether to follow redirects. Defaults to True.
:param max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited.
:param retries: Number of retry attempts. Defaults to 3.
:param retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second.
:param proxy: Proxy URL to use. Format: "http://username:password@localhost:8030".
Cannot be used together with the `proxies` parameter.
:param proxy_auth: HTTP basic auth for proxy, tuple of (username, password).
:param auth: HTTP basic auth tuple of (username, password). Only basic auth is supported.
:param verify: Whether to verify HTTPS certificates.
:param http3: Whether to use HTTP3. Defaults to False. It might be problematic if used it with `impersonate`.
:param stealthy_headers: If enabled (default), it creates and adds real browser headers. It also sets the referer header as if this request came from a Google search of URL's domain.
"""
page = Fetcher.get(
url,
auth=auth,
proxy=proxy,
http3=http3,
verify=verify,
params=params,
proxy_auth=proxy_auth,
retry_delay=retry_delay,
stealthy_headers=stealthy_headers,
impersonate=impersonate,
headers=headers,
cookies=cookies,
timeout=timeout,
retries=retries,
max_redirects=max_redirects,
follow_redirects=follow_redirects,
)
return _ContentTranslator(
Convertor._extract_content(
page,
css_selector=css_selector,
extraction_type=extraction_type,
main_content_only=main_content_only,
),
page,
)
@staticmethod
@_server.tool()
async def bulk_get(
urls: Tuple[str, ...],
impersonate: Optional[BrowserTypeLiteral] = "chrome",
extraction_type: extraction_types = "markdown",
css_selector: Optional[str] = None,
main_content_only: bool = True,
params: Optional[Dict | List | Tuple] = None,
headers: Optional[Mapping[str, Optional[str]]] = None,
cookies: Optional[Dict[str, str] | list[tuple[str, str]]] = None,
timeout: Optional[int | float] = 30,
follow_redirects: bool = True,
max_redirects: int = 30,
retries: Optional[int] = 3,
retry_delay: Optional[int] = 1,
proxy: Optional[str] = None,
proxy_auth: Optional[Tuple[str, str]] = None,
auth: Optional[Tuple[str, str]] = None,
verify: Optional[bool] = True,
http3: Optional[bool] = False,
stealthy_headers: Optional[bool] = True,
) -> List[ResponseModel]:
"""Make GET HTTP request to a group of URLs and for each URL, return a structured output of the result.
Note: This is only suitable for low-mid protection levels. For high-protection levels or websites that require JS loading, use the other tools directly.
Note: If the `css_selector` resolves to more than one element, all the elements will be returned.
:param urls: A tuple of the URLs to request.
:param impersonate: Browser version to impersonate its fingerprint. It's using the latest chrome version by default.
:param extraction_type: The type of content to extract from the page. Defaults to "markdown". Options are:
- Markdown will convert the page content to Markdown format.
- HTML will return the raw HTML content of the page.
- Text will return the text content of the page.
:param css_selector: CSS selector to extract the content from the page. If main_content_only is True, then it will be executed on the main content of the page. Defaults to None.
:param main_content_only: Whether to extract only the main content of the page. Defaults to True. The main content here is the data inside the `<body>` tag.
:param params: Query string parameters for the request.
:param headers: Headers to include in the request.
:param cookies: Cookies to use in the request.
:param timeout: Number of seconds to wait before timing out.
:param follow_redirects: Whether to follow redirects. Defaults to True.
:param max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited.
:param retries: Number of retry attempts. Defaults to 3.
:param retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second.
:param proxy: Proxy URL to use. Format: "http://username:password@localhost:8030".
Cannot be used together with the `proxies` parameter.
:param proxy_auth: HTTP basic auth for proxy, tuple of (username, password).
:param auth: HTTP basic auth tuple of (username, password). Only basic auth is supported.
:param verify: Whether to verify HTTPS certificates.
:param http3: Whether to use HTTP3. Defaults to False. It might be problematic if used it with `impersonate`.
:param stealthy_headers: If enabled (default), it creates and adds real browser headers. It also sets the referer header as if this request came from a Google search of URL's domain.
"""
async with FetcherSession() as session:
tasks = [
session.get(
url,
auth=auth,
proxy=proxy,
http3=http3,
verify=verify,
params=params,
headers=headers,
cookies=cookies,
timeout=timeout,
retries=retries,
proxy_auth=proxy_auth,
retry_delay=retry_delay,
impersonate=impersonate,
max_redirects=max_redirects,
follow_redirects=follow_redirects,
stealthy_headers=stealthy_headers,
)
for url in urls
]
responses = await gather(*tasks)
return [
_ContentTranslator(
Convertor._extract_content(
page,
css_selector=css_selector,
extraction_type=extraction_type,
main_content_only=main_content_only,
),
page,
)
for page in responses
]
@staticmethod
@_server.tool()
async def fetch(
url: str,
extraction_type: extraction_types = "markdown",
css_selector: Optional[str] = None,
main_content_only: bool = True,
headless: bool = False,
google_search: bool = True,
hide_canvas: bool = False,
disable_webgl: bool = False,
real_chrome: bool = False,
stealth: bool = False,
wait: int | float = 0,
proxy: Optional[str | Dict[str, str]] = None,
locale: str = "en-US",
extra_headers: Optional[Dict[str, str]] = None,
useragent: Optional[str] = None,
cdp_url: Optional[str] = None,
timeout: int | float = 30000,
disable_resources: bool = False,
wait_selector: Optional[str] = None,
cookies: Optional[List[Dict]] = None,
network_idle: bool = False,
wait_selector_state: SelectorWaitStates = "attached",
) -> ResponseModel:
"""Use playwright to open a browser to fetch a URL and return a structured output of the result.
Note: This is only suitable for low-mid protection levels.
Note: If the `css_selector` resolves to more than one element, all the elements will be returned.
:param url: The URL to request.
:param extraction_type: The type of content to extract from the page. Defaults to "markdown". Options are:
- Markdown will convert the page content to Markdown format.
- HTML will return the raw HTML content of the page.
- Text will return the text content of the page.
:param css_selector: CSS selector to extract the content from the page. If main_content_only is True, then it will be executed on the main content of the page. Defaults to None.
:param main_content_only: Whether to extract only the main content of the page. Defaults to True. The main content here is the data inside the `<body>` tag.
:param headless: Run the browser in headless/hidden (default), or headful/visible mode.
:param disable_resources: Drop requests of unnecessary resources for a speed boost. It depends, but it made requests ~25% faster in my tests for some websites.
Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
This can help save your proxy usage but be careful with this option as it makes some websites never finish loading.
:param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it.
:param cookies: Set cookies for the next request. It should be in a dictionary format that Playwright accepts.
:param network_idle: Wait for the page until there are no network connections for at least 500 ms.
:param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
:param wait_selector: Wait for a specific CSS selector to be in a specific state.
:param locale: Set the locale for the browser if wanted. The default value is `en-US`.
:param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.
:param stealth: Enables stealth mode, check the documentation to see what stealth mode does currently.
:param real_chrome: If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch an instance of your browser and use it.
:param hide_canvas: Add random noise to canvas operations to prevent fingerprinting.
:param disable_webgl: Disables WebGL and WebGL 2.0 support entirely.
:param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers/NSTBrowser through CDP.
:param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name.
:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
:param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.
"""
page = await DynamicFetcher.async_fetch(
url,
wait=wait,
proxy=proxy,
locale=locale,
timeout=timeout,
cookies=cookies,
stealth=stealth,
cdp_url=cdp_url,
headless=headless,
useragent=useragent,
hide_canvas=hide_canvas,
real_chrome=real_chrome,
network_idle=network_idle,
wait_selector=wait_selector,
disable_webgl=disable_webgl,
extra_headers=extra_headers,
google_search=google_search,
disable_resources=disable_resources,
wait_selector_state=wait_selector_state,
)
return _ContentTranslator(
Convertor._extract_content(
page,
css_selector=css_selector,
extraction_type=extraction_type,
main_content_only=main_content_only,
),
page,
)
@staticmethod
@_server.tool()
async def bulk_fetch(
urls: Tuple[str, ...],
extraction_type: extraction_types = "markdown",
css_selector: Optional[str] = None,
main_content_only: bool = True,
headless: bool = False,
google_search: bool = True,
hide_canvas: bool = False,
disable_webgl: bool = False,
real_chrome: bool = False,
stealth: bool = False,
wait: int | float = 0,
proxy: Optional[str | Dict[str, str]] = None,
locale: str = "en-US",
extra_headers: Optional[Dict[str, str]] = None,
useragent: Optional[str] = None,
cdp_url: Optional[str] = None,
timeout: int | float = 30000,
disable_resources: bool = False,
wait_selector: Optional[str] = None,
cookies: Optional[List[Dict]] = None,
network_idle: bool = False,
wait_selector_state: SelectorWaitStates = "attached",
) -> List[ResponseModel]:
"""Use playwright to open a browser, then fetch a group of URLs at the same time, and for each page return a structured output of the result.
Note: This is only suitable for low-mid protection levels.
Note: If the `css_selector` resolves to more than one element, all the elements will be returned.
:param urls: A tuple of the URLs to request.
:param extraction_type: The type of content to extract from the page. Defaults to "markdown". Options are:
- Markdown will convert the page content to Markdown format.
- HTML will return the raw HTML content of the page.
- Text will return the text content of the page.
:param css_selector: CSS selector to extract the content from the page. If main_content_only is True, then it will be executed on the main content of the page. Defaults to None.
:param main_content_only: Whether to extract only the main content of the page. Defaults to True. The main content here is the data inside the `<body>` tag.
:param headless: Run the browser in headless/hidden (default), or headful/visible mode.
:param disable_resources: Drop requests of unnecessary resources for a speed boost. It depends, but it made requests ~25% faster in my tests for some websites.
Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
This can help save your proxy usage but be careful with this option as it makes some websites never finish loading.
:param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it.
:param cookies: Set cookies for the next request. It should be in a dictionary format that Playwright accepts.
:param network_idle: Wait for the page until there are no network connections for at least 500 ms.
:param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
:param wait_selector: Wait for a specific CSS selector to be in a specific state.
:param locale: Set the locale for the browser if wanted. The default value is `en-US`.
:param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.
:param stealth: Enables stealth mode, check the documentation to see what stealth mode does currently.
:param real_chrome: If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch an instance of your browser and use it.
:param hide_canvas: Add random noise to canvas operations to prevent fingerprinting.
:param disable_webgl: Disables WebGL and WebGL 2.0 support entirely.
:param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers/NSTBrowser through CDP.
:param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name.
:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
:param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.
"""
async with AsyncDynamicSession(
wait=wait,
proxy=proxy,
locale=locale,
timeout=timeout,
cookies=cookies,
stealth=stealth,
cdp_url=cdp_url,
headless=headless,
max_pages=len(urls),
useragent=useragent,
hide_canvas=hide_canvas,
real_chrome=real_chrome,
network_idle=network_idle,
wait_selector=wait_selector,
google_search=google_search,
disable_webgl=disable_webgl,
extra_headers=extra_headers,
disable_resources=disable_resources,
wait_selector_state=wait_selector_state,
) as session:
tasks = [session.fetch(url) for url in urls]
responses = await gather(*tasks)
return [
_ContentTranslator(
Convertor._extract_content(
page,
css_selector=css_selector,
extraction_type=extraction_type,
main_content_only=main_content_only,
),
page,
)
for page in responses
]
@staticmethod
@_server.tool()
async def stealthy_fetch(
url: str,
extraction_type: extraction_types = "markdown",
css_selector: Optional[str] = None,
main_content_only: bool = True,
headless: bool = True, # noqa: F821
block_images: bool = False,
disable_resources: bool = False,
block_webrtc: bool = False,
allow_webgl: bool = True,
network_idle: bool = False,
humanize: bool | float = True,
solve_cloudflare: bool = False,
wait: int | float = 0,
timeout: int | float = 30000,
wait_selector: Optional[str] = None,
addons: Optional[List[str]] = None,
wait_selector_state: SelectorWaitStates = "attached",
cookies: Optional[List[Dict]] = None,
google_search: bool = True,
extra_headers: Optional[Dict[str, str]] = None,
proxy: Optional[str | Dict[str, str]] = None,
os_randomize: bool = False,
disable_ads: bool = False,
geoip: bool = False,
additional_args: Optional[Dict] = None,
) -> ResponseModel:
"""Use Scrapling's version of the Camoufox browser to fetch a URL and return a structured output of the result.
Note: This is best suitable for high protection levels. It's slower than the other tools.
Note: If the `css_selector` resolves to more than one element, all the elements will be returned.
:param url: The URL to request.
:param extraction_type: The type of content to extract from the page. Defaults to "markdown". Options are:
- Markdown will convert the page content to Markdown format.
- HTML will return the raw HTML content of the page.
- Text will return the text content of the page.
:param css_selector: CSS selector to extract the content from the page. If main_content_only is True, then it will be executed on the main content of the page. Defaults to None.
:param main_content_only: Whether to extract only the main content of the page. Defaults to True. The main content here is the data inside the `<body>` tag.
:param headless: Run the browser in headless/hidden (default), or headful/visible mode.
:param block_images: Prevent the loading of images through Firefox preferences.
This can help save your proxy usage but be careful with this option as it makes some websites never finish loading.
:param disable_resources: Drop requests of unnecessary resources for a speed boost. It depends, but it made requests ~25% faster in my tests for some websites.
Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
This can help save your proxy usage but be careful with this option as it makes some websites never finish loading.
:param block_webrtc: Blocks WebRTC entirely.
:param cookies: Set cookies for the next request.
:param addons: List of Firefox addons to use. Must be paths to extracted addons.
:param humanize: Humanize the cursor movement. Takes either True or the MAX duration in seconds of the cursor movement. The cursor typically takes up to 1.5 seconds to move across the window.
:param solve_cloudflare: Solves all 3 types of the Cloudflare's Turnstile wait page before returning the response to you.
:param allow_webgl: Enabled by default. Disabling WebGL is not recommended as many WAFs now check if WebGL is enabled.
:param network_idle: Wait for the page until there are no network connections for at least 500 ms.
:param disable_ads: Disabled by default, this installs the `uBlock Origin` addon on the browser if enabled.
:param os_randomize: If enabled, Scrapling will randomize the OS fingerprints used. The default is Scrapling matching the fingerprints with the current OS.
:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
:param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
:param wait_selector: Wait for a specific CSS selector to be in a specific state.
:param geoip: Recommended to use with proxies; Automatically use IP's longitude, latitude, timezone, country, locale, and spoof the WebRTC IP address.
It will also calculate and spoof the browser's language based on the distribution of language speakers in the target region.
:param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.
:param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name.
:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
:param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.
:param additional_args: Additional arguments to be passed to Camoufox as additional settings, and it takes higher priority than Scrapling's settings.
"""
page = await StealthyFetcher.async_fetch(
url,
wait=wait,
proxy=proxy,
geoip=geoip,
addons=addons,
timeout=timeout,
cookies=cookies,
headless=headless,
humanize=humanize,
allow_webgl=allow_webgl,
disable_ads=disable_ads,
network_idle=network_idle,
block_images=block_images,
block_webrtc=block_webrtc,
os_randomize=os_randomize,
wait_selector=wait_selector,
google_search=google_search,
extra_headers=extra_headers,
solve_cloudflare=solve_cloudflare,
disable_resources=disable_resources,
wait_selector_state=wait_selector_state,
additional_args=additional_args,
)
return _ContentTranslator(
Convertor._extract_content(
page,
css_selector=css_selector,
extraction_type=extraction_type,
main_content_only=main_content_only,
),
page,
)
@staticmethod
@_server.tool()
async def bulk_stealthy_fetch(
urls: Tuple[str, ...],
extraction_type: extraction_types = "markdown",
css_selector: Optional[str] = None,
main_content_only: bool = True,
headless: bool = True, # noqa: F821
block_images: bool = False,
disable_resources: bool = False,
block_webrtc: bool = False,
allow_webgl: bool = True,
network_idle: bool = False,
humanize: bool | float = True,
solve_cloudflare: bool = False,
wait: int | float = 0,
timeout: int | float = 30000,
wait_selector: Optional[str] = None,
addons: Optional[List[str]] = None,
wait_selector_state: SelectorWaitStates = "attached",
cookies: Optional[List[Dict]] = None,
google_search: bool = True,
extra_headers: Optional[Dict[str, str]] = None,
proxy: Optional[str | Dict[str, str]] = None,
os_randomize: bool = False,
disable_ads: bool = False,
geoip: bool = False,
additional_args: Optional[Dict] = None,
) -> List[ResponseModel]:
"""Use Scrapling's version of the Camoufox browser to fetch a group of URLs at the same time, and for each page return a structured output of the result.
Note: This is best suitable for high protection levels. It's slower than the other tools.
Note: If the `css_selector` resolves to more than one element, all the elements will be returned.
:param urls: A tuple of the URLs to request.
:param extraction_type: The type of content to extract from the page. Defaults to "markdown". Options are:
- Markdown will convert the page content to Markdown format.
- HTML will return the raw HTML content of the page.
- Text will return the text content of the page.
:param css_selector: CSS selector to extract the content from the page. If main_content_only is True, then it will be executed on the main content of the page. Defaults to None.
:param main_content_only: Whether to extract only the main content of the page. Defaults to True. The main content here is the data inside the `<body>` tag.
:param headless: Run the browser in headless/hidden (default), or headful/visible mode.
:param block_images: Prevent the loading of images through Firefox preferences.
This can help save your proxy usage but be careful with this option as it makes some websites never finish loading.
:param disable_resources: Drop requests of unnecessary resources for a speed boost. It depends, but it made requests ~25% faster in my tests for some websites.
Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
This can help save your proxy usage but be careful with this option as it makes some websites never finish loading.
:param block_webrtc: Blocks WebRTC entirely.
:param cookies: Set cookies for the next request.
:param addons: List of Firefox addons to use. Must be paths to extracted addons.
:param humanize: Humanize the cursor movement. Takes either True or the MAX duration in seconds of the cursor movement. The cursor typically takes up to 1.5 seconds to move across the window.
:param solve_cloudflare: Solves all 3 types of the Cloudflare's Turnstile wait page before returning the response to you.
:param allow_webgl: Enabled by default. Disabling WebGL is not recommended as many WAFs now check if WebGL is enabled.
:param network_idle: Wait for the page until there are no network connections for at least 500 ms.
:param disable_ads: Disabled by default, this installs the `uBlock Origin` addon on the browser if enabled.
:param os_randomize: If enabled, Scrapling will randomize the OS fingerprints used. The default is Scrapling matching the fingerprints with the current OS.
:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
:param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
:param wait_selector: Wait for a specific CSS selector to be in a specific state.
:param geoip: Recommended to use with proxies; Automatically use IP's longitude, latitude, timezone, country, locale, and spoof the WebRTC IP address.
It will also calculate and spoof the browser's language based on the distribution of language speakers in the target region.
:param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.
:param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name.
:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
:param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.
:param additional_args: Additional arguments to be passed to Camoufox as additional settings, and it takes higher priority than Scrapling's settings.
"""
async with AsyncStealthySession(
wait=wait,
proxy=proxy,
geoip=geoip,
addons=addons,
timeout=timeout,
cookies=cookies,
headless=headless,
humanize=humanize,
max_pages=len(urls),
allow_webgl=allow_webgl,
disable_ads=disable_ads,
block_images=block_images,
block_webrtc=block_webrtc,
network_idle=network_idle,
os_randomize=os_randomize,
wait_selector=wait_selector,
google_search=google_search,
extra_headers=extra_headers,
solve_cloudflare=solve_cloudflare,
disable_resources=disable_resources,
wait_selector_state=wait_selector_state,
additional_args=additional_args,
) as session:
tasks = [session.fetch(url) for url in urls]
responses = await gather(*tasks)
return [
_ContentTranslator(
Convertor._extract_content(
page,
css_selector=css_selector,
extraction_type=extraction_type,
main_content_only=main_content_only,
),
page,
)
for page in responses
]
def serve(self):
"""Serve the MCP server."""
self._server.run(transport="stdio")
scrapling/core/custom_types.py
+183 -100
View File
@@ -1,159 +1,198 @@
import re
import typing
from collections.abc import Mapping
from types import MappingProxyType
from re import compile as re_compile, UNICODE, IGNORECASE
from orjson import dumps, loads
from w3lib.html import replace_entities as _replace_entities
from scrapling.core._types import (Dict, Iterable, List, Literal, Optional,
Pattern, SupportsIndex, TypeVar, Union)
from scrapling.core.utils import _is_iterable, flatten
from scrapling.core._types import (
cast,
Dict,
List,
Union,
overload,
TypeVar,
Literal,
Pattern,
Iterable,
Optional,
Generator,
SupportsIndex,
)
from scrapling.core.utils import _is_iterable, flatten, __CONSECUTIVE_SPACES_REGEX__
from scrapling.core._html_utils import _replace_entities
# Define type variable for AttributeHandler value type
_TextHandlerType = TypeVar('_TextHandlerType', bound='TextHandler')
_TextHandlerType = TypeVar("_TextHandlerType", bound="TextHandler")
__CLEANING_TABLE__ = str.maketrans("\t\r\n", " ")
class TextHandler(str):
"""Extends standard Python string by adding more functionality"""
__slots__ = ()
def __new__(cls, string):
return super().__new__(cls, str(string))
def __getitem__(self, key: Union[SupportsIndex, slice]) -> "TextHandler":
def __getitem__(
self, key: SupportsIndex | slice
) -> "TextHandler": # pragma: no cover
lst = super().__getitem__(key)
return typing.cast(_TextHandlerType, TextHandler(lst))
return cast(_TextHandlerType, TextHandler(lst))
def split(self, sep: str = None, maxsplit: SupportsIndex = -1) -> 'TextHandlers':
def split(
self, sep: str = None, maxsplit: SupportsIndex = -1
) -> "TextHandlers": # pragma: no cover
return TextHandlers(
typing.cast(List[_TextHandlerType], [TextHandler(s) for s in super().split(sep, maxsplit)])
cast(
List[_TextHandlerType],
[TextHandler(s) for s in super().split(sep, maxsplit)],
)
)
def strip(self, chars: str = None) -> Union[str, 'TextHandler']:
def strip(self, chars: str = None) -> Union[str, "TextHandler"]: # pragma: no cover
return TextHandler(super().strip(chars))
def lstrip(self, chars: str = None) -> Union[str, 'TextHandler']:
def lstrip(
self, chars: str = None
) -> Union[str, "TextHandler"]: # pragma: no cover
return TextHandler(super().lstrip(chars))
def rstrip(self, chars: str = None) -> Union[str, 'TextHandler']:
def rstrip(
self, chars: str = None
) -> Union[str, "TextHandler"]: # pragma: no cover
return TextHandler(super().rstrip(chars))
def capitalize(self) -> Union[str, 'TextHandler']:
def capitalize(self) -> Union[str, "TextHandler"]: # pragma: no cover
return TextHandler(super().capitalize())
def casefold(self) -> Union[str, 'TextHandler']:
def casefold(self) -> Union[str, "TextHandler"]: # pragma: no cover
return TextHandler(super().casefold())
def center(self, width: SupportsIndex, fillchar: str = ' ') -> Union[str, 'TextHandler']:
def center(
self, width: SupportsIndex, fillchar: str = " "
) -> Union[str, "TextHandler"]: # pragma: no cover
return TextHandler(super().center(width, fillchar))
def expandtabs(self, tabsize: SupportsIndex = 8) -> Union[str, 'TextHandler']:
def expandtabs(
self, tabsize: SupportsIndex = 8
) -> Union[str, "TextHandler"]: # pragma: no cover
return TextHandler(super().expandtabs(tabsize))
def format(self, *args: str, **kwargs: str) -> Union[str, 'TextHandler']:
def format(
self, *args: str, **kwargs: str
) -> Union[str, "TextHandler"]: # pragma: no cover
return TextHandler(super().format(*args, **kwargs))
def format_map(self, mapping) -> Union[str, 'TextHandler']:
def format_map(self, mapping) -> Union[str, "TextHandler"]: # pragma: no cover
return TextHandler(super().format_map(mapping))
def join(self, iterable: Iterable[str]) -> Union[str, 'TextHandler']:
def join(
self, iterable: Iterable[str]
) -> Union[str, "TextHandler"]: # pragma: no cover
return TextHandler(super().join(iterable))
def ljust(self, width: SupportsIndex, fillchar: str = ' ') -> Union[str, 'TextHandler']:
def ljust(
self, width: SupportsIndex, fillchar: str = " "
) -> Union[str, "TextHandler"]: # pragma: no cover
return TextHandler(super().ljust(width, fillchar))
def rjust(self, width: SupportsIndex, fillchar: str = ' ') -> Union[str, 'TextHandler']:
def rjust(
self, width: SupportsIndex, fillchar: str = " "
) -> Union[str, "TextHandler"]: # pragma: no cover
return TextHandler(super().rjust(width, fillchar))
def swapcase(self) -> Union[str, 'TextHandler']:
def swapcase(self) -> Union[str, "TextHandler"]: # pragma: no cover
return TextHandler(super().swapcase())
def title(self) -> Union[str, 'TextHandler']:
def title(self) -> Union[str, "TextHandler"]: # pragma: no cover
return TextHandler(super().title())
def translate(self, table) -> Union[str, 'TextHandler']:
def translate(self, table) -> Union[str, "TextHandler"]: # pragma: no cover
return TextHandler(super().translate(table))
def zfill(self, width: SupportsIndex) -> Union[str, 'TextHandler']:
def zfill(
self, width: SupportsIndex
) -> Union[str, "TextHandler"]: # pragma: no cover
return TextHandler(super().zfill(width))
def replace(self, old: str, new: str, count: SupportsIndex = -1) -> Union[str, 'TextHandler']:
def replace(
self, old: str, new: str, count: SupportsIndex = -1
) -> Union[str, "TextHandler"]:
return TextHandler(super().replace(old, new, count))
def upper(self) -> Union[str, 'TextHandler']:
def upper(self) -> Union[str, "TextHandler"]:
return TextHandler(super().upper())
def lower(self) -> Union[str, 'TextHandler']:
def lower(self) -> Union[str, "TextHandler"]:
return TextHandler(super().lower())
##############
def sort(self, reverse: bool = False) -> Union[str, 'TextHandler']:
def sort(self, reverse: bool = False) -> Union[str, "TextHandler"]:
"""Return a sorted version of the string"""
return self.__class__("".join(sorted(self, reverse=reverse)))
def clean(self) -> Union[str, 'TextHandler']:
def clean(self) -> Union[str, "TextHandler"]:
"""Return a new version of the string after removing all white spaces and consecutive spaces"""
data = re.sub(r'[\t|\r|\n]', '', self)
data = re.sub(' +', ' ', data)
return self.__class__(data.strip())
data = self.translate(__CLEANING_TABLE__)
return self.__class__(__CONSECUTIVE_SPACES_REGEX__.sub(" ", data).strip())
# For easy copy-paste from Scrapy/parsel code when needed :)
def get(self, default=None):
def get(self, default=None): # pragma: no cover
return self
def get_all(self):
def get_all(self): # pragma: no cover
return self
extract = get_all
extract_first = get
def json(self) -> Dict:
"""Return json response if the response is jsonable otherwise throw error"""
# Using str function as a workaround for orjson issue with subclasses of str
"""Return JSON response if the response is jsonable otherwise throw error"""
# Using str function as a workaround for orjson issue with subclasses of str.
# Check this out: https://github.com/ijl/orjson/issues/445
return loads(str(self))
@typing.overload
@overload
def re(
self,
regex: Union[str, Pattern[str]],
regex: str | Pattern,
check_match: Literal[True],
replace_entities: bool = True,
clean_match: bool = False,
case_sensitive: bool = True,
) -> bool:
...
) -> bool: ...
@typing.overload
@overload
def re(
self,
regex: Union[str, Pattern[str]],
regex: str | Pattern,
replace_entities: bool = True,
clean_match: bool = False,
case_sensitive: bool = True,
check_match: Literal[False] = False,
) -> "TextHandlers[TextHandler]":
...
) -> "TextHandlers[TextHandler]": ...
def re(
self, regex: Union[str, Pattern[str]], replace_entities: bool = True, clean_match: bool = False,
case_sensitive: bool = True, check_match: bool = False
) -> Union["TextHandlers[TextHandler]", bool]:
self,
regex: str | Pattern,
replace_entities: bool = True,
clean_match: bool = False,
case_sensitive: bool = True,
check_match: bool = False,
) -> Union["TextHandlers", bool]:
"""Apply the given regex to the current text and return a list of strings with the matches.
:param regex: Can be either a compiled regular expression or a string.
:param replace_entities: if enabled character entity references are replaced by their corresponding character
:param clean_match: if enabled, this will ignore all whitespaces and consecutive spaces while matching
:param case_sensitive: if disabled, function will set the regex to ignore letters case while compiling it
:param check_match: used to quickly check if this regex matches or not without any operations on the results
:param replace_entities: If enabled character entity references are replaced by their corresponding character
:param clean_match: If enabled, this will ignore all whitespaces and consecutive spaces while matching
:param case_sensitive: If disabled, function will set the regex to ignore the letters-case while compiling it
:param check_match: Used to quickly check if this regex matches or not without any operations on the results
"""
if isinstance(regex, str):
if case_sensitive:
regex = re.compile(regex, re.UNICODE)
regex = re_compile(regex, UNICODE)
else:
regex = re.compile(regex, flags=re.UNICODE | re.IGNORECASE)
regex = re_compile(regex, flags=UNICODE | IGNORECASE)
input_text = self.clean() if clean_match else self
results = regex.findall(input_text)
@@ -164,22 +203,42 @@ class TextHandler(str):
results = flatten(results)
if not replace_entities:
return TextHandlers(typing.cast(List[_TextHandlerType], [TextHandler(string) for string in results]))
return TextHandlers(
cast(
List[_TextHandlerType], [TextHandler(string) for string in results]
)
)
return TextHandlers(typing.cast(List[_TextHandlerType], [TextHandler(_replace_entities(s)) for s in results]))
return TextHandlers(
cast(
List[_TextHandlerType],
[TextHandler(_replace_entities(s)) for s in results],
)
)
def re_first(self, regex: Union[str, Pattern[str]], default=None, replace_entities: bool = True,
clean_match: bool = False, case_sensitive: bool = True) -> "TextHandler":
def re_first(
self,
regex: str | Pattern,
default=None,
replace_entities: bool = True,
clean_match: bool = False,
case_sensitive: bool = True,
) -> "TextHandler":
"""Apply the given regex to text and return the first match if found, otherwise return the default value.
:param regex: Can be either a compiled regular expression or a string.
:param default: The default value to be returned if there is no match
:param replace_entities: if enabled character entity references are replaced by their corresponding character
:param clean_match: if enabled, this will ignore all whitespaces and consecutive spaces while matching
:param case_sensitive: if disabled, function will set the regex to ignore letters case while compiling it
:param replace_entities: If enabled character entity references are replaced by their corresponding character
:param clean_match: If enabled, this will ignore all whitespaces and consecutive spaces while matching
:param case_sensitive: If disabled, function will set the regex to ignore the letters-case while compiling it
"""
result = self.re(regex, replace_entities, clean_match=clean_match, case_sensitive=case_sensitive)
result = self.re(
regex,
replace_entities,
clean_match=clean_match,
case_sensitive=case_sensitive,
)
return result[0] if result else default
@@ -187,48 +246,61 @@ class TextHandlers(List[TextHandler]):
"""
The :class:`TextHandlers` class is a subclass of the builtin ``List`` class, which provides a few additional methods.
"""
__slots__ = ()
@typing.overload
def __getitem__(self, pos: SupportsIndex) -> TextHandler:
@overload
def __getitem__(self, pos: SupportsIndex) -> TextHandler: # pragma: no cover
pass
@typing.overload
def __getitem__(self, pos: slice) -> "TextHandlers":
@overload
def __getitem__(self, pos: slice) -> "TextHandlers": # pragma: no cover
pass
def __getitem__(self, pos: Union[SupportsIndex, slice]) -> Union[TextHandler, "TextHandlers"]:
def __getitem__(
self, pos: SupportsIndex | slice
) -> Union[TextHandler, "TextHandlers"]:
lst = super().__getitem__(pos)
if isinstance(pos, slice):
lst = [TextHandler(s) for s in lst]
return TextHandlers(typing.cast(List[_TextHandlerType], lst))
return typing.cast(_TextHandlerType, TextHandler(lst))
return TextHandlers(cast(List[_TextHandlerType], lst))
return cast(_TextHandlerType, TextHandler(lst))
def re(self, regex: Union[str, Pattern[str]], replace_entities: bool = True, clean_match: bool = False,
case_sensitive: bool = True) -> 'TextHandlers[TextHandler]':
def re(
self,
regex: str | Pattern,
replace_entities: bool = True,
clean_match: bool = False,
case_sensitive: bool = True,
) -> "TextHandlers[TextHandler]":
"""Call the ``.re()`` method for each element in this list and return
their results flattened as TextHandlers.
:param regex: Can be either a compiled regular expression or a string.
:param replace_entities: if enabled character entity references are replaced by their corresponding character
:param replace_entities: If enabled character entity references are replaced by their corresponding character
:param clean_match: if enabled, this will ignore all whitespaces and consecutive spaces while matching
:param case_sensitive: if disabled, function will set the regex to ignore letters case while compiling it
:param case_sensitive: if disabled, the function will set the regex to ignore the letters-case while compiling it
"""
results = [
n.re(regex, replace_entities, clean_match, case_sensitive) for n in self
]
return TextHandlers(flatten(results))
def re_first(self, regex: Union[str, Pattern[str]], default=None, replace_entities: bool = True,
clean_match: bool = False, case_sensitive: bool = True) -> TextHandler:
def re_first(
self,
regex: str | Pattern,
default=None,
replace_entities: bool = True,
clean_match: bool = False,
case_sensitive: bool = True,
) -> TextHandler: # pragma: no cover
"""Call the ``.re_first()`` method for each element in this list and return
the first result or the default value otherwise.
:param regex: Can be either a compiled regular expression or a string.
:param default: The default value to be returned if there is no match
:param replace_entities: if enabled character entity references are replaced by their corresponding character
:param clean_match: if enabled, this will ignore all whitespaces and consecutive spaces while matching
:param case_sensitive: if disabled, function will set the regex to ignore letters case while compiling it
:param replace_entities: If enabled character entity references are replaced by their corresponding character
:param clean_match: If enabled, this will ignore all whitespaces and consecutive spaces while matching
:param case_sensitive: If disabled, function will set the regex to ignore the letters-case while compiling it
"""
for n in self:
for result in n.re(regex, replace_entities, clean_match, case_sensitive):
@@ -250,33 +322,44 @@ class TextHandlers(List[TextHandler]):
class AttributesHandler(Mapping[str, _TextHandlerType]):
"""A read-only mapping to use instead of the standard dictionary for the speed boost but at the same time I use it to add more functionalities.
If standard dictionary is needed, just convert this class to dictionary with `dict` function
"""A read-only mapping to use instead of the standard dictionary for the speed boost, but at the same time I use it to add more functionalities.
If the standard dictionary is needed, convert this class to a dictionary with the `dict` function
"""
__slots__ = ('_data',)
__slots__ = ("_data",)
def __init__(self, mapping=None, **kwargs):
mapping = {
key: TextHandler(value) if type(value) is str else value
for key, value in mapping.items()
} if mapping is not None else {}
mapping = (
{
key: TextHandler(value) if isinstance(value, str) else value
for key, value in mapping.items()
}
if mapping is not None
else {}
)
if kwargs:
mapping.update({
key: TextHandler(value) if type(value) is str else value
for key, value in kwargs.items()
})
mapping.update(
{
key: TextHandler(value) if isinstance(value, str) else value
for key, value in kwargs.items()
}
)
# Fastest read-only mapping type
self._data = MappingProxyType(mapping)
def get(self, key: str, default: Optional[str] = None) -> Union[_TextHandlerType, None]:
"""Acts like standard dictionary `.get()` method"""
def get(
self, key: str, default: Optional[str] = None
) -> Optional[_TextHandlerType]:
"""Acts like the standard dictionary `.get()` method"""
return self._data.get(key, default)
def search_values(self, keyword, partial=False):
"""Search current attributes by values and return dictionary of each matching item
:param keyword: The keyword to search for in the attributes values
def search_values(
self, keyword: str, partial: bool = False
) -> Generator["AttributesHandler", None, None]:
"""Search current attributes by values and return a dictionary of each matching item
:param keyword: The keyword to search for in the attribute values
:param partial: If True, the function will search if keyword in each value instead of perfect match
"""
for key, value in self._data.items():
scrapling/core/mixins.py
+27 -19
View File
@@ -1,32 +1,37 @@
class SelectorsGeneration:
"""Selectors generation functions
"""
Functions for generating selectors
Trying to generate selectors like Firefox or maybe cleaner ones!? Ehm
Inspiration: https://searchfox.org/mozilla-central/source/devtools/shared/inspector/css-logic.js#591"""
Inspiration: https://searchfox.org/mozilla-central/source/devtools/shared/inspector/css-logic.js#591
"""
def __general_selection(self, selection: str = 'css', full_path=False) -> str:
def __general_selection(
self, selection: str = "css", full_path: bool = False
) -> str:
"""Generate a selector for the current element.
:return: A string of the generated selector.
"""
selectorPath = []
target = self
css = selection.lower() == 'css'
css = selection.lower() == "css"
while target is not None:
if target.parent:
if target.attrib.get('id'):
if target.attrib.get("id"):
# id is enough
part = (
f'#{target.attrib["id"]}' if css
f"#{target.attrib['id']}"
if css
else f"[@id='{target.attrib['id']}']"
)
selectorPath.append(part)
if not full_path:
return (
" > ".join(reversed(selectorPath)) if css
else '//*' + "/".join(reversed(selectorPath))
" > ".join(reversed(selectorPath))
if css
else "//*" + "/".join(reversed(selectorPath))
)
else:
part = f'{target.tag}'
part = f"{target.tag}"
# We won't use classes anymore because I some websites share exact classes between elements
# classes = target.attrib.get('class', '').split()
# if classes and css:
@@ -41,23 +46,26 @@ class SelectorsGeneration:
if counter[target.tag] > 1:
part += (
f":nth-of-type({counter[target.tag]})" if css
f":nth-of-type({counter[target.tag]})"
if css
else f"[{counter[target.tag]}]"
)
selectorPath.append(part)
target = target.parent
if target is None or target.tag == 'html':
if target is None or target.tag == "html":
return (
" > ".join(reversed(selectorPath)) if css
else '//' + "/".join(reversed(selectorPath))
" > ".join(reversed(selectorPath))
if css
else "//" + "/".join(reversed(selectorPath))
)
else:
break
return (
" > ".join(reversed(selectorPath)) if css
else '//' + "/".join(reversed(selectorPath))
" > ".join(reversed(selectorPath))
if css
else "//" + "/".join(reversed(selectorPath))
)
@property
@@ -76,14 +84,14 @@ class SelectorsGeneration:
@property
def generate_xpath_selector(self) -> str:
"""Generate a XPath selector for the current element
"""Generate an XPath selector for the current element
:return: A string of the generated selector.
"""
return self.__general_selection('xpath')
return self.__general_selection("xpath")
@property
def generate_full_xpath_selector(self) -> str:
"""Generate a complete XPath selector for the current element
:return: A string of the generated selector.
"""
return self.__general_selection('xpath', full_path=True)
return self.__general_selection("xpath", full_path=True)
scrapling/core/shell.py
+647
View File
@@ -0,0 +1,647 @@
# -*- coding: utf-8 -*-
from re import sub as re_sub
from sys import stderr
from functools import wraps
from http import cookies as Cookie
from collections import namedtuple
from shlex import split as shlex_split
from tempfile import mkstemp as make_temp_file
from urllib.parse import urlparse, urlunparse, parse_qsl
from argparse import ArgumentParser, SUPPRESS
from webbrowser import open as open_in_browser
from logging import (
DEBUG,
INFO,
WARNING,
ERROR,
CRITICAL,
FATAL,
getLogger,
getLevelName,
)
from IPython.terminal.embed import InteractiveShellEmbed
from orjson import loads as json_loads, JSONDecodeError
from scrapling import __version__
from scrapling.core.custom_types import TextHandler
from scrapling.core.utils import log
from scrapling.parser import Selector, Selectors
from scrapling.core._types import (
List,
Optional,
Dict,
Tuple,
Any,
extraction_types,
Generator,
)
from scrapling.fetchers import (
Fetcher,
AsyncFetcher,
DynamicFetcher,
StealthyFetcher,
Response,
)
_known_logging_levels = {
"debug": DEBUG,
"info": INFO,
"warning": WARNING,
"error": ERROR,
"critical": CRITICAL,
"fatal": FATAL,
}
# Define the structure for parsed context - Simplified for Fetcher args
Request = namedtuple(
"Request",
[
"method",
"url",
"params",
"data", # Can be str, bytes, or dict (for urlencoded)
"json_data", # Python object (dict/list) for JSON payload
"headers",
"cookies",
"proxy",
"follow_redirects", # Added for -L flag
],
)
def _CookieParser(cookie_string):
# Errors will be handled on call so the log can be specified
cookie_parser = Cookie.SimpleCookie()
cookie_parser.load(cookie_string)
for key, morsel in cookie_parser.items():
yield key, morsel.value
def _ParseHeaders(
header_lines: List[str], parse_cookies: bool = True
) -> Tuple[Dict[str, str], Dict[str, str]]:
"""Parses headers into separate header and cookie dictionaries."""
header_dict = dict()
cookie_dict = dict()
for header_line in header_lines:
if ":" not in header_line:
if header_line.endswith(";"):
header_key = header_line[:-1].strip()
header_value = ""
header_dict[header_key] = header_value
else:
raise ValueError(
f"Could not parse header without colon: '{header_line}'."
)
else:
header_key, header_value = header_line.split(":", 1)
header_key = header_key.strip()
header_value = header_value.strip()
if parse_cookies:
if header_key.lower() == "cookie":
try:
cookie_dict = {
key: value for key, value in _CookieParser(header_value)
}
except Exception as e: # pragma: no cover
raise ValueError(
f"Could not parse cookie string from header '{header_value}': {e}"
)
else:
header_dict[header_key] = header_value
else:
header_dict[header_key] = header_value
return header_dict, cookie_dict
# Suppress exit on error to handle parsing errors gracefully
class NoExitArgumentParser(ArgumentParser): # pragma: no cover
def error(self, message):
log.error(f"Curl arguments parsing error: {message}")
raise ValueError(f"Curl arguments parsing error: {message}")
def exit(self, status=0, message=None):
if message:
log.error(f"Scrapling shell exited with status {status}: {message}")
self._print_message(message, stderr)
raise ValueError(
f"Scrapling shell exited with status {status}: {message or 'Unknown reason'}"
)
class CurlParser:
"""Builds the argument parser for relevant curl flags from DevTools."""
def __init__(self):
# We will use argparse parser to parse the curl command directly instead of regex
# We will focus more on flags that will show up on curl commands copied from DevTools's network tab
_parser = NoExitArgumentParser(add_help=False) # Disable default help
# Basic curl arguments
_parser.add_argument("curl_command_placeholder", nargs="?", help=SUPPRESS)
_parser.add_argument("url")
_parser.add_argument("-X", "--request", dest="method", default=None)
_parser.add_argument("-H", "--header", action="append", default=[])
_parser.add_argument(
"-A", "--user-agent", help="Will be parsed from -H if present"
) # Note: DevTools usually includes this in -H
# Data arguments (prioritizing types common from DevTools)
_parser.add_argument("-d", "--data", default=None)
_parser.add_argument(
"--data-raw", default=None
) # Often used by browsers for JSON body
_parser.add_argument("--data-binary", default=None)
# Keep urlencode for completeness, though less common from browser copy/paste
_parser.add_argument("--data-urlencode", action="append", default=[])
_parser.add_argument(
"-G", "--get", action="store_true"
) # Use GET and put data in URL
_parser.add_argument(
"-b",
"--cookie",
default=None,
help="Send cookies from string/file (string format used by DevTools)",
)
# Proxy
_parser.add_argument("-x", "--proxy", default=None)
_parser.add_argument("-U", "--proxy-user", default=None) # Basic proxy auth
# Connection/Security
_parser.add_argument("-k", "--insecure", action="store_true")
_parser.add_argument(
"--compressed", action="store_true"
) # Very common from browsers
# Other flags often included but may not map directly to request args
_parser.add_argument("-i", "--include", action="store_true")
_parser.add_argument("-s", "--silent", action="store_true")
_parser.add_argument("-v", "--verbose", action="store_true")
self.parser: NoExitArgumentParser = _parser
self._supported_methods = ("get", "post", "put", "delete")
# --- Main Parsing Logic ---
def parse(self, curl_command: str) -> Optional[Request]:
"""Parses the curl command string into a structured context for Fetcher."""
clean_command = curl_command.strip().lstrip("curl").strip().replace("\\\n", " ")
try:
tokens = shlex_split(
clean_command
) # Split the string using shell-like syntax
except ValueError as e: # pragma: no cover
log.error(f"Could not split command line: {e}")
return None
try:
parsed_args, unknown = self.parser.parse_known_args(tokens)
if unknown:
raise AttributeError(f"Unknown/Unsupported curl arguments: {unknown}")
except ValueError: # pragma: no cover
return None
except AttributeError:
raise
except Exception as e: # pragma: no cover
log.error(
f"An unexpected error occurred during curl arguments parsing: {e}"
)
return None
# --- Determine Method ---
method = "get" # Default
if parsed_args.get: # `-G` forces GET
method = "get"
elif parsed_args.method:
method = parsed_args.method.strip().lower()
# Infer POST if data is present (unless overridden by -X or -G)
elif any(
[
parsed_args.data,
parsed_args.data_raw,
parsed_args.data_binary,
parsed_args.data_urlencode,
]
):
method = "post"
headers, cookies = _ParseHeaders(parsed_args.header)
if parsed_args.cookie:
# We are focusing on the string format from DevTools.
try:
for key, value in _CookieParser(parsed_args.cookie):
# Update the cookie dict, potentially overwriting cookies with the same name from -H 'cookie:'
cookies[key] = value
log.debug(f"Parsed cookies from -b argument: {list(cookies.keys())}")
except Exception as e: # pragma: no cover
log.error(
f"Could not parse cookie string from -b '{parsed_args.cookie}': {e}"
)
# --- Process Data Payload ---
params = dict()
data_payload: Optional[str | bytes | Dict] = None
json_payload: Optional[Any] = None
# DevTools often uses --data-raw for JSON bodies
# Precedence: --data-binary > --data-raw / -d > --data-urlencode
if parsed_args.data_binary is not None: # pragma: no cover
try:
data_payload = parsed_args.data_binary.encode("utf-8")
log.debug("Using data from --data-binary as bytes.")
except Exception as e:
log.warning(
f"Could not encode binary data '{parsed_args.data_binary}' as bytes: {e}. Using raw string."
)
data_payload = parsed_args.data_binary # Fallback to string
elif parsed_args.data_raw is not None:
data_payload = parsed_args.data_raw
elif parsed_args.data is not None:
data_payload = parsed_args.data
elif parsed_args.data_urlencode: # pragma: no cover
# Combine and parse urlencoded data
combined_data = "&".join(parsed_args.data_urlencode)
try:
data_payload = dict(parse_qsl(combined_data, keep_blank_values=True))
except Exception as e:
log.warning(
f"Could not parse urlencoded data '{combined_data}': {e}. Treating as raw string."
)
data_payload = combined_data
# Check if raw data looks like JSON, prefer 'json' param if so
if isinstance(data_payload, str):
try:
maybe_json = json_loads(data_payload)
if isinstance(maybe_json, (dict, list)):
json_payload = maybe_json
data_payload = None
except JSONDecodeError:
pass # Not JSON, keep it in data_payload
# Handle `-G`: Move data to params if the method is GET
if method == "get" and data_payload: # pragma: no cover
if isinstance(data_payload, dict): # From --data-urlencode likely
params.update(data_payload)
elif isinstance(data_payload, str):
try:
params.update(dict(parse_qsl(data_payload, keep_blank_values=True)))
except ValueError:
log.warning(
f"Could not parse data '{data_payload}' into GET parameters for -G."
)
if params:
data_payload = None # Clear data as it's moved to params
json_payload = None # Should not have JSON body with -G
# --- Process Proxy ---
proxies: Optional[Dict[str, str]] = None
if parsed_args.proxy:
proxy_url = (
f"http://{parsed_args.proxy}"
if "://" not in parsed_args.proxy
else parsed_args.proxy
)
if parsed_args.proxy_user:
user_pass = parsed_args.proxy_user
parts = urlparse(proxy_url)
netloc_parts = parts.netloc.split("@")
netloc = (
f"{user_pass}@{netloc_parts[-1]}"
if len(netloc_parts) > 1
else f"{user_pass}@{parts.netloc}"
)
proxy_url = urlunparse(
(
parts.scheme,
netloc,
parts.path,
parts.params,
parts.query,
parts.fragment,
)
)
# Standard proxy dict format
proxies = {"http": proxy_url, "https": proxy_url}
log.debug(f"Using proxy configuration: {proxies}")
# --- Final Context ---
return Request(
method=method,
url=parsed_args.url,
params=params,
data=data_payload,
json_data=json_payload,
headers=headers,
cookies=cookies,
proxy=proxies,
follow_redirects=True, # Scrapling default is True
)
def convert2fetcher(self, curl_command: Request | str) -> Optional[Response]:
if isinstance(curl_command, (Request, str)):
request = (
self.parse(curl_command)
if isinstance(curl_command, str)
else curl_command
)
# Ensure request parsing was successful before proceeding
if request is None: # pragma: no cover
log.error("Failed to parse curl command, cannot convert to fetcher.")
return None
request_args = request._asdict()
method = request_args.pop("method").strip().lower()
if method in self._supported_methods:
request_args["json"] = request_args.pop("json_data")
# Ensure data/json are removed for non-POST/PUT methods
if method not in ("post", "put"):
_ = request_args.pop("data", None)
_ = request_args.pop("json", None)
try:
return getattr(Fetcher, method)(**request_args)
except Exception as e: # pragma: no cover
log.error(f"Error calling Fetcher.{method}: {e}")
return None
else: # pragma: no cover
log.error(
f'Request method "{method}" isn\'t supported by Scrapling yet'
)
return None
else: # pragma: no cover
log.error("Input must be a valid curl command string or a Request object.")
return None
def show_page_in_browser(page: Selector): # pragma: no cover
if not page or not isinstance(page, Selector):
log.error("Input must be of type `Selector`")
return
try:
fd, fname = make_temp_file(prefix="scrapling_view_", suffix=".html")
with open(fd, "w", encoding="utf-8") as f:
f.write(page.body)
open_in_browser(f"file://{fname}")
except IOError as e:
log.error(f"Failed to write temporary file for viewing: {e}")
except Exception as e:
log.error(f"An unexpected error occurred while viewing the page: {e}")
class CustomShell:
"""A custom IPython shell with minimal dependencies"""
def __init__(self, code, log_level="debug"):
self.code = code
self.page = None
self.pages = Selectors([])
self._curl_parser = CurlParser()
log_level = log_level.strip().lower()
if _known_logging_levels.get(log_level):
self.log_level = _known_logging_levels[log_level]
else: # pragma: no cover
log.warning(f'Unknown log level "{log_level}", defaulting to "DEBUG"')
self.log_level = DEBUG
self.shell = None
# Initialize your application components
self.init_components()
def init_components(self):
"""Initialize application components"""
# This is where you'd set up your application-specific objects
if self.log_level:
getLogger("scrapling").setLevel(self.log_level)
settings = Fetcher.display_config()
settings.pop("storage", None)
settings.pop("storage_args", None)
log.info(f"Scrapling {__version__} shell started")
log.info(f"Logging level is set to '{getLevelName(self.log_level)}'")
log.info(f"Fetchers' parsing settings: {settings}")
@staticmethod
def banner():
"""Create a custom banner for the shell"""
return f"""
-> Available Scrapling objects:
- Fetcher/AsyncFetcher
- DynamicFetcher
- StealthyFetcher
- Selector
-> Useful shortcuts:
- {"get":<30} Shortcut for `Fetcher.get`
- {"post":<30} Shortcut for `Fetcher.post`
- {"put":<30} Shortcut for `Fetcher.put`
- {"delete":<30} Shortcut for `Fetcher.delete`
- {"fetch":<30} Shortcut for `DynamicFetcher.fetch`
- {"stealthy_fetch":<30} Shortcut for `StealthyFetcher.fetch`
-> Useful commands
- {"page / response":<30} The response object of the last page you fetched
- {"pages":<30} Selectors object of the last 5 response objects you fetched
- {"uncurl('curl_command')":<30} Convert curl command to a Request object. (Optimized to handle curl commands copied from DevTools network tab.)
- {"curl2fetcher('curl_command')":<30} Convert curl command and make the request with Fetcher. (Optimized to handle curl commands copied from DevTools network tab.)
- {"view(page)":<30} View page in a browser
- {"help()":<30} Show this help message (Shell help)
Type 'exit' or press Ctrl+D to exit.
"""
def update_page(self, result): # pragma: no cover
"""Update the current page and add to pages history"""
self.page = result
if isinstance(result, (Response, Selector)):
self.pages.append(result)
if len(self.pages) > 5:
self.pages.pop(0) # Remove oldest item
# Update in IPython namespace too
if self.shell:
self.shell.user_ns["page"] = self.page
self.shell.user_ns["response"] = self.page
self.shell.user_ns["pages"] = self.pages
return result
def create_wrapper(self, func):
"""Create a wrapper that preserves function signature but updates page"""
@wraps(func)
def wrapper(*args, **kwargs):
result = func(*args, **kwargs)
return self.update_page(result)
return wrapper
def get_namespace(self):
"""Create a namespace with application-specific objects"""
# Create wrapped versions of fetch functions
get = self.create_wrapper(Fetcher.get)
post = self.create_wrapper(Fetcher.post)
put = self.create_wrapper(Fetcher.put)
delete = self.create_wrapper(Fetcher.delete)
dynamic_fetch = self.create_wrapper(DynamicFetcher.fetch)
stealthy_fetch = self.create_wrapper(StealthyFetcher.fetch)
curl2fetcher = self.create_wrapper(self._curl_parser.convert2fetcher)
# Create the namespace dictionary
return {
"get": get,
"post": post,
"put": put,
"delete": delete,
"Fetcher": Fetcher,
"AsyncFetcher": AsyncFetcher,
"fetch": dynamic_fetch,
"DynamicFetcher": DynamicFetcher,
"stealthy_fetch": stealthy_fetch,
"StealthyFetcher": StealthyFetcher,
"Selector": Selector,
"page": self.page,
"response": self.page,
"pages": self.pages,
"view": show_page_in_browser,
"uncurl": self._curl_parser.parse,
"curl2fetcher": curl2fetcher,
"help": self.show_help,
}
def show_help(self): # pragma: no cover
"""Show help information"""
print(self.banner())
def start(self): # pragma: no cover
"""Start the interactive shell"""
# Get our namespace with application objects
namespace = self.get_namespace()
ipython_shell = InteractiveShellEmbed(
banner1=self.banner(),
banner2="",
enable_tip=False,
exit_msg="Bye Bye",
user_ns=namespace,
)
self.shell = ipython_shell
# If a command was provided, execute it and exit
if self.code:
log.info(f"Executing provided code: {self.code}")
try:
ipython_shell.run_cell(self.code, store_history=False)
except Exception as e:
log.error(f"Error executing initial code: {e}")
return
ipython_shell()
class Convertor:
"""Utils for the extract shell command"""
_extension_map: Dict[str, extraction_types] = {
"md": "markdown",
"html": "html",
"txt": "text",
}
@classmethod
def _convert_to_markdown(cls, body: TextHandler) -> str:
"""Convert HTML content to Markdown"""
from markdownify import markdownify
return markdownify(body)
@classmethod
def _extract_content(
cls,
page: Selector,
extraction_type: extraction_types = "markdown",
css_selector: Optional[str] = None,
main_content_only: bool = False,
) -> Generator[str, None, None]:
"""Extract the content of a Selector"""
if not page or not isinstance(page, Selector): # pragma: no cover
raise TypeError("Input must be of type `Selector`")
elif not extraction_type or extraction_type not in cls._extension_map.values():
raise ValueError(f"Unknown extraction type: {extraction_type}")
else:
if main_content_only:
page = page.css_first("body") or page
pages = [page] if not css_selector else page.css(css_selector)
for page in pages:
match extraction_type:
case "markdown":
yield cls._convert_to_markdown(page.body)
case "html":
yield page.body
case "text":
txt_content = page.get_all_text(strip=True)
for s in (
"\n",
"\r",
"\t",
" ",
):
# Remove consecutive white-spaces
txt_content = re_sub(f"[{s}]+", s, txt_content)
yield txt_content
yield ""
@classmethod
def write_content_to_file(
cls, page: Selector, filename: str, css_selector: Optional[str] = None
) -> None:
"""Write a Selector's content to a file"""
if not page or not isinstance(page, Selector): # pragma: no cover
raise TypeError("Input must be of type `Selector`")
elif not filename or not isinstance(filename, str) or not filename.strip():
raise ValueError("Filename must be provided")
elif not filename.endswith((".md", ".html", ".txt")):
raise ValueError(
"Unknown file type: filename must end with '.md', '.html', or '.txt'"
)
else:
with open(filename, "w", encoding="utf-8") as f:
extension = filename.split(".")[-1]
f.write(
"".join(
cls._extract_content(
page,
cls._extension_map[extension],
css_selector=css_selector,
)
)
)
scrapling/core/storage_adaptors.py → scrapling/core/storage.py
+41 -33
View File
@@ -1,44 +1,49 @@
import sqlite3
import threading
from abc import ABC, abstractmethod
from hashlib import sha256
from threading import RLock
from functools import lru_cache
from abc import ABC, abstractmethod
from sqlite3 import connect as db_connect
import orjson
from lxml import html
from orjson import dumps, loads
from lxml.html import HtmlElement
from tldextract import extract as tld
from scrapling.core._types import Dict, Optional, Union
from scrapling.core.utils import _StorageTools, log, lru_cache
from scrapling.core.utils import _StorageTools, log
from scrapling.core._types import Dict, Optional, Any
class StorageSystemMixin(ABC):
class StorageSystemMixin(ABC): # pragma: no cover
# If you want to make your own storage system, you have to inherit from this
def __init__(self, url: Union[str, None] = None):
def __init__(self, url: Optional[str] = None):
"""
:param url: URL of the website we are working on to separate it from other websites data
"""
self.url = url
@lru_cache(64, typed=True)
def _get_base_url(self, default_value: str = 'default') -> str:
if not self.url or type(self.url) is not str:
def _get_base_url(self, default_value: str = "default") -> str:
if not self.url or not isinstance(self.url, str):
return default_value
try:
extracted = tld(self.url)
return extracted.registered_domain or extracted.domain or default_value
return (
extracted.top_domain_under_public_suffix
or extracted.domain
or default_value
)
except AttributeError:
return default_value
@abstractmethod
def save(self, element: html.HtmlElement, identifier: str) -> None:
def save(self, element: HtmlElement, identifier: str) -> None:
"""Saves the element's unique properties to the storage for retrieval and relocation later
:param element: The element itself that we want to save to storage.
:param element: The element itself which we want to save to storage.
:param identifier: This is the identifier that will be used to retrieve the element later from the storage. See
the docs for more info.
"""
raise NotImplementedError('Storage system must implement `save` method')
raise NotImplementedError("Storage system must implement `save` method")
@abstractmethod
def retrieve(self, identifier: str) -> Optional[Dict]:
@@ -48,7 +53,7 @@ class StorageSystemMixin(ABC):
the docs for more info.
:return: A dictionary of the unique properties
"""
raise NotImplementedError('Storage system must implement `save` method')
raise NotImplementedError("Storage system must implement `save` method")
@staticmethod
@lru_cache(128, typed=True)
@@ -57,7 +62,7 @@ class StorageSystemMixin(ABC):
identifier = identifier.lower().strip()
if isinstance(identifier, str):
# Hash functions have to take bytes
identifier = identifier.encode('utf-8')
identifier = identifier.encode("utf-8")
hash_value = sha256(identifier).hexdigest()
return f"{hash_value}_{len(identifier)}" # Length to reduce collision chance
@@ -66,21 +71,21 @@ class StorageSystemMixin(ABC):
@lru_cache(1, typed=True)
class SQLiteStorageSystem(StorageSystemMixin):
"""The recommended system to use, it's race condition safe and thread safe.
Mainly built so the library can run in threaded frameworks like scrapy or threaded tools
> It's optimized for threaded applications but running it without threads shouldn't make it slow."""
def __init__(self, storage_file: str, url: Union[str, None] = None):
Mainly built, so the library can run in threaded frameworks like scrapy or threaded tools
> It's optimized for threaded applications, but running it without threads shouldn't make it slow."""
def __init__(self, storage_file: str, url: Optional[str] = None):
"""
:param storage_file: File to be used to store elements
:param storage_file: File to be used to store elements' data.
:param url: URL of the website we are working on to separate it from other websites data
"""
super().__init__(url)
self.storage_file = storage_file
# We use a threading.Lock to ensure thread-safety instead of relying on thread-local storage.
self.lock = threading.Lock()
# >SQLite default mode in earlier version is 1 not 2 (1=thread-safe 2=serialized)
self.lock = RLock() # Better than Lock for reentrancy
# >SQLite default mode in the earlier version is 1 not 2 (1=thread-safe 2=serialized)
# `check_same_thread=False` to allow it to be used across different threads.
self.connection = sqlite3.connect(self.storage_file, check_same_thread=False)
self.connection = db_connect(self.storage_file, check_same_thread=False)
# WAL (Write-Ahead Logging) allows for better concurrency.
self.connection.execute("PRAGMA journal_mode=WAL")
self.cursor = self.connection.cursor()
@@ -101,24 +106,27 @@ class SQLiteStorageSystem(StorageSystemMixin):
""")
self.connection.commit()
def save(self, element: html.HtmlElement, identifier: str):
def save(self, element: HtmlElement, identifier: str) -> None:
"""Saves the elements unique properties to the storage for retrieval and relocation later
:param element: The element itself that we want to save to storage.
:param element: The element itself which we want to save to storage.
:param identifier: This is the identifier that will be used to retrieve the element later from the storage. See
the docs for more info.
"""
url = self._get_base_url()
element_data = _StorageTools.element_to_dict(element)
with self.lock:
self.cursor.execute("""
self.cursor.execute(
"""
INSERT OR REPLACE INTO storage (url, identifier, element_data)
VALUES (?, ?, ?)
""", (url, identifier, orjson.dumps(element_data)))
""",
(url, identifier, dumps(element_data)),
)
self.cursor.fetchall()
self.connection.commit()
def retrieve(self, identifier: str) -> Optional[Dict]:
def retrieve(self, identifier: str) -> Optional[Dict[str, Any]]:
"""Using the identifier, we search the storage and return the unique properties of the element
:param identifier: This is the identifier that will be used to retrieve the element from the storage. See
@@ -129,15 +137,15 @@ class SQLiteStorageSystem(StorageSystemMixin):
with self.lock:
self.cursor.execute(
"SELECT element_data FROM storage WHERE url = ? AND identifier = ?",
(url, identifier)
(url, identifier),
)
result = self.cursor.fetchone()
if result:
return orjson.loads(result[0])
return loads(result[0])
return None
def close(self):
"""Close all connections, will be useful when with some things like scrapy Spider.closed() function/signal"""
"""Close all connections. It will be useful when with some things like scrapy Spider.closed() function/signal"""
with self.lock:
self.connection.commit()
self.cursor.close()
scrapling/core/translator.py
+20 -26
View File
@@ -1,30 +1,24 @@
"""
Most of this file is adapted version of the translator of parsel library with some modifications simply for 1 important reason...
Most of this file is an adapted version of the parsel library's translator with some modifications simply for 1 important reason...
To add pseudo-elements ``::text`` and ``::attr(ATTR_NAME)`` so we match Parsel/Scrapy selectors format which will be important in future releases but most importantly...
To add pseudo-elements ``::text`` and ``::attr(ATTR_NAME)`` so we match the Parsel/Scrapy selectors format which will be important in future releases but most importantly...
So you don't have to learn a new selectors/api method like what bs4 done with soupsieve :)
if you want to learn about this, head to https://cssselect.readthedocs.io/en/latest/#cssselect.FunctionalPseudoElement
If you want to learn about this, head to https://cssselect.readthedocs.io/en/latest/#cssselect.FunctionalPseudoElement
"""
import re
from functools import lru_cache
from cssselect import HTMLTranslator as OriginalHTMLTranslator
from cssselect.parser import Element, FunctionalPseudoElement, PseudoElement
from cssselect.xpath import ExpressionError
from cssselect.xpath import XPathExpr as OriginalXPathExpr
from w3lib.html import HTML5_WHITESPACE
from scrapling.core._types import Any, Optional, Protocol, Self
from scrapling.core.utils import lru_cache
regex = f"[{HTML5_WHITESPACE}]+"
replace_html5_whitespaces = re.compile(regex).sub
class XPathExpr(OriginalXPathExpr):
textnode: bool = False
attribute: Optional[str] = None
@@ -34,7 +28,7 @@ class XPathExpr(OriginalXPathExpr):
xpath: OriginalXPathExpr,
textnode: bool = False,
attribute: Optional[str] = None,
) -> "Self":
) -> Self:
x = cls(path=xpath.path, element=xpath.element, condition=xpath.condition)
x.textnode = textnode
x.attribute = attribute
@@ -43,29 +37,29 @@ class XPathExpr(OriginalXPathExpr):
def __str__(self) -> str:
path = super().__str__()
if self.textnode:
if path == "*":
if path == "*": # pragma: no cover
path = "text()"
elif path.endswith("::*/*"):
elif path.endswith("::*/*"): # pragma: no cover
path = path[:-3] + "text()"
else:
path += "/text()"
if self.attribute is not None:
if path.endswith("::*/*"):
if path.endswith("::*/*"): # pragma: no cover
path = path[:-2]
path += f"/@{self.attribute}"
return path
def join(
self: "Self",
self: Self,
combiner: str,
other: OriginalXPathExpr,
*args: Any,
**kwargs: Any,
) -> "Self":
) -> Self:
if not isinstance(other, XPathExpr):
raise ValueError(
raise ValueError( # pragma: no cover
f"Expressions of type {__name__}.XPathExpr can ony join expressions"
f" of the same type (or its descendants), got {type(other)}"
)
@@ -77,10 +71,10 @@ class XPathExpr(OriginalXPathExpr):
# e.g. cssselect.GenericTranslator, cssselect.HTMLTranslator
class TranslatorProtocol(Protocol):
def xpath_element(self, selector: Element) -> OriginalXPathExpr:
def xpath_element(self, selector: Element) -> OriginalXPathExpr: # pragma: no cover
pass
def css_to_xpath(self, css: str, prefix: str = ...) -> str:
def css_to_xpath(self, css: str, prefix: str = ...) -> str: # pragma: no cover
pass
@@ -91,7 +85,7 @@ class TranslatorMixin:
"""
def xpath_element(self: TranslatorProtocol, selector: Element) -> XPathExpr:
# https://github.com/python/mypy/issues/12344
# https://github.com/python/mypy/issues/14757
xpath = super().xpath_element(selector) # type: ignore[safe-super]
return XPathExpr.from_xpath(xpath)
@@ -99,12 +93,12 @@ class TranslatorMixin:
self, xpath: OriginalXPathExpr, pseudo_element: PseudoElement
) -> OriginalXPathExpr:
"""
Dispatch method that transforms XPath to support pseudo-elements.
Dispatch method that transforms XPath to support the pseudo-element.
"""
if isinstance(pseudo_element, FunctionalPseudoElement):
method_name = f"xpath_{pseudo_element.name.replace('-', '_')}_functional_pseudo_element"
method = getattr(self, method_name, None)
if not method:
if not method: # pragma: no cover
raise ExpressionError(
f"The functional pseudo-element ::{pseudo_element.name}() is unknown"
)
@@ -114,7 +108,7 @@ class TranslatorMixin:
f"xpath_{pseudo_element.replace('-', '_')}_simple_pseudo_element"
)
method = getattr(self, method_name, None)
if not method:
if not method: # pragma: no cover
raise ExpressionError(
f"The pseudo-element ::{pseudo_element} is unknown"
)
@@ -123,10 +117,10 @@ class TranslatorMixin:
@staticmethod
def xpath_attr_functional_pseudo_element(
xpath: OriginalXPathExpr, function: FunctionalPseudoElement
xpath: OriginalXPathExpr, function: FunctionalPseudoElement
) -> XPathExpr:
"""Support selecting attribute values using ::attr() pseudo-element"""
if function.argument_types() not in (["STRING"], ["IDENT"]):
if function.argument_types() not in (["STRING"], ["IDENT"]): # pragma: no cover
raise ExpressionError(
f"Expected a single string or ident for ::attr(), got {function.arguments!r}"
)
@@ -144,4 +138,4 @@ class HTMLTranslator(TranslatorMixin, OriginalHTMLTranslator):
return super().css_to_xpath(css, prefix)
translator_instance = HTMLTranslator()
translator = HTMLTranslator()
scrapling/core/utils.py
+49 -54
View File
@@ -1,17 +1,18 @@
import logging
import re
from itertools import chain
from re import compile as re_compile
import orjson
from lxml import html
from scrapling.core._types import Any, Dict, Iterable, Union
from scrapling.core._types import Any, Dict, Iterable, List
# Using cache on top of a class is brilliant way to achieve Singleton design pattern without much code
# functools.cache is available on Python 3.9+ only so let's keep lru_cache
# Using cache on top of a class is a brilliant way to achieve a Singleton design pattern without much code
from functools import lru_cache # isort:skip
html_forbidden = {html.HtmlComment, }
html_forbidden = (html.HtmlComment,)
__CLEANING_TABLE__ = str.maketrans({"\t": " ", "\n": None, "\r": None})
__CONSECUTIVE_SPACES_REGEX__ = re_compile(r" +")
@lru_cache(1, typed=True)
@@ -20,12 +21,11 @@ def setup_logger():
:returns: logging.Logger: Configured logger instance
"""
logger = logging.getLogger('scrapling')
logger = logging.getLogger("scrapling")
logger.setLevel(logging.INFO)
formatter = logging.Formatter(
fmt="[%(asctime)s] %(levelname)s: %(message)s",
datefmt="%Y-%m-%d %H:%M:%S"
fmt="[%(asctime)s] %(levelname)s: %(message)s", datefmt="%Y-%m-%d %H:%M:%S"
)
console_handler = logging.StreamHandler()
@@ -41,24 +41,19 @@ def setup_logger():
log = setup_logger()
def is_jsonable(content: Union[bytes, str]) -> bool:
if type(content) is bytes:
content = content.decode()
try:
_ = orjson.loads(content)
return True
except orjson.JSONDecodeError:
return False
def flatten(lst: Iterable):
def flatten(lst: Iterable[Any]) -> List[Any]:
return list(chain.from_iterable(lst))
def _is_iterable(s: Any):
def _is_iterable(obj: Any) -> bool:
# This will be used only in regex functions to make sure it's iterable but not string/bytes
return isinstance(s, (list, tuple,))
return isinstance(
obj,
(
list,
tuple,
),
)
class _StorageTools:
@@ -66,31 +61,43 @@ class _StorageTools:
def __clean_attributes(element: html.HtmlElement, forbidden: tuple = ()) -> Dict:
if not element.attrib:
return {}
return {k: v.strip() for k, v in element.attrib.items() if v and v.strip() and k not in forbidden}
return {
k: v.strip()
for k, v in element.attrib.items()
if v and v.strip() and k not in forbidden
}
@classmethod
def element_to_dict(cls, element: html.HtmlElement) -> Dict:
parent = element.getparent()
result = {
'tag': str(element.tag),
'attributes': cls.__clean_attributes(element),
'text': element.text.strip() if element.text else None,
'path': cls._get_element_path(element)
"tag": str(element.tag),
"attributes": cls.__clean_attributes(element),
"text": element.text.strip() if element.text else None,
"path": cls._get_element_path(element),
}
if parent is not None:
result.update({
'parent_name': parent.tag,
'parent_attribs': dict(parent.attrib),
'parent_text': parent.text.strip() if parent.text else None
})
result.update(
{
"parent_name": parent.tag,
"parent_attribs": dict(parent.attrib),
"parent_text": parent.text.strip() if parent.text else None,
}
)
siblings = [child.tag for child in parent.iterchildren() if child != element]
siblings = [
child.tag for child in parent.iterchildren() if child != element
]
if siblings:
result.update({'siblings': tuple(siblings)})
result.update({"siblings": tuple(siblings)})
children = [child.tag for child in element.iterchildren() if type(child) not in html_forbidden]
children = [
child.tag
for child in element.iterchildren()
if not isinstance(child, html_forbidden)
]
if children:
result.update({'children': tuple(children)})
result.update({"children": tuple(children)})
return result
@@ -98,25 +105,13 @@ class _StorageTools:
def _get_element_path(cls, element: html.HtmlElement):
parent = element.getparent()
return tuple(
(element.tag,) if parent is None else (
cls._get_element_path(parent) + (element.tag,)
)
(element.tag,)
if parent is None
else (cls._get_element_path(parent) + (element.tag,))
)
# def _root_type_verifier(method):
# # Just to make sure we are safe
# @wraps(method)
# def _impl(self, *args, **kw):
# # All html types inherits from HtmlMixin so this to check for all at once
# if not issubclass(type(self._root), html.HtmlMixin):
# raise ValueError(f"Cannot use function on a Node of type {type(self._root)!r}")
# return method(self, *args, **kw)
# return _impl
@lru_cache(128, typed=True)
def clean_spaces(string):
string = string.replace('\t', ' ')
string = re.sub('[\n|\r]', '', string)
return re.sub(' +', ' ', string)
string = string.translate(__CLEANING_TABLE__)
return __CONSECUTIVE_SPACES_REGEX__.sub(" ", string)
scrapling/defaults.py
-25
View File
@@ -1,25 +0,0 @@
# Left this file for backward-compatibility before 0.2.99
from scrapling.core.utils import log
# A lightweight approach to create lazy loader for each import for backward compatibility
# This will reduces initial memory footprint significantly (only loads what's used)
def __getattr__(name):
if name == 'Fetcher':
from scrapling.fetchers import Fetcher as cls
log.warning('This import is deprecated now and it will be removed with v0.3. Use `from scrapling.fetchers import Fetcher` instead')
return cls
elif name == 'AsyncFetcher':
from scrapling.fetchers import AsyncFetcher as cls
log.warning('This import is deprecated now and it will be removed with v0.3. Use `from scrapling.fetchers import AsyncFetcher` instead')
return cls
elif name == 'StealthyFetcher':
from scrapling.fetchers import StealthyFetcher as cls
log.warning('This import is deprecated now and it will be removed with v0.3. Use `from scrapling.fetchers import StealthyFetcher` instead')
return cls
elif name == 'PlayWrightFetcher':
from scrapling.fetchers import PlayWrightFetcher as cls
log.warning('This import is deprecated now and it will be removed with v0.3. Use `from scrapling.fetchers import PlayWrightFetcher` instead')
return cls
else:
raise AttributeError(f"module 'scrapling' has no attribute '{name}'")
scrapling/engines/__init__.py
+15 -6
View File
@@ -1,7 +1,16 @@
from .camo import CamoufoxEngine
from .constants import DEFAULT_DISABLED_RESOURCES, DEFAULT_STEALTH_FLAGS
from .pw import PlaywrightEngine
from .static import StaticEngine
from .toolbelt import check_if_engine_usable
from .constants import DEFAULT_DISABLED_RESOURCES, DEFAULT_STEALTH_FLAGS, DEFAULT_FLAGS
from .static import FetcherSession, FetcherClient, AsyncFetcherClient
from ._browsers import (
DynamicSession,
AsyncDynamicSession,
StealthySession,
AsyncStealthySession,
)
__all__ = ['CamoufoxEngine', 'PlaywrightEngine']
__all__ = [
"FetcherSession",
"DynamicSession",
"AsyncDynamicSession",
"StealthySession",
"AsyncStealthySession",
]
scrapling/engines/_browsers/__init__.py
+2
View File
@@ -0,0 +1,2 @@
from ._controllers import DynamicSession, AsyncDynamicSession
from ._camoufox import StealthySession, AsyncStealthySession
scrapling/engines/_browsers/_camoufox.py
+745
View File
@@ -0,0 +1,745 @@
from time import time, sleep
from re import compile as re_compile
from asyncio import sleep as asyncio_sleep, Lock
from camoufox import DefaultAddons
from camoufox.utils import launch_options as generate_launch_options
from playwright.sync_api import (
Response as SyncPlaywrightResponse,
sync_playwright,
BrowserContext,
Playwright,
Locator,
Page,
)
from playwright.async_api import (
async_playwright,
Response as AsyncPlaywrightResponse,
BrowserContext as AsyncBrowserContext,
Playwright as AsyncPlaywright,
Locator as AsyncLocator,
Page as async_Page,
)
from scrapling.core.utils import log
from ._page import PageInfo, PagePool
from ._validators import validate, CamoufoxConfig
from scrapling.core._types import (
Dict,
List,
Optional,
Callable,
SelectorWaitStates,
)
from scrapling.engines.toolbelt import (
Response,
ResponseFactory,
async_intercept_route,
generate_convincing_referer,
get_os_name,
intercept_route,
)
__CF_PATTERN__ = re_compile("challenges.cloudflare.com/cdn-cgi/challenge-platform/.*")
class StealthySession:
"""A Stealthy session manager with page pooling."""
__slots__ = (
"max_pages",
"headless",
"block_images",
"disable_resources",
"block_webrtc",
"allow_webgl",
"network_idle",
"humanize",
"solve_cloudflare",
"wait",
"timeout",
"page_action",
"wait_selector",
"addons",
"wait_selector_state",
"cookies",
"google_search",
"extra_headers",
"proxy",
"os_randomize",
"disable_ads",
"geoip",
"selector_config",
"additional_args",
"playwright",
"browser",
"context",
"page_pool",
"_closed",
"launch_options",
"_headers_keys",
)
def __init__(
self,
max_pages: int = 1,
headless: bool = True, # noqa: F821
block_images: bool = False,
disable_resources: bool = False,
block_webrtc: bool = False,
allow_webgl: bool = True,
network_idle: bool = False,
humanize: bool | float = True,
solve_cloudflare: bool = False,
wait: int | float = 0,
timeout: int | float = 30000,
page_action: Optional[Callable] = None,
wait_selector: Optional[str] = None,
addons: Optional[List[str]] = None,
wait_selector_state: SelectorWaitStates = "attached",
cookies: Optional[List[Dict]] = None,
google_search: bool = True,
extra_headers: Optional[Dict[str, str]] = None,
proxy: Optional[str | Dict[str, str]] = None,
os_randomize: bool = False,
disable_ads: bool = False,
geoip: bool = False,
selector_config: Optional[Dict] = None,
additional_args: Optional[Dict] = None,
):
"""A Browser session manager with page pooling
:param headless: Run the browser in headless/hidden (default), or headful/visible mode.
:param block_images: Prevent the loading of images through Firefox preferences.
This can help save your proxy usage but be careful with this option as it makes some websites never finish loading.
:param disable_resources: Drop requests of unnecessary resources for a speed boost. It depends, but it made requests ~25% faster in my tests for some websites.
Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
This can help save your proxy usage but be careful with this option as it makes some websites never finish loading.
:param block_webrtc: Blocks WebRTC entirely.
:param cookies: Set cookies for the next request.
:param addons: List of Firefox addons to use. Must be paths to extracted addons.
:param humanize: Humanize the cursor movement. Takes either True or the MAX duration in seconds of the cursor movement. The cursor typically takes up to 1.5 seconds to move across the window.
:param solve_cloudflare: Solves all 3 types of the Cloudflare's Turnstile wait page before returning the response to you.
:param allow_webgl: Enabled by default. Disabling WebGL is not recommended as many WAFs now check if WebGL is enabled.
:param network_idle: Wait for the page until there are no network connections for at least 500 ms.
:param disable_ads: Disabled by default, this installs the `uBlock Origin` addon on the browser if enabled.
:param os_randomize: If enabled, Scrapling will randomize the OS fingerprints used. The default is Scrapling matching the fingerprints with the current OS.
:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
:param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
:param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again.
:param wait_selector: Wait for a specific CSS selector to be in a specific state.
:param geoip: Recommended to use with proxies; Automatically use IP's longitude, latitude, timezone, country, locale, and spoof the WebRTC IP address.
It will also calculate and spoof the browser's language based on the distribution of language speakers in the target region.
:param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.
:param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name.
:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
:param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.
:param max_pages: The maximum number of tabs to be opened at the same time. It will be used in rotation through a PagePool.
:param selector_config: The arguments that will be passed in the end while creating the final Selector's class.
:param additional_args: Additional arguments to be passed to Camoufox as additional settings, and it takes higher priority than Scrapling's settings.
"""
params = {
"max_pages": max_pages,
"headless": headless,
"block_images": block_images,
"disable_resources": disable_resources,
"block_webrtc": block_webrtc,
"allow_webgl": allow_webgl,
"network_idle": network_idle,
"humanize": humanize,
"solve_cloudflare": solve_cloudflare,
"wait": wait,
"timeout": timeout,
"page_action": page_action,
"wait_selector": wait_selector,
"addons": addons,
"wait_selector_state": wait_selector_state,
"cookies": cookies,
"google_search": google_search,
"extra_headers": extra_headers,
"proxy": proxy,
"os_randomize": os_randomize,
"disable_ads": disable_ads,
"geoip": geoip,
"selector_config": selector_config,
"additional_args": additional_args,
}
config = validate(params, CamoufoxConfig)
self.max_pages = config.max_pages
self.headless = config.headless
self.block_images = config.block_images
self.disable_resources = config.disable_resources
self.block_webrtc = config.block_webrtc
self.allow_webgl = config.allow_webgl
self.network_idle = config.network_idle
self.humanize = config.humanize
self.solve_cloudflare = config.solve_cloudflare
self.wait = config.wait
self.timeout = config.timeout
self.page_action = config.page_action
self.wait_selector = config.wait_selector
self.addons = config.addons
self.wait_selector_state = config.wait_selector_state
self.cookies = config.cookies
self.google_search = config.google_search
self.extra_headers = config.extra_headers
self.proxy = config.proxy
self.os_randomize = config.os_randomize
self.disable_ads = config.disable_ads
self.geoip = config.geoip
self.selector_config = config.selector_config
self.additional_args = config.additional_args
self.playwright: Optional[Playwright] = None
self.context: Optional[BrowserContext] = None
self.page_pool = PagePool(self.max_pages)
self._closed = False
self.selector_config = config.selector_config
self.page_action = config.page_action
self._headers_keys = (
set(map(str.lower, self.extra_headers.keys()))
if self.extra_headers
else set()
)
self.__initiate_browser_options__()
def __initiate_browser_options__(self):
"""Initiate browser options."""
self.launch_options = generate_launch_options(
**{
"geoip": self.geoip,
"proxy": dict(self.proxy) if self.proxy else self.proxy,
"enable_cache": True,
"addons": self.addons,
"exclude_addons": [] if self.disable_ads else [DefaultAddons.UBO],
"headless": self.headless,
"humanize": True if self.solve_cloudflare else self.humanize,
"i_know_what_im_doing": True, # To turn warnings off with the user configurations
"allow_webgl": self.allow_webgl,
"block_webrtc": self.block_webrtc,
"block_images": self.block_images, # Careful! it makes some websites don't finish loading at all like stackoverflow even in headful mode.
"os": None if self.os_randomize else get_os_name(),
"user_data_dir": "",
**self.additional_args,
}
)
def __create__(self):
"""Create a browser for this instance and context."""
self.playwright = sync_playwright().start()
self.context = (
self.playwright.firefox.launch_persistent_context( # pragma: no cover
**self.launch_options
)
)
if self.cookies: # pragma: no cover
self.context.add_cookies(self.cookies)
def __enter__(self): # pragma: no cover
self.__create__()
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self.close()
def close(self): # pragma: no cover
"""Close all resources"""
if self._closed: # pragma: no cover
return
if self.context:
self.context.close()
self.context = None
if self.playwright:
self.playwright.stop()
self.playwright = None
self._closed = True
def _get_or_create_page(self) -> PageInfo: # pragma: no cover
"""Get an available page or create a new one"""
# Try to get a ready page first
page_info = self.page_pool.get_ready_page()
if page_info:
return page_info
# Create a new page if under limit
if self.page_pool.pages_count < self.max_pages:
page = self.context.new_page()
page.set_default_navigation_timeout(self.timeout)
page.set_default_timeout(self.timeout)
if self.extra_headers:
page.set_extra_http_headers(self.extra_headers)
if self.disable_resources:
page.route("**/*", intercept_route)
return self.page_pool.add_page(page)
# Wait for a page to become available
max_wait = 30
start_time = time()
while time() - start_time < max_wait:
page_info = self.page_pool.get_ready_page()
if page_info:
return page_info
sleep(0.05)
raise TimeoutError("No pages available within timeout period")
@staticmethod
def _detect_cloudflare(page_content):
"""
Detect the type of Cloudflare challenge present in the provided page content.
This function analyzes the given page content to identify whether a specific
type of Cloudflare challenge is present. It checks for three predefined
challenge types: non-interactive, managed, and interactive. If a challenge
type is detected, it returns the corresponding type as a string. If no
challenge type is detected, it returns None.
Args:
page_content (str): The content of the page to analyze for Cloudflare
challenge types.
Returns:
str: A string representing the detected Cloudflare challenge type, if
found. Returns None if no challenge matches.
"""
challenge_types = (
"non-interactive",
"managed",
"interactive",
)
for ctype in challenge_types:
if f"cType: '{ctype}'" in page_content:
return ctype
return None
def _solve_cloudflare(self, page: Page) -> None: # pragma: no cover
"""Solve the cloudflare challenge displayed on the playwright page passed
:param page: The targeted page
:return:
"""
challenge_type = self._detect_cloudflare(page.content())
if not challenge_type:
log.error("No Cloudflare challenge found.")
return
else:
log.info(f'The turnstile version discovered is "{challenge_type}"')
if challenge_type == "non-interactive":
while "<title>Just a moment...</title>" in (page.content()):
log.info("Waiting for Cloudflare wait page to disappear.")
page.wait_for_timeout(1000)
page.wait_for_load_state()
log.info("Cloudflare captcha is solved")
return
else:
while "Verifying you are human." in page.content():
# Waiting for the verify spinner to disappear, checking every 1s if it disappeared
page.wait_for_timeout(500)
iframe = page.frame(url=__CF_PATTERN__)
if iframe is None:
log.info("Didn't find Cloudflare iframe!")
return
while not iframe.frame_element().is_visible():
# Double-checking that the iframe is loaded
page.wait_for_timeout(500)
# Calculate the Captcha coordinates for any viewport
outer_box = page.locator(".main-content p+div>div>div").bounding_box()
captcha_x, captcha_y = outer_box["x"] + 26, outer_box["y"] + 25
# Move the mouse to the center of the window, then press and hold the left mouse button
page.mouse.click(captcha_x, captcha_y, delay=60, button="left")
page.locator(".zone-name-title").wait_for(state="hidden")
page.wait_for_load_state(state="domcontentloaded")
log.info("Cloudflare captcha is solved")
return
def fetch(self, url: str) -> Response:
"""Opens up the browser and do your request based on your chosen options.
:param url: The Target url.
:return: A `Response` object.
"""
if self._closed: # pragma: no cover
raise RuntimeError("Context manager has been closed")
final_response = None
referer = (
generate_convincing_referer(url)
if (self.google_search and "referer" not in self._headers_keys)
else None
)
def handle_response(finished_response: SyncPlaywrightResponse):
nonlocal final_response
if (
finished_response.request.resource_type == "document"
and finished_response.request.is_navigation_request()
):
final_response = finished_response
page_info = self._get_or_create_page()
page_info.mark_busy(url=url)
try: # pragma: no cover
# Navigate to URL and wait for a specified state
page_info.page.on("response", handle_response)
first_response = page_info.page.goto(url, referer=referer)
page_info.page.wait_for_load_state(state="domcontentloaded")
if self.network_idle:
page_info.page.wait_for_load_state("networkidle")
if not first_response:
raise RuntimeError(f"Failed to get response for {url}")
if self.solve_cloudflare:
self._solve_cloudflare(page_info.page)
# Make sure the page is fully loaded after the captcha
page_info.page.wait_for_load_state(state="load")
page_info.page.wait_for_load_state(state="domcontentloaded")
if self.network_idle:
page_info.page.wait_for_load_state("networkidle")
if self.page_action is not None:
try:
page_info.page = self.page_action(page_info.page)
except Exception as e:
log.error(f"Error executing page_action: {e}")
if self.wait_selector:
try:
waiter: Locator = page_info.page.locator(self.wait_selector)
waiter.first.wait_for(state=self.wait_selector_state)
# Wait again after waiting for the selector, helpful with protections like Cloudflare
page_info.page.wait_for_load_state(state="load")
page_info.page.wait_for_load_state(state="domcontentloaded")
if self.network_idle:
page_info.page.wait_for_load_state("networkidle")
except Exception as e:
log.error(f"Error waiting for selector {self.wait_selector}: {e}")
page_info.page.wait_for_timeout(self.wait)
response = ResponseFactory.from_playwright_response(
page_info.page, first_response, final_response, self.selector_config
)
# Mark the page as ready for next use
page_info.mark_ready()
return response
except Exception as e: # pragma: no cover
page_info.mark_error()
raise e
def get_pool_stats(self) -> Dict[str, int]:
"""Get statistics about the current page pool"""
return {
"total_pages": self.page_pool.pages_count,
"ready_pages": self.page_pool.ready_count,
"busy_pages": self.page_pool.busy_count,
"max_pages": self.max_pages,
}
class AsyncStealthySession(StealthySession):
"""A Stealthy session manager with page pooling."""
def __init__(
self,
max_pages: int = 1,
headless: bool = True, # noqa: F821
block_images: bool = False,
disable_resources: bool = False,
block_webrtc: bool = False,
allow_webgl: bool = True,
network_idle: bool = False,
humanize: bool | float = True,
solve_cloudflare: bool = False,
wait: int | float = 0,
timeout: int | float = 30000,
page_action: Optional[Callable] = None,
wait_selector: Optional[str] = None,
addons: Optional[List[str]] = None,
wait_selector_state: SelectorWaitStates = "attached",
cookies: Optional[List[Dict]] = None,
google_search: bool = True,
extra_headers: Optional[Dict[str, str]] = None,
proxy: Optional[str | Dict[str, str]] = None,
os_randomize: bool = False,
disable_ads: bool = False,
geoip: bool = False,
selector_config: Optional[Dict] = None,
additional_args: Optional[Dict] = None,
):
"""A Browser session manager with page pooling
:param headless: Run the browser in headless/hidden (default), or headful/visible mode.
:param block_images: Prevent the loading of images through Firefox preferences.
This can help save your proxy usage but be careful with this option as it makes some websites never finish loading.
:param disable_resources: Drop requests of unnecessary resources for a speed boost. It depends, but it made requests ~25% faster in my tests for some websites.
Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
This can help save your proxy usage but be careful with this option as it makes some websites never finish loading.
:param block_webrtc: Blocks WebRTC entirely.
:param cookies: Set cookies for the next request.
:param addons: List of Firefox addons to use. Must be paths to extracted addons.
:param humanize: Humanize the cursor movement. Takes either True or the MAX duration in seconds of the cursor movement. The cursor typically takes up to 1.5 seconds to move across the window.
:param solve_cloudflare: Solves all 3 types of the Cloudflare's Turnstile wait page before returning the response to you.
:param allow_webgl: Enabled by default. Disabling WebGL is not recommended as many WAFs now check if WebGL is enabled.
:param network_idle: Wait for the page until there are no network connections for at least 500 ms.
:param disable_ads: Disabled by default, this installs the `uBlock Origin` addon on the browser if enabled.
:param os_randomize: If enabled, Scrapling will randomize the OS fingerprints used. The default is Scrapling matching the fingerprints with the current OS.
:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
:param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
:param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again.
:param wait_selector: Wait for a specific CSS selector to be in a specific state.
:param geoip: Recommended to use with proxies; Automatically use IP's longitude, latitude, timezone, country, locale, and spoof the WebRTC IP address.
It will also calculate and spoof the browser's language based on the distribution of language speakers in the target region.
:param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.
:param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name.
:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
:param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.
:param max_pages: The maximum number of tabs to be opened at the same time. It will be used in rotation through a PagePool.
:param selector_config: The arguments that will be passed in the end while creating the final Selector's class.
:param additional_args: Additional arguments to be passed to Camoufox as additional settings, and it takes higher priority than Scrapling's settings.
"""
super().__init__(
max_pages,
headless,
block_images,
disable_resources,
block_webrtc,
allow_webgl,
network_idle,
humanize,
solve_cloudflare,
wait,
timeout,
page_action,
wait_selector,
addons,
wait_selector_state,
cookies,
google_search,
extra_headers,
proxy,
os_randomize,
disable_ads,
geoip,
selector_config,
additional_args,
)
self.playwright: Optional[AsyncPlaywright] = None
self.context: Optional[AsyncBrowserContext] = None
self._lock = Lock()
self.__enter__ = None
self.__exit__ = None
async def __create__(self):
"""Create a browser for this instance and context."""
self.playwright: AsyncPlaywright = await async_playwright().start()
self.context: AsyncBrowserContext = (
await self.playwright.firefox.launch_persistent_context(
**self.launch_options
)
)
if self.cookies:
await self.context.add_cookies(self.cookies)
async def __aenter__(self):
await self.__create__()
return self
async def __aexit__(self, exc_type, exc_val, exc_tb):
await self.close()
async def close(self):
"""Close all resources"""
if self._closed: # pragma: no cover
return
if self.context:
await self.context.close()
self.context = None
if self.playwright:
await self.playwright.stop()
self.playwright = None
self._closed = True
async def _get_or_create_page(self) -> PageInfo:
"""Get an available page or create a new one"""
async with self._lock:
# Try to get a ready page first
page_info = self.page_pool.get_ready_page()
if page_info:
return page_info
# Create a new page if under limit
if self.page_pool.pages_count < self.max_pages:
page = await self.context.new_page()
page.set_default_navigation_timeout(self.timeout)
page.set_default_timeout(self.timeout)
if self.extra_headers:
await page.set_extra_http_headers(self.extra_headers)
if self.disable_resources:
await page.route("**/*", async_intercept_route)
return self.page_pool.add_page(page)
# Wait for a page to become available
max_wait = 30
start_time = time()
while time() - start_time < max_wait: # pragma: no cover
page_info = self.page_pool.get_ready_page()
if page_info:
return page_info
await asyncio_sleep(0.05)
raise TimeoutError("No pages available within timeout period")
async def _solve_cloudflare(self, page: async_Page):
"""Solve the cloudflare challenge displayed on the playwright page passed. The async version
:param page: The async targeted page
:return:
"""
challenge_type = self._detect_cloudflare(await page.content())
if not challenge_type:
log.error("No Cloudflare challenge found.")
return
else:
log.info(f'The turnstile version discovered is "{challenge_type}"')
if challenge_type == "non-interactive": # pragma: no cover
while "<title>Just a moment...</title>" in (await page.content()):
log.info("Waiting for Cloudflare wait page to disappear.")
await page.wait_for_timeout(1000)
await page.wait_for_load_state()
log.info("Cloudflare captcha is solved")
return
else:
while "Verifying you are human." in (await page.content()):
# Waiting for the verify spinner to disappear, checking every 1s if it disappeared
await page.wait_for_timeout(500)
iframe = page.frame(url=__CF_PATTERN__)
if iframe is None:
log.info("Didn't find Cloudflare iframe!")
return
while not await (await iframe.frame_element()).is_visible():
# Double-checking that the iframe is loaded
await page.wait_for_timeout(500)
# Calculate the Captcha coordinates for any viewport
outer_box = await page.locator(
".main-content p+div>div>div"
).bounding_box()
captcha_x, captcha_y = outer_box["x"] + 26, outer_box["y"] + 25
# Move the mouse to the center of the window, then press and hold the left mouse button
await page.mouse.click(captcha_x, captcha_y, delay=60, button="left")
await page.locator(".zone-name-title").wait_for(state="hidden")
await page.wait_for_load_state(state="domcontentloaded")
log.info("Cloudflare captcha is solved")
return
async def fetch(self, url: str) -> Response:
"""Opens up the browser and do your request based on your chosen options.
:param url: The Target url.
:return: A `Response` object.
"""
if self._closed: # pragma: no cover
raise RuntimeError("Context manager has been closed")
final_response = None
referer = (
generate_convincing_referer(url)
if (self.google_search and "referer" not in self._headers_keys)
else None
)
async def handle_response(finished_response: AsyncPlaywrightResponse):
nonlocal final_response
if (
finished_response.request.resource_type == "document"
and finished_response.request.is_navigation_request()
):
final_response = finished_response
page_info = await self._get_or_create_page()
page_info.mark_busy(url=url)
try:
# Navigate to URL and wait for a specified state
page_info.page.on("response", handle_response)
first_response = await page_info.page.goto(url, referer=referer)
await page_info.page.wait_for_load_state(state="domcontentloaded")
if self.network_idle:
await page_info.page.wait_for_load_state("networkidle")
if not first_response:
raise RuntimeError(f"Failed to get response for {url}")
if self.solve_cloudflare:
await self._solve_cloudflare(page_info.page)
# Make sure the page is fully loaded after the captcha
await page_info.page.wait_for_load_state(state="load")
await page_info.page.wait_for_load_state(state="domcontentloaded")
if self.network_idle:
await page_info.page.wait_for_load_state("networkidle")
if self.page_action is not None:
try:
page_info.page = await self.page_action(page_info.page)
except Exception as e:
log.error(f"Error executing page_action: {e}")
if self.wait_selector:
try:
waiter: AsyncLocator = page_info.page.locator(self.wait_selector)
await waiter.first.wait_for(state=self.wait_selector_state)
# Wait again after waiting for the selector, helpful with protections like Cloudflare
await page_info.page.wait_for_load_state(state="load")
await page_info.page.wait_for_load_state(state="domcontentloaded")
if self.network_idle:
await page_info.page.wait_for_load_state("networkidle")
except Exception as e:
log.error(f"Error waiting for selector {self.wait_selector}: {e}")
await page_info.page.wait_for_timeout(self.wait)
# Create response object
response = await ResponseFactory.from_async_playwright_response(
page_info.page, first_response, final_response, self.selector_config
)
# Mark the page as ready for next use
page_info.mark_ready()
return response
except Exception as e:
page_info.mark_error()
raise e
scrapling/engines/_browsers/_config_tools.py
+130
View File
@@ -0,0 +1,130 @@
from functools import lru_cache
from scrapling.core._types import Tuple
from scrapling.engines.constants import (
DEFAULT_STEALTH_FLAGS,
HARMFUL_DEFAULT_ARGS,
DEFAULT_FLAGS,
)
from scrapling.engines.toolbelt import js_bypass_path, generate_headers
__default_useragent__ = generate_headers(browser_mode=True).get("User-Agent")
@lru_cache(1)
def _compiled_stealth_scripts():
"""Pre-read and compile stealth scripts"""
# Basic bypasses nothing fancy as I'm still working on it
# But with adding these bypasses to the above config, it bypasses many online tests like
# https://bot.sannysoft.com/
# https://kaliiiiiiiiii.github.io/brotector/
# https://pixelscan.net/
# https://iphey.com/
# https://www.browserscan.net/bot-detection <== this one also checks for the CDP runtime fingerprint
# https://arh.antoinevastel.com/bots/areyouheadless/
# https://prescience-data.github.io/execution-monitor.html
stealth_scripts_paths = tuple(
js_bypass_path(script)
for script in (
# Order is important
"webdriver_fully.js",
"window_chrome.js",
"navigator_plugins.js",
"notification_permission.js",
"screen_props.js",
"playwright_fingerprint.js",
)
)
scripts = []
for script_path in stealth_scripts_paths:
with open(script_path, "r") as f:
scripts.append(f.read())
return tuple(scripts)
@lru_cache(2, typed=True)
def _set_flags(hide_canvas, disable_webgl): # pragma: no cover
"""Returns the flags that will be used while launching the browser if stealth mode is enabled"""
flags = DEFAULT_STEALTH_FLAGS
if hide_canvas:
flags += ("--fingerprinting-canvas-image-data-noise",)
if disable_webgl:
flags += (
"--disable-webgl",
"--disable-webgl-image-chromium",
"--disable-webgl2",
)
return flags
@lru_cache(2, typed=True)
def _launch_kwargs(
headless,
proxy,
locale,
extra_headers,
useragent,
real_chrome,
stealth,
hide_canvas,
disable_webgl,
) -> Tuple:
"""Creates the arguments we will use while launching playwright's browser"""
launch_kwargs = {
"locale": locale,
"headless": headless,
"args": DEFAULT_FLAGS,
"color_scheme": "dark", # Bypasses the 'prefersLightColor' check in creepjs
"proxy": proxy or tuple(),
"device_scale_factor": 2,
"ignore_default_args": HARMFUL_DEFAULT_ARGS,
"channel": "chrome" if real_chrome else "chromium",
"extra_http_headers": extra_headers or tuple(),
"user_agent": useragent or __default_useragent__,
}
if stealth:
launch_kwargs.update(
{
"args": DEFAULT_FLAGS + _set_flags(hide_canvas, disable_webgl),
"chromium_sandbox": True,
"is_mobile": False,
"has_touch": False,
# I'm thinking about disabling it to rest from all Service Workers' headache, but let's keep it as it is for now
"service_workers": "allow",
"ignore_https_errors": True,
"screen": {"width": 1920, "height": 1080},
"viewport": {"width": 1920, "height": 1080},
"permissions": ["geolocation", "notifications"],
}
)
return tuple(launch_kwargs.items())
@lru_cache(2, typed=True)
def _context_kwargs(proxy, locale, extra_headers, useragent, stealth) -> Tuple:
"""Creates the arguments for the browser context"""
context_kwargs = {
"proxy": proxy or tuple(),
"locale": locale,
"color_scheme": "dark", # Bypasses the 'prefersLightColor' check in creepjs
"device_scale_factor": 2,
"extra_http_headers": extra_headers or tuple(),
"user_agent": useragent or __default_useragent__,
}
if stealth:
context_kwargs.update(
{
"is_mobile": False,
"has_touch": False,
# I'm thinking about disabling it to rest from all Service Workers' headache, but let's keep it as it is for now
"service_workers": "allow",
"ignore_https_errors": True,
"screen": {"width": 1920, "height": 1080},
"viewport": {"width": 1920, "height": 1080},
"permissions": ["geolocation", "notifications"],
}
)
return tuple(context_kwargs.items())
scrapling/engines/_browsers/_controllers.py
+630
View File
@@ -0,0 +1,630 @@
from time import time, sleep
from asyncio import sleep as asyncio_sleep, Lock
from playwright.sync_api import (
Response as SyncPlaywrightResponse,
sync_playwright,
BrowserContext,
Playwright,
Locator,
)
from playwright.async_api import (
async_playwright,
Response as AsyncPlaywrightResponse,
BrowserContext as AsyncBrowserContext,
Playwright as AsyncPlaywright,
Locator as AsyncLocator,
)
from rebrowser_playwright.sync_api import sync_playwright as sync_rebrowser_playwright
from rebrowser_playwright.async_api import (
async_playwright as async_rebrowser_playwright,
)
from scrapling.core.utils import log
from ._page import PageInfo, PagePool
from ._validators import validate, PlaywrightConfig
from ._config_tools import _compiled_stealth_scripts, _launch_kwargs, _context_kwargs
from scrapling.core._types import (
Dict,
List,
Optional,
Callable,
SelectorWaitStates,
)
from scrapling.engines.toolbelt import (
Response,
ResponseFactory,
generate_convincing_referer,
intercept_route,
async_intercept_route,
)
class DynamicSession:
"""A Browser session manager with page pooling."""
__slots__ = (
"max_pages",
"headless",
"hide_canvas",
"disable_webgl",
"real_chrome",
"stealth",
"google_search",
"proxy",
"locale",
"extra_headers",
"useragent",
"timeout",
"cookies",
"disable_resources",
"network_idle",
"wait_selector",
"wait_selector_state",
"wait",
"playwright",
"browser",
"context",
"page_pool",
"_closed",
"selector_config",
"page_action",
"launch_options",
"context_options",
"cdp_url",
"_headers_keys",
)
def __init__(
self,
__max_pages: int = 1,
headless: bool = True,
google_search: bool = True,
hide_canvas: bool = False,
disable_webgl: bool = False,
real_chrome: bool = False,
stealth: bool = False,
wait: int | float = 0,
page_action: Optional[Callable] = None,
proxy: Optional[str | Dict[str, str]] = None,
locale: str = "en-US",
extra_headers: Optional[Dict[str, str]] = None,
useragent: Optional[str] = None,
cdp_url: Optional[str] = None,
timeout: int | float = 30000,
disable_resources: bool = False,
wait_selector: Optional[str] = None,
cookies: Optional[List[Dict]] = None,
network_idle: bool = False,
wait_selector_state: SelectorWaitStates = "attached",
selector_config: Optional[Dict] = None,
):
"""A Browser session manager with page pooling, it's using a persistent browser Context by default with a temporary user profile directory.
:param headless: Run the browser in headless/hidden (default), or headful/visible mode.
:param disable_resources: Drop requests of unnecessary resources for a speed boost. It depends, but it made requests ~25% faster in my tests for some websites.
Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
This can help save your proxy usage but be careful with this option as it makes some websites never finish loading.
:param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it.
:param cookies: Set cookies for the next request.
:param network_idle: Wait for the page until there are no network connections for at least 500 ms.
:param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
:param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again.
:param wait_selector: Wait for a specific CSS selector to be in a specific state.
:param locale: Set the locale for the browser if wanted. The default value is `en-US`.
:param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.
:param stealth: Enables stealth mode, check the documentation to see what stealth mode does currently.
:param real_chrome: If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch an instance of your browser and use it.
:param hide_canvas: Add random noise to canvas operations to prevent fingerprinting.
:param disable_webgl: Disables WebGL and WebGL 2.0 support entirely.
:param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers/NSTBrowser through CDP.
:param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name.
:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
:param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.
:param selector_config: The arguments that will be passed in the end while creating the final Selector's class.
"""
params = {
"max_pages": __max_pages,
"headless": headless,
"google_search": google_search,
"hide_canvas": hide_canvas,
"disable_webgl": disable_webgl,
"real_chrome": real_chrome,
"stealth": stealth,
"wait": wait,
"page_action": page_action,
"proxy": proxy,
"locale": locale,
"extra_headers": extra_headers,
"useragent": useragent,
"timeout": timeout,
"selector_config": selector_config,
"disable_resources": disable_resources,
"wait_selector": wait_selector,
"cookies": cookies,
"network_idle": network_idle,
"wait_selector_state": wait_selector_state,
"cdp_url": cdp_url,
}
config = validate(params, PlaywrightConfig)
self.max_pages = config.max_pages
self.headless = config.headless
self.hide_canvas = config.hide_canvas
self.disable_webgl = config.disable_webgl
self.real_chrome = config.real_chrome
self.stealth = config.stealth
self.google_search = config.google_search
self.wait = config.wait
self.proxy = config.proxy
self.locale = config.locale
self.extra_headers = config.extra_headers
self.useragent = config.useragent
self.timeout = config.timeout
self.cookies = config.cookies
self.disable_resources = config.disable_resources
self.cdp_url = config.cdp_url
self.network_idle = config.network_idle
self.wait_selector = config.wait_selector
self.wait_selector_state = config.wait_selector_state
self.playwright: Optional[Playwright] = None
self.context: Optional[BrowserContext] = None
self.page_pool = PagePool(self.max_pages)
self._closed = False
self.selector_config = config.selector_config
self.page_action = config.page_action
self._headers_keys = (
set(map(str.lower, self.extra_headers.keys()))
if self.extra_headers
else set()
)
self.__initiate_browser_options__()
def __initiate_browser_options__(self):
if not self.cdp_url:
# `launch_options` is used with persistent context
self.launch_options = dict(
_launch_kwargs(
self.headless,
self.proxy,
self.locale,
tuple(self.extra_headers.items())
if self.extra_headers
else tuple(),
self.useragent,
self.real_chrome,
self.stealth,
self.hide_canvas,
self.disable_webgl,
)
)
self.launch_options["extra_http_headers"] = dict(
self.launch_options["extra_http_headers"]
)
self.launch_options["proxy"] = dict(self.launch_options["proxy"]) or None
self.context_options = dict()
else:
# while `context_options` is left to be used when cdp mode is enabled
self.launch_options = dict()
self.context_options = dict(
_context_kwargs(
self.proxy,
self.locale,
tuple(self.extra_headers.items())
if self.extra_headers
else tuple(),
self.useragent,
self.stealth,
)
)
self.context_options["extra_http_headers"] = dict(
self.context_options["extra_http_headers"]
)
self.context_options["proxy"] = dict(self.context_options["proxy"]) or None
def __create__(self):
"""Create a browser for this instance and context."""
sync_context = sync_rebrowser_playwright
if not self.stealth or self.real_chrome:
# Because rebrowser_playwright doesn't play well with real browsers
sync_context = sync_playwright
self.playwright = sync_context().start()
if self.cdp_url: # pragma: no cover
self.context = self.playwright.chromium.connect_over_cdp(
endpoint_url=self.cdp_url
).new_context(**self.context_options)
else:
self.context = self.playwright.chromium.launch_persistent_context(
user_data_dir="", **self.launch_options
)
if self.cookies: # pragma: no cover
self.context.add_cookies(self.cookies)
def __enter__(self):
self.__create__()
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self.close()
def close(self): # pragma: no cover
"""Close all resources"""
if self._closed:
return
if self.context:
self.context.close()
self.context = None
if self.playwright:
self.playwright.stop()
self.playwright = None
self._closed = True
def _get_or_create_page(self) -> PageInfo: # pragma: no cover
"""Get an available page or create a new one"""
# Try to get a ready page first
page_info = self.page_pool.get_ready_page()
if page_info:
return page_info
# Create a new page if under limit
if self.page_pool.pages_count < self.max_pages:
page = self.context.new_page()
page.set_default_navigation_timeout(self.timeout)
page.set_default_timeout(self.timeout)
if self.extra_headers:
page.set_extra_http_headers(self.extra_headers)
if self.disable_resources:
page.route("**/*", intercept_route)
if self.stealth:
for script in _compiled_stealth_scripts():
page.add_init_script(script=script)
return self.page_pool.add_page(page)
# Wait for a page to become available
max_wait = 30
start_time = time()
while time() - start_time < max_wait:
page_info = self.page_pool.get_ready_page()
if page_info:
return page_info
sleep(0.05)
raise TimeoutError("No pages available within timeout period")
def fetch(self, url: str) -> Response:
"""Opens up the browser and do your request based on your chosen options.
:param url: The Target url.
:return: A `Response` object.
"""
if self._closed: # pragma: no cover
raise RuntimeError("Context manager has been closed")
final_response = None
referer = (
generate_convincing_referer(url)
if (self.google_search and "referer" not in self._headers_keys)
else None
)
def handle_response(finished_response: SyncPlaywrightResponse):
nonlocal final_response
if (
finished_response.request.resource_type == "document"
and finished_response.request.is_navigation_request()
):
final_response = finished_response
page_info = self._get_or_create_page()
page_info.mark_busy(url=url)
try: # pragma: no cover
# Navigate to URL and wait for a specified state
page_info.page.on("response", handle_response)
first_response = page_info.page.goto(url, referer=referer)
page_info.page.wait_for_load_state(state="domcontentloaded")
if self.network_idle:
page_info.page.wait_for_load_state("networkidle")
if not first_response:
raise RuntimeError(f"Failed to get response for {url}")
if self.page_action is not None:
try:
page_info.page = self.page_action(page_info.page)
except Exception as e: # pragma: no cover
log.error(f"Error executing page_action: {e}")
if self.wait_selector:
try:
waiter: Locator = page_info.page.locator(self.wait_selector)
waiter.first.wait_for(state=self.wait_selector_state)
# Wait again after waiting for the selector, helpful with protections like Cloudflare
page_info.page.wait_for_load_state(state="load")
page_info.page.wait_for_load_state(state="domcontentloaded")
if self.network_idle:
page_info.page.wait_for_load_state("networkidle")
except Exception as e: # pragma: no cover
log.error(f"Error waiting for selector {self.wait_selector}: {e}")
page_info.page.wait_for_timeout(self.wait)
# Create response object
response = ResponseFactory.from_playwright_response(
page_info.page, first_response, final_response, self.selector_config
)
# Mark the page as ready for next use
page_info.mark_ready()
return response
except Exception as e:
page_info.mark_error()
raise e
def get_pool_stats(self) -> Dict[str, int]:
"""Get statistics about the current page pool"""
return {
"total_pages": self.page_pool.pages_count,
"ready_pages": self.page_pool.ready_count,
"busy_pages": self.page_pool.busy_count,
"max_pages": self.max_pages,
}
class AsyncDynamicSession(DynamicSession):
"""An async Browser session manager with page pooling, it's using a persistent browser Context by default with a temporary user profile directory."""
def __init__(
self,
max_pages: int = 1,
headless: bool = True,
google_search: bool = True,
hide_canvas: bool = False,
disable_webgl: bool = False,
real_chrome: bool = False,
stealth: bool = False,
wait: int | float = 0,
page_action: Optional[Callable] = None,
proxy: Optional[str | Dict[str, str]] = None,
locale: str = "en-US",
extra_headers: Optional[Dict[str, str]] = None,
useragent: Optional[str] = None,
cdp_url: Optional[str] = None,
timeout: int | float = 30000,
disable_resources: bool = False,
wait_selector: Optional[str] = None,
cookies: Optional[List[Dict]] = None,
network_idle: bool = False,
wait_selector_state: SelectorWaitStates = "attached",
selector_config: Optional[Dict] = None,
):
"""A Browser session manager with page pooling
:param headless: Run the browser in headless/hidden (default), or headful/visible mode.
:param disable_resources: Drop requests of unnecessary resources for a speed boost. It depends, but it made requests ~25% faster in my tests for some websites.
Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
This can help save your proxy usage but be careful with this option as it makes some websites never finish loading.
:param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it.
:param cookies: Set cookies for the next request.
:param network_idle: Wait for the page until there are no network connections for at least 500 ms.
:param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
:param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again.
:param wait_selector: Wait for a specific CSS selector to be in a specific state.
:param locale: Set the locale for the browser if wanted. The default value is `en-US`.
:param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.
:param stealth: Enables stealth mode, check the documentation to see what stealth mode does currently.
:param real_chrome: If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch an instance of your browser and use it.
:param hide_canvas: Add random noise to canvas operations to prevent fingerprinting.
:param disable_webgl: Disables WebGL and WebGL 2.0 support entirely.
:param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers/NSTBrowser through CDP.
:param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name.
:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
:param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.
:param max_pages: The maximum number of tabs to be opened at the same time. It will be used in rotation through a PagePool.
:param selector_config: The arguments that will be passed in the end while creating the final Selector's class.
"""
super().__init__(
max_pages,
headless,
google_search,
hide_canvas,
disable_webgl,
real_chrome,
stealth,
wait,
page_action,
proxy,
locale,
extra_headers,
useragent,
cdp_url,
timeout,
disable_resources,
wait_selector,
cookies,
network_idle,
wait_selector_state,
selector_config,
)
self.playwright: Optional[AsyncPlaywright] = None
self.context: Optional[AsyncBrowserContext] = None
self._lock = Lock()
self.__enter__ = None
self.__exit__ = None
async def __create__(self):
"""Create a browser for this instance and context."""
async_context = async_rebrowser_playwright
if not self.stealth or self.real_chrome:
# Because rebrowser_playwright doesn't play well with real browsers
async_context = async_playwright
self.playwright: AsyncPlaywright = await async_context().start()
if self.cdp_url:
browser = await self.playwright.chromium.connect_over_cdp(
endpoint_url=self.cdp_url
)
self.context: AsyncBrowserContext = await browser.new_context(
**self.context_options
)
else:
self.context: AsyncBrowserContext = (
await self.playwright.chromium.launch_persistent_context(
user_data_dir="", **self.launch_options
)
)
if self.cookies:
await self.context.add_cookies(self.cookies)
async def __aenter__(self):
await self.__create__()
return self
async def __aexit__(self, exc_type, exc_val, exc_tb):
await self.close()
async def close(self):
"""Close all resources"""
if self._closed: # pragma: no cover
return
if self.context:
await self.context.close()
self.context = None
if self.playwright:
await self.playwright.stop()
self.playwright = None
self._closed = True
async def _get_or_create_page(self) -> PageInfo:
"""Get an available page or create a new one"""
async with self._lock:
# Try to get a ready page first
page_info = self.page_pool.get_ready_page()
if page_info:
return page_info
# Create a new page if under limit
if self.page_pool.pages_count < self.max_pages:
page = await self.context.new_page()
page.set_default_navigation_timeout(self.timeout)
page.set_default_timeout(self.timeout)
if self.extra_headers:
await page.set_extra_http_headers(self.extra_headers)
if self.disable_resources:
await page.route("**/*", async_intercept_route)
if self.stealth:
for script in _compiled_stealth_scripts():
await page.add_init_script(script=script)
return self.page_pool.add_page(page)
# Wait for a page to become available
max_wait = 30 # seconds
start_time = time()
while time() - start_time < max_wait: # pragma: no cover
page_info = self.page_pool.get_ready_page()
if page_info:
return page_info
await asyncio_sleep(0.05)
raise TimeoutError("No pages available within timeout period")
async def fetch(self, url: str) -> Response:
"""Opens up the browser and do your request based on your chosen options.
:param url: The Target url.
:return: A `Response` object.
"""
if self._closed: # pragma: no cover
raise RuntimeError("Context manager has been closed")
final_response = None
referer = (
generate_convincing_referer(url)
if (self.google_search and "referer" not in self._headers_keys)
else None
)
async def handle_response(finished_response: AsyncPlaywrightResponse):
nonlocal final_response
if (
finished_response.request.resource_type == "document"
and finished_response.request.is_navigation_request()
):
final_response = finished_response
page_info = await self._get_or_create_page()
page_info.mark_busy(url=url)
try:
# Navigate to URL and wait for a specified state
page_info.page.on("response", handle_response)
first_response = await page_info.page.goto(url, referer=referer)
await page_info.page.wait_for_load_state(state="domcontentloaded")
if self.network_idle:
await page_info.page.wait_for_load_state("networkidle")
if not first_response:
raise RuntimeError(f"Failed to get response for {url}")
if self.page_action is not None:
try:
page_info.page = await self.page_action(page_info.page)
except Exception as e:
log.error(f"Error executing page_action: {e}")
if self.wait_selector:
try:
waiter: AsyncLocator = page_info.page.locator(self.wait_selector)
await waiter.first.wait_for(state=self.wait_selector_state)
# Wait again after waiting for the selector, helpful with protections like Cloudflare
await page_info.page.wait_for_load_state(state="load")
await page_info.page.wait_for_load_state(state="domcontentloaded")
if self.network_idle:
await page_info.page.wait_for_load_state("networkidle")
except Exception as e:
log.error(f"Error waiting for selector {self.wait_selector}: {e}")
await page_info.page.wait_for_timeout(self.wait)
# Create response object
response = await ResponseFactory.from_async_playwright_response(
page_info.page, first_response, final_response, self.selector_config
)
# Mark the page as ready for next use
page_info.mark_ready()
return response
except Exception as e: # pragma: no cover
page_info.mark_error()
raise e
scrapling/engines/_browsers/_page.py
+93
View File
@@ -0,0 +1,93 @@
from threading import RLock
from dataclasses import dataclass
from playwright.sync_api import Page as SyncPage
from playwright.async_api import Page as AsyncPage
from scrapling.core._types import Optional, List, Literal
PageState = Literal["ready", "busy", "error"] # States that a page can be in
@dataclass
class PageInfo:
"""Information about the page and its current state"""
__slots__ = ("page", "state", "url")
page: SyncPage | AsyncPage
state: PageState
url: Optional[str]
def mark_busy(self, url: str = ""):
"""Mark the page as busy"""
self.state = "busy"
self.url = url
def mark_ready(self):
"""Mark the page as ready for new requests"""
self.state = "ready"
self.url = ""
def mark_error(self):
"""Mark the page as having an error"""
self.state = "error"
def __repr__(self):
return f'Page(URL="{self.url!r}", state={self.state!r})'
def __eq__(self, other_page):
"""Comparing this page to another page object."""
if other_page.__class__ is not self.__class__:
return NotImplemented
return self.page == other_page.page
class PagePool:
"""Manages a pool of browser pages/tabs with state tracking"""
__slots__ = ("max_pages", "pages", "_lock")
def __init__(self, max_pages: int = 5):
self.max_pages = max_pages
self.pages: List[PageInfo] = []
self._lock = RLock()
def add_page(self, page: SyncPage | AsyncPage) -> PageInfo:
"""Add a new page to the pool"""
with self._lock:
if len(self.pages) >= self.max_pages:
raise RuntimeError(f"Maximum page limit ({self.max_pages}) reached")
page_info = PageInfo(page, "ready", "")
self.pages.append(page_info)
return page_info
def get_ready_page(self) -> Optional[PageInfo]:
"""Get a page that's ready for use"""
with self._lock:
for page_info in self.pages:
if page_info.state == "ready":
return page_info
return None
@property
def pages_count(self) -> int:
"""Get the total number of pages"""
return len(self.pages)
@property
def ready_count(self) -> int:
"""Get the number of ready pages"""
with self._lock:
return sum(1 for p in self.pages if p.state == "ready")
@property
def busy_count(self) -> int:
"""Get the number of busy pages"""
with self._lock:
return sum(1 for p in self.pages if p.state == "busy")
def cleanup_error_pages(self):
"""Remove pages in error state"""
with self._lock:
self.pages = [p for p in self.pages if p.state != "error"]
scrapling/engines/_browsers/_validators.py
+150
View File
@@ -0,0 +1,150 @@
from msgspec import Struct, convert, ValidationError
from urllib.parse import urlparse
from pathlib import Path
from scrapling.core._types import (
Optional,
Dict,
Callable,
List,
SelectorWaitStates,
)
from scrapling.engines.toolbelt import construct_proxy_dict
class PlaywrightConfig(Struct, kw_only=True, frozen=False):
"""Configuration struct for validation"""
max_pages: int = 1
cdp_url: Optional[str] = None
headless: bool = True
google_search: bool = True
hide_canvas: bool = False
disable_webgl: bool = False
real_chrome: bool = False
stealth: bool = False
wait: int | float = 0
page_action: Optional[Callable] = None
proxy: Optional[str | Dict[str, str]] = (
None # The default value for proxy in Playwright's source is `None`
)
locale: str = "en-US"
extra_headers: Optional[Dict[str, str]] = None
useragent: Optional[str] = None
timeout: int | float = 30000
disable_resources: bool = False
wait_selector: Optional[str] = None
cookies: Optional[List[Dict]] = None
network_idle: bool = False
wait_selector_state: SelectorWaitStates = "attached"
selector_config: Optional[Dict] = None
def __post_init__(self):
"""Custom validation after msgspec validation"""
if self.max_pages < 1 or self.max_pages > 50:
raise ValueError("max_pages must be between 1 and 50")
if self.timeout < 0:
raise ValueError("timeout must be >= 0")
if self.page_action is not None and not callable(self.page_action):
raise TypeError(
f"page_action must be callable, got {type(self.page_action).__name__}"
)
if self.proxy:
self.proxy = construct_proxy_dict(self.proxy, as_tuple=True)
if self.cdp_url:
self.__validate_cdp(self.cdp_url)
if not self.cookies:
self.cookies = []
if not self.selector_config:
self.selector_config = {}
@staticmethod
def __validate_cdp(cdp_url):
try:
# Check the scheme
if not cdp_url.startswith(("ws://", "wss://")):
raise ValueError("CDP URL must use 'ws://' or 'wss://' scheme")
# Validate hostname and port
if not urlparse(cdp_url).netloc:
raise ValueError("Invalid hostname for the CDP URL")
except AttributeError as e:
raise ValueError(f"Malformed CDP URL: {cdp_url}: {str(e)}")
except Exception as e:
raise ValueError(f"Invalid CDP URL '{cdp_url}': {str(e)}")
class CamoufoxConfig(Struct, kw_only=True, frozen=False):
"""Configuration struct for validation"""
max_pages: int = 1
headless: bool = True # noqa: F821
block_images: bool = False
disable_resources: bool = False
block_webrtc: bool = False
allow_webgl: bool = True
network_idle: bool = False
humanize: bool | float = True
solve_cloudflare: bool = False
wait: int | float = 0
timeout: int | float = 30000
page_action: Optional[Callable] = None
wait_selector: Optional[str] = None
addons: Optional[List[str]] = None
wait_selector_state: SelectorWaitStates = "attached"
cookies: Optional[List[Dict]] = None
google_search: bool = True
extra_headers: Optional[Dict[str, str]] = None
proxy: Optional[str | Dict[str, str]] = (
None # The default value for proxy in Playwright's source is `None`
)
os_randomize: bool = False
disable_ads: bool = False
geoip: bool = False
selector_config: Optional[Dict] = None
additional_args: Optional[Dict] = None
def __post_init__(self):
"""Custom validation after msgspec validation"""
if self.max_pages < 1 or self.max_pages > 50:
raise ValueError("max_pages must be between 1 and 50")
if self.timeout < 0:
raise ValueError("timeout must be >= 0")
if self.page_action is not None and not callable(self.page_action):
raise TypeError(
f"page_action must be callable, got {type(self.page_action).__name__}"
)
if self.proxy:
self.proxy = construct_proxy_dict(self.proxy, as_tuple=True)
if not self.addons:
self.addons = []
else:
for addon in self.addons:
addon_path = Path(addon)
if not addon_path.exists():
raise FileNotFoundError(f"Addon's path not found: {addon}")
elif not addon_path.is_dir():
raise ValueError(
f"Addon's path is not a folder, you need to pass a folder of the extracted addon: {addon}"
)
if not self.cookies:
self.cookies = []
if self.solve_cloudflare and self.timeout < 60_000:
self.timeout = 60_000
if not self.selector_config:
self.selector_config = {}
if not self.additional_args:
self.additional_args = {}
def validate(params, model):
try:
config = convert(params, model)
except ValidationError as e:
raise TypeError(f"Invalid argument type: {e}")
return config
scrapling/engines/camo.py
-339
View File
@@ -1,339 +0,0 @@
from camoufox import DefaultAddons
from camoufox.async_api import AsyncCamoufox
from camoufox.sync_api import Camoufox
from scrapling.core._types import (Callable, Dict, List, Literal, Optional,
SelectorWaitStates, Union)
from scrapling.core.utils import log
from scrapling.engines.toolbelt import (Response, StatusText,
async_intercept_route,
check_type_validity,
construct_proxy_dict,
generate_convincing_referer,
get_os_name, intercept_route)
class CamoufoxEngine:
def __init__(
self, headless: Union[bool, Literal['virtual']] = True, block_images: bool = False, disable_resources: bool = False,
block_webrtc: bool = False, allow_webgl: bool = True, network_idle: bool = False, humanize: Union[bool, float] = True, wait: Optional[int] = 0,
timeout: Optional[float] = 30000, page_action: Callable = None, wait_selector: Optional[str] = None, addons: Optional[List[str]] = None,
wait_selector_state: SelectorWaitStates = 'attached', google_search: bool = True, extra_headers: Optional[Dict[str, str]] = None,
proxy: Optional[Union[str, Dict[str, str]]] = None, os_randomize: bool = False, disable_ads: bool = False,
geoip: bool = False,
adaptor_arguments: Dict = None,
additional_arguments: Dict = None
):
"""An engine that utilizes Camoufox library, check the `StealthyFetcher` class for more documentation.
:param headless: Run the browser in headless/hidden (default), virtual screen mode, or headful/visible mode.
:param block_images: Prevent the loading of images through Firefox preferences.
This can help save your proxy usage but be careful with this option as it makes some websites never finish loading.
:param disable_resources: Drop requests of unnecessary resources for a speed boost. It depends but it made requests ~25% faster in my tests for some websites.
Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
This can help save your proxy usage but be careful with this option as it makes some websites never finish loading.
:param block_webrtc: Blocks WebRTC entirely.
:param addons: List of Firefox addons to use. Must be paths to extracted addons.
:param humanize: Humanize the cursor movement. Takes either True or the MAX duration in seconds of the cursor movement. The cursor typically takes up to 1.5 seconds to move across the window.
:param allow_webgl: Enabled by default. Disabling it WebGL not recommended as many WAFs now checks if WebGL is enabled.
:param network_idle: Wait for the page until there are no network connections for at least 500 ms.
:param disable_ads: Disabled by default, this installs `uBlock Origin` addon on the browser if enabled.
:param os_randomize: If enabled, Scrapling will randomize the OS fingerprints used. The default is Scrapling matching the fingerprints with the current OS.
:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning `Response` object.
:param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30000
:param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again.
:param wait_selector: Wait for a specific css selector to be in a specific state.
:param geoip: Recommended to use with proxies; Automatically use IP's longitude, latitude, timezone, country, locale, & spoof the WebRTC IP address.
It will also calculate and spoof the browser's language based on the distribution of language speakers in the target region.
:param wait_selector_state: The state to wait for the selector given with `wait_selector`. Default state is `attached`.
:param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search for this website's domain name.
:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
:param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.
:param adaptor_arguments: The arguments that will be passed in the end while creating the final Adaptor's class.
:param additional_arguments: Additional arguments to be passed to Camoufox as additional settings and it takes higher priority than Scrapling's settings.
"""
self.headless = headless
self.block_images = bool(block_images)
self.disable_resources = bool(disable_resources)
self.block_webrtc = bool(block_webrtc)
self.allow_webgl = bool(allow_webgl)
self.network_idle = bool(network_idle)
self.google_search = bool(google_search)
self.os_randomize = bool(os_randomize)
self.disable_ads = bool(disable_ads)
self.geoip = bool(geoip)
self.extra_headers = extra_headers or {}
self.additional_arguments = additional_arguments or {}
self.proxy = construct_proxy_dict(proxy)
self.addons = addons or []
self.humanize = humanize
self.timeout = check_type_validity(timeout, [int, float], 30000)
self.wait = check_type_validity(wait, [int, float], 0)
# Page action callable validation
self.page_action = None
if page_action is not None:
if callable(page_action):
self.page_action = page_action
else:
log.error('[Ignored] Argument "page_action" must be callable')
self.wait_selector = wait_selector
self.wait_selector_state = wait_selector_state
self.adaptor_arguments = adaptor_arguments if adaptor_arguments else {}
def _get_camoufox_options(self):
"""Return consistent browser options dictionary for both sync and async methods"""
return {
"geoip": self.geoip,
"proxy": self.proxy,
"enable_cache": True,
"addons": self.addons,
"exclude_addons": [] if self.disable_ads else [DefaultAddons.UBO],
"headless": self.headless,
"humanize": self.humanize,
"i_know_what_im_doing": True, # To turn warnings off with the user configurations
"allow_webgl": self.allow_webgl,
"block_webrtc": self.block_webrtc,
"block_images": self.block_images, # Careful! it makes some websites doesn't finish loading at all like stackoverflow even in headful
"os": None if self.os_randomize else get_os_name(),
**self.additional_arguments
}
def _process_response_history(self, first_response):
"""Process response history to build a list of Response objects"""
history = []
current_request = first_response.request.redirected_from
try:
while current_request:
try:
current_response = current_request.response()
history.insert(0, Response(
url=current_request.url,
# using current_response.text() will trigger "Error: Response.text: Response body is unavailable for redirect responses"
text='',
body=b'',
status=current_response.status if current_response else 301,
reason=(current_response.status_text or StatusText.get(current_response.status)) if current_response else StatusText.get(301),
encoding=current_response.headers.get('content-type', '') or 'utf-8',
cookies={},
headers=current_response.all_headers() if current_response else {},
request_headers=current_request.all_headers(),
**self.adaptor_arguments
))
except Exception as e:
log.error(f"Error processing redirect: {e}")
break
current_request = current_request.redirected_from
except Exception as e:
log.error(f"Error processing response history: {e}")
return history
async def _async_process_response_history(self, first_response):
"""Process response history to build a list of Response objects"""
history = []
current_request = first_response.request.redirected_from
try:
while current_request:
try:
current_response = await current_request.response()
history.insert(0, Response(
url=current_request.url,
# using current_response.text() will trigger "Error: Response.text: Response body is unavailable for redirect responses"
text='',
body=b'',
status=current_response.status if current_response else 301,
reason=(current_response.status_text or StatusText.get(current_response.status)) if current_response else StatusText.get(301),
encoding=current_response.headers.get('content-type', '') or 'utf-8',
cookies={},
headers=await current_response.all_headers() if current_response else {},
request_headers=await current_request.all_headers(),
**self.adaptor_arguments
))
except Exception as e:
log.error(f"Error processing redirect: {e}")
break
current_request = current_request.redirected_from
except Exception as e:
log.error(f"Error processing response history: {e}")
return history
def fetch(self, url: str) -> Response:
"""Opens up the browser and do your request based on your chosen options.
:param url: Target url.
:return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
"""
final_response = None
referer = generate_convincing_referer(url) if self.google_search else None
def handle_response(finished_response):
nonlocal final_response
if finished_response.request.resource_type == "document" and finished_response.request.is_navigation_request():
final_response = finished_response
with Camoufox(**self._get_camoufox_options()) as browser:
context = browser.new_context()
page = context.new_page()
page.set_default_navigation_timeout(self.timeout)
page.set_default_timeout(self.timeout)
page.on("response", handle_response)
if self.disable_resources:
page.route("**/*", intercept_route)
if self.extra_headers:
page.set_extra_http_headers(self.extra_headers)
first_response = page.goto(url, referer=referer)
page.wait_for_load_state(state="domcontentloaded")
if self.network_idle:
page.wait_for_load_state('networkidle')
if self.page_action is not None:
try:
page = self.page_action(page)
except Exception as e:
log.error(f"Error executing page_action: {e}")
if self.wait_selector and type(self.wait_selector) is str:
try:
waiter = page.locator(self.wait_selector)
waiter.first.wait_for(state=self.wait_selector_state)
# Wait again after waiting for the selector, helpful with protections like Cloudflare
page.wait_for_load_state(state="load")
page.wait_for_load_state(state="domcontentloaded")
if self.network_idle:
page.wait_for_load_state('networkidle')
except Exception as e:
log.error(f"Error waiting for selector {self.wait_selector}: {e}")
page.wait_for_timeout(self.wait)
# In case we didn't catch a document type somehow
final_response = final_response if final_response else first_response
if not final_response:
raise ValueError("Failed to get a response from the page")
# This will be parsed inside `Response`
encoding = final_response.headers.get('content-type', '') or 'utf-8' # default encoding
# PlayWright API sometimes give empty status text for some reason!
status_text = final_response.status_text or StatusText.get(final_response.status)
history = self._process_response_history(first_response)
try:
page_content = page.content()
except Exception as e:
log.error(f"Error getting page content: {e}")
page_content = ""
response = Response(
url=page.url,
text=page_content,
body=page_content.encode('utf-8'),
status=final_response.status,
reason=status_text,
encoding=encoding,
cookies={cookie['name']: cookie['value'] for cookie in page.context.cookies()},
headers=first_response.all_headers(),
request_headers=first_response.request.all_headers(),
history=history,
**self.adaptor_arguments
)
page.close()
context.close()
return response
async def async_fetch(self, url: str) -> Response:
"""Opens up the browser and do your request based on your chosen options.
:param url: Target url.
:return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
"""
final_response = None
referer = generate_convincing_referer(url) if self.google_search else None
async def handle_response(finished_response):
nonlocal final_response
if finished_response.request.resource_type == "document" and finished_response.request.is_navigation_request():
final_response = finished_response
async with AsyncCamoufox(**self._get_camoufox_options()) as browser:
context = await browser.new_context()
page = await context.new_page()
page.set_default_navigation_timeout(self.timeout)
page.set_default_timeout(self.timeout)
page.on("response", handle_response)
if self.disable_resources:
await page.route("**/*", async_intercept_route)
if self.extra_headers:
await page.set_extra_http_headers(self.extra_headers)
first_response = await page.goto(url, referer=referer)
await page.wait_for_load_state(state="domcontentloaded")
if self.network_idle:
await page.wait_for_load_state('networkidle')
if self.page_action is not None:
try:
page = await self.page_action(page)
except Exception as e:
log.error(f"Error executing async page_action: {e}")
if self.wait_selector and type(self.wait_selector) is str:
try:
waiter = page.locator(self.wait_selector)
await waiter.first.wait_for(state=self.wait_selector_state)
# Wait again after waiting for the selector, helpful with protections like Cloudflare
await page.wait_for_load_state(state="load")
await page.wait_for_load_state(state="domcontentloaded")
if self.network_idle:
await page.wait_for_load_state('networkidle')
except Exception as e:
log.error(f"Error waiting for selector {self.wait_selector}: {e}")
await page.wait_for_timeout(self.wait)
# In case we didn't catch a document type somehow
final_response = final_response if final_response else first_response
if not final_response:
raise ValueError("Failed to get a response from the page")
# This will be parsed inside `Response`
encoding = final_response.headers.get('content-type', '') or 'utf-8' # default encoding
# PlayWright API sometimes give empty status text for some reason!
status_text = final_response.status_text or StatusText.get(final_response.status)
history = await self._async_process_response_history(first_response)
try:
page_content = await page.content()
except Exception as e:
log.error(f"Error getting page content in async: {e}")
page_content = ""
response = Response(
url=page.url,
text=page_content,
body=page_content.encode('utf-8'),
status=final_response.status,
reason=status_text,
encoding=encoding,
cookies={cookie['name']: cookie['value'] for cookie in await page.context.cookies()},
headers=await first_response.all_headers(),
request_headers=await first_response.request.all_headers(),
history=history,
**self.adaptor_arguments
)
await page.close()
await context.close()
return response
scrapling/engines/constants.py
+101 -88
View File
@@ -1,92 +1,108 @@
# Disable loading these resources for speed
DEFAULT_DISABLED_RESOURCES = {
'font',
'image',
'media',
'beacon',
'object',
'imageset',
'texttrack',
'websocket',
'csp_report',
'stylesheet',
"font",
"image",
"media",
"beacon",
"object",
"imageset",
"texttrack",
"websocket",
"csp_report",
"stylesheet",
}
HARMFUL_DEFAULT_ARGS = (
# This will be ignored to avoid detection more and possibly avoid the popup crashing bug abuse: https://issues.chromium.org/issues/340836884
"--enable-automation",
"--disable-popup-blocking",
# '--disable-component-update',
# '--disable-default-apps',
# '--disable-extensions',
)
DEFAULT_FLAGS = (
# Speed up chromium browsers by default
"--no-pings",
"--no-first-run",
"--disable-infobars",
"--disable-breakpad",
"--no-service-autorun",
"--homepage=about:blank",
"--password-store=basic",
"--no-default-browser-check",
"--disable-session-crashed-bubble",
"--disable-search-engine-choice-screen",
)
DEFAULT_STEALTH_FLAGS = (
# Explanation: https://peter.sh/experiments/chromium-command-line-switches/
# Generally this will make the browser faster and less detectable
'--no-pings',
'--incognito',
'--test-type',
'--lang=en-US',
'--mute-audio',
'--no-first-run',
'--disable-sync',
'--hide-scrollbars',
'--disable-logging',
'--start-maximized', # For headless check bypass
'--enable-async-dns',
'--disable-breakpad',
'--disable-infobars',
'--accept-lang=en-US',
'--use-mock-keychain',
'--disable-translate',
'--disable-extensions',
'--disable-voice-input',
'--window-position=0,0',
'--disable-wake-on-wifi',
'--ignore-gpu-blocklist',
'--enable-tcp-fast-open',
'--enable-web-bluetooth',
'--disable-hang-monitor',
'--password-store=basic',
'--disable-cloud-import',
'--disable-default-apps',
'--disable-print-preview',
'--disable-dev-shm-usage',
"--incognito",
"--test-type",
"--lang=en-US",
"--mute-audio",
"--disable-sync",
"--hide-scrollbars",
"--disable-logging",
"--start-maximized", # For headless check bypass
"--enable-async-dns",
"--accept-lang=en-US",
"--use-mock-keychain",
"--disable-translate",
"--disable-extensions",
"--disable-voice-input",
"--window-position=0,0",
"--disable-wake-on-wifi",
"--ignore-gpu-blocklist",
"--enable-tcp-fast-open",
"--enable-web-bluetooth",
"--disable-hang-monitor",
"--disable-cloud-import",
"--disable-default-apps",
"--disable-print-preview",
"--disable-dev-shm-usage",
# '--disable-popup-blocking',
'--metrics-recording-only',
'--disable-crash-reporter',
'--disable-partial-raster',
'--disable-gesture-typing',
'--disable-checker-imaging',
'--disable-prompt-on-repost',
'--force-color-profile=srgb',
'--font-render-hinting=none',
'--no-default-browser-check',
'--aggressive-cache-discard',
'--disable-component-update',
'--disable-cookie-encryption',
'--disable-domain-reliability',
'--disable-threaded-animation',
'--disable-threaded-scrolling',
# '--disable-reading-from-canvas', # For Firefox
'--enable-simple-cache-backend',
'--disable-background-networking',
'--disable-session-crashed-bubble',
'--enable-surface-synchronization',
'--disable-image-animation-resync',
'--disable-renderer-backgrounding',
'--disable-ipc-flooding-protection',
'--prerender-from-omnibox=disabled',
'--safebrowsing-disable-auto-update',
'--disable-offer-upload-credit-cards',
'--disable-features=site-per-process',
'--disable-background-timer-throttling',
'--disable-new-content-rendering-timeout',
'--run-all-compositor-stages-before-draw',
'--disable-client-side-phishing-detection',
'--disable-backgrounding-occluded-windows',
'--disable-layer-tree-host-memory-pressure',
'--autoplay-policy=no-user-gesture-required',
'--disable-offer-store-unmasked-wallet-cards',
'--disable-blink-features=AutomationControlled',
'--webrtc-ip-handling-policy=disable_non_proxied_udp',
'--disable-component-extensions-with-background-pages',
'--force-webrtc-ip-handling-policy=disable_non_proxied_udp',
'--enable-features=NetworkService,NetworkServiceInProcess,TrustTokens,TrustTokensAlwaysAllowIssuance',
'--blink-settings=primaryHoverType=2,availableHoverTypes=2,primaryPointerType=4,availablePointerTypes=4',
'--disable-features=AudioServiceOutOfProcess,IsolateOrigins,site-per-process,TranslateUI,BlinkGenPropertyTrees',
"--metrics-recording-only",
"--disable-crash-reporter",
"--disable-partial-raster",
"--disable-gesture-typing",
"--disable-checker-imaging",
"--disable-prompt-on-repost",
"--force-color-profile=srgb",
"--font-render-hinting=none",
"--aggressive-cache-discard",
"--disable-component-update",
"--disable-cookie-encryption",
"--disable-domain-reliability",
"--disable-threaded-animation",
"--disable-threaded-scrolling",
# '--disable-reading-from-canvas', # For Firefox
"--enable-simple-cache-backend",
"--disable-background-networking",
"--enable-surface-synchronization",
"--disable-image-animation-resync",
"--disable-renderer-backgrounding",
"--disable-ipc-flooding-protection",
"--prerender-from-omnibox=disabled",
"--safebrowsing-disable-auto-update",
"--disable-offer-upload-credit-cards",
"--disable-features=site-per-process",
"--disable-background-timer-throttling",
"--disable-new-content-rendering-timeout",
"--run-all-compositor-stages-before-draw",
"--disable-client-side-phishing-detection",
"--disable-backgrounding-occluded-windows",
"--disable-layer-tree-host-memory-pressure",
"--autoplay-policy=no-user-gesture-required",
"--disable-offer-store-unmasked-wallet-cards",
"--disable-blink-features=AutomationControlled",
"--webrtc-ip-handling-policy=disable_non_proxied_udp",
"--disable-component-extensions-with-background-pages",
"--force-webrtc-ip-handling-policy=disable_non_proxied_udp",
"--enable-features=NetworkService,NetworkServiceInProcess,TrustTokens,TrustTokensAlwaysAllowIssuance",
"--blink-settings=primaryHoverType=2,availableHoverTypes=2,primaryPointerType=4,availablePointerTypes=4",
"--disable-features=AudioServiceOutOfProcess,IsolateOrigins,site-per-process,TranslateUI,BlinkGenPropertyTrees",
)
# Defaulting to the docker mode, token doesn't matter in it as it's passed for the container
@@ -95,13 +111,10 @@ NSTBROWSER_DEFAULT_QUERY = {
"headless": True,
"autoClose": True,
"fingerprint": {
"flags": {
"timezone": "BasedOnIp",
"screen": "Custom"
},
"platform": 'linux', # support: windows, mac, linux
"kernel": 'chromium', # only support: chromium
"kernelMilestone": '128',
"flags": {"timezone": "BasedOnIp", "screen": "Custom"},
"platform": "linux", # support: windows, mac, linux
"kernel": "chromium", # only support: chromium
"kernelMilestone": "128",
"hardwareConcurrency": 8,
"deviceMemory": 8,
},
scrapling/engines/pw.py
-465
View File
@@ -1,465 +0,0 @@
import json
from scrapling.core._types import (Callable, Dict, Optional,
SelectorWaitStates, Union)
from scrapling.core.utils import log, lru_cache
from scrapling.engines.constants import (DEFAULT_STEALTH_FLAGS,
NSTBROWSER_DEFAULT_QUERY)
from scrapling.engines.toolbelt import (Response, StatusText,
async_intercept_route,
check_type_validity, construct_cdp_url,
construct_proxy_dict,
generate_convincing_referer,
generate_headers, intercept_route,
js_bypass_path)
class PlaywrightEngine:
def __init__(
self, headless: Union[bool, str] = True,
disable_resources: bool = False,
useragent: Optional[str] = None,
network_idle: bool = False,
timeout: Optional[float] = 30000,
wait: Optional[int] = 0,
page_action: Callable = None,
wait_selector: Optional[str] = None,
locale: Optional[str] = 'en-US',
wait_selector_state: SelectorWaitStates = 'attached',
stealth: bool = False,
real_chrome: bool = False,
hide_canvas: bool = False,
disable_webgl: bool = False,
cdp_url: Optional[str] = None,
nstbrowser_mode: bool = False,
nstbrowser_config: Optional[Dict] = None,
google_search: bool = True,
extra_headers: Optional[Dict[str, str]] = None,
proxy: Optional[Union[str, Dict[str, str]]] = None,
adaptor_arguments: Dict = None
):
"""An engine that utilizes PlayWright library, check the `PlayWrightFetcher` class for more documentation.
:param headless: Run the browser in headless/hidden (default), or headful/visible mode.
:param disable_resources: Drop requests of unnecessary resources for a speed boost. It depends but it made requests ~25% faster in my tests for some websites.
Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
This can help save your proxy usage but be careful with this option as it makes some websites never finish loading.
:param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it.
:param network_idle: Wait for the page until there are no network connections for at least 500 ms.
:param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30000
:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning `Response` object.
:param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again.
:param wait_selector: Wait for a specific css selector to be in a specific state.
:param locale: Set the locale for the browser if wanted. The default value is `en-US`.
:param wait_selector_state: The state to wait for the selector given with `wait_selector`. Default state is `attached`.
:param stealth: Enables stealth mode, check the documentation to see what stealth mode does currently.
:param real_chrome: If you have chrome browser installed on your device, enable this and the Fetcher will launch an instance of your browser and use it.
:param hide_canvas: Add random noise to canvas operations to prevent fingerprinting.
:param disable_webgl: Disables WebGL and WebGL 2.0 support entirely.
:param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers/NSTBrowser through CDP.
:param nstbrowser_mode: Enables NSTBrowser mode, it have to be used with `cdp_url` argument or it will get completely ignored.
:param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search for this website's domain name.
:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
:param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.
:param nstbrowser_config: The config you want to send with requests to the NSTBrowser. If left empty, Scrapling defaults to an optimized NSTBrowser's docker browserless config.
:param adaptor_arguments: The arguments that will be passed in the end while creating the final Adaptor's class.
"""
self.headless = headless
self.locale = check_type_validity(locale, [str], 'en-US', param_name='locale')
self.disable_resources = disable_resources
self.network_idle = bool(network_idle)
self.stealth = bool(stealth)
self.hide_canvas = bool(hide_canvas)
self.disable_webgl = bool(disable_webgl)
self.real_chrome = bool(real_chrome)
self.google_search = bool(google_search)
self.extra_headers = extra_headers or {}
self.proxy = construct_proxy_dict(proxy)
self.cdp_url = cdp_url
self.useragent = useragent
self.timeout = check_type_validity(timeout, [int, float], 30000)
self.wait = check_type_validity(wait, [int, float], 0)
if page_action is not None:
if callable(page_action):
self.page_action = page_action
else:
self.page_action = None
log.error('[Ignored] Argument "page_action" must be callable')
else:
self.page_action = None
self.wait_selector = wait_selector
self.wait_selector_state = wait_selector_state
self.nstbrowser_mode = bool(nstbrowser_mode)
self.nstbrowser_config = nstbrowser_config
self.adaptor_arguments = adaptor_arguments if adaptor_arguments else {}
self.harmful_default_args = [
# This will be ignored to avoid detection more and possibly avoid the popup crashing bug abuse: https://issues.chromium.org/issues/340836884
'--enable-automation',
'--disable-popup-blocking',
# '--disable-component-update',
# '--disable-default-apps',
# '--disable-extensions',
]
def _cdp_url_logic(self) -> str:
"""Constructs new CDP URL if NSTBrowser is enabled otherwise return CDP URL as it is
:return: CDP URL
"""
cdp_url = self.cdp_url
if self.nstbrowser_mode:
if self.nstbrowser_config and isinstance(self.nstbrowser_config, dict):
config = self.nstbrowser_config
else:
query = NSTBROWSER_DEFAULT_QUERY.copy()
if self.stealth:
flags = self.__set_flags()
query.update({
"args": dict(zip(flags, [''] * len(flags))), # browser args should be a dictionary
})
config = {
'config': json.dumps(query),
# 'token': ''
}
cdp_url = construct_cdp_url(cdp_url, config)
else:
# To validate it
cdp_url = construct_cdp_url(cdp_url)
return cdp_url
@lru_cache(32, typed=True)
def __set_flags(self):
"""Returns the flags that will be used while launching the browser if stealth mode is enabled"""
flags = DEFAULT_STEALTH_FLAGS
if self.hide_canvas:
flags += ('--fingerprinting-canvas-image-data-noise',)
if self.disable_webgl:
flags += ('--disable-webgl', '--disable-webgl-image-chromium', '--disable-webgl2',)
return flags
def __launch_kwargs(self):
"""Creates the arguments we will use while launching playwright's browser"""
launch_kwargs = {'headless': self.headless, 'ignore_default_args': self.harmful_default_args, 'channel': 'chrome' if self.real_chrome else 'chromium'}
if self.stealth:
launch_kwargs.update({'args': self.__set_flags(), 'chromium_sandbox': True})
return launch_kwargs
def __context_kwargs(self):
"""Creates the arguments for the browser context"""
context_kwargs = {
"proxy": self.proxy,
"locale": self.locale,
"color_scheme": 'dark', # Bypasses the 'prefersLightColor' check in creepjs
"device_scale_factor": 2,
"extra_http_headers": self.extra_headers if self.extra_headers else {},
"user_agent": self.useragent if self.useragent else generate_headers(browser_mode=True).get('User-Agent'),
}
if self.stealth:
context_kwargs.update({
'is_mobile': False,
'has_touch': False,
# I'm thinking about disabling it to rest from all Service Workers headache but let's keep it as it is for now
'service_workers': 'allow',
'ignore_https_errors': True,
'screen': {'width': 1920, 'height': 1080},
'viewport': {'width': 1920, 'height': 1080},
'permissions': ['geolocation', 'notifications']
})
return context_kwargs
@lru_cache(1)
def __stealth_scripts(self):
# Basic bypasses nothing fancy as I'm still working on it
# But with adding these bypasses to the above config, it bypasses many online tests like
# https://bot.sannysoft.com/
# https://kaliiiiiiiiii.github.io/brotector/
# https://pixelscan.net/
# https://iphey.com/
# https://www.browserscan.net/bot-detection <== this one also checks for the CDP runtime fingerprint
# https://arh.antoinevastel.com/bots/areyouheadless/
# https://prescience-data.github.io/execution-monitor.html
return tuple(
js_bypass_path(script) for script in (
# Order is important
'webdriver_fully.js', 'window_chrome.js', 'navigator_plugins.js', 'pdf_viewer.js',
'notification_permission.js', 'screen_props.js', 'playwright_fingerprint.js'
)
)
def _process_response_history(self, first_response):
"""Process response history to build a list of Response objects"""
history = []
current_request = first_response.request.redirected_from
try:
while current_request:
try:
current_response = current_request.response()
history.insert(0, Response(
url=current_request.url,
# using current_response.text() will trigger "Error: Response.text: Response body is unavailable for redirect responses"
text='',
body=b'',
status=current_response.status if current_response else 301,
reason=(current_response.status_text or StatusText.get(current_response.status)) if current_response else StatusText.get(301),
encoding=current_response.headers.get('content-type', '') or 'utf-8',
cookies={},
headers=current_response.all_headers() if current_response else {},
request_headers=current_request.all_headers(),
**self.adaptor_arguments
))
except Exception as e:
log.error(f"Error processing redirect: {e}")
break
current_request = current_request.redirected_from
except Exception as e:
log.error(f"Error processing response history: {e}")
return history
async def _async_process_response_history(self, first_response):
"""Process response history to build a list of Response objects"""
history = []
current_request = first_response.request.redirected_from
try:
while current_request:
try:
current_response = await current_request.response()
history.insert(0, Response(
url=current_request.url,
# using current_response.text() will trigger "Error: Response.text: Response body is unavailable for redirect responses"
text='',
body=b'',
status=current_response.status if current_response else 301,
reason=(current_response.status_text or StatusText.get(current_response.status)) if current_response else StatusText.get(301),
encoding=current_response.headers.get('content-type', '') or 'utf-8',
cookies={},
headers=await current_response.all_headers() if current_response else {},
request_headers=await current_request.all_headers(),
**self.adaptor_arguments
))
except Exception as e:
log.error(f"Error processing redirect: {e}")
break
current_request = current_request.redirected_from
except Exception as e:
log.error(f"Error processing response history: {e}")
return history
def fetch(self, url: str) -> Response:
"""Opens up the browser and do your request based on your chosen options.
:param url: Target url.
:return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
"""
from playwright.sync_api import Response as PlaywrightResponse
if not self.stealth or self.real_chrome:
# Because rebrowser_playwright doesn't play well with real browsers
from playwright.sync_api import sync_playwright
else:
from rebrowser_playwright.sync_api import sync_playwright
final_response = None
referer = generate_convincing_referer(url) if self.google_search else None
def handle_response(finished_response: PlaywrightResponse):
nonlocal final_response
if finished_response.request.resource_type == "document" and finished_response.request.is_navigation_request():
final_response = finished_response
with sync_playwright() as p:
# Creating the browser
if self.cdp_url:
cdp_url = self._cdp_url_logic()
browser = p.chromium.connect_over_cdp(endpoint_url=cdp_url)
else:
browser = p.chromium.launch(**self.__launch_kwargs())
context = browser.new_context(**self.__context_kwargs())
page = context.new_page()
page.set_default_navigation_timeout(self.timeout)
page.set_default_timeout(self.timeout)
page.on("response", handle_response)
if self.extra_headers:
page.set_extra_http_headers(self.extra_headers)
if self.disable_resources:
page.route("**/*", intercept_route)
if self.stealth:
for script in self.__stealth_scripts():
page.add_init_script(path=script)
first_response = page.goto(url, referer=referer)
page.wait_for_load_state(state="domcontentloaded")
if self.network_idle:
page.wait_for_load_state('networkidle')
if self.page_action is not None:
try:
page = self.page_action(page)
except Exception as e:
log.error(f"Error executing page_action: {e}")
if self.wait_selector and type(self.wait_selector) is str:
try:
waiter = page.locator(self.wait_selector)
waiter.first.wait_for(state=self.wait_selector_state)
# Wait again after waiting for the selector, helpful with protections like Cloudflare
page.wait_for_load_state(state="load")
page.wait_for_load_state(state="domcontentloaded")
if self.network_idle:
page.wait_for_load_state('networkidle')
except Exception as e:
log.error(f"Error waiting for selector {self.wait_selector}: {e}")
page.wait_for_timeout(self.wait)
# In case we didn't catch a document type somehow
final_response = final_response if final_response else first_response
if not final_response:
raise ValueError("Failed to get a response from the page")
# This will be parsed inside `Response`
encoding = final_response.headers.get('content-type', '') or 'utf-8' # default encoding
# PlayWright API sometimes give empty status text for some reason!
status_text = final_response.status_text or StatusText.get(final_response.status)
history = self._process_response_history(first_response)
try:
page_content = page.content()
except Exception as e:
log.error(f"Error getting page content: {e}")
page_content = ""
response = Response(
url=page.url,
text=page_content,
body=page_content.encode('utf-8'),
status=final_response.status,
reason=status_text,
encoding=encoding,
cookies={cookie['name']: cookie['value'] for cookie in page.context.cookies()},
headers=first_response.all_headers(),
request_headers=first_response.request.all_headers(),
history=history,
**self.adaptor_arguments
)
page.close()
context.close()
return response
async def async_fetch(self, url: str) -> Response:
"""Async version of `fetch`
:param url: Target url.
:return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
"""
from playwright.async_api import Response as PlaywrightResponse
if not self.stealth or self.real_chrome:
# Because rebrowser_playwright doesn't play well with real browsers
from playwright.async_api import async_playwright
else:
from rebrowser_playwright.async_api import async_playwright
final_response = None
referer = generate_convincing_referer(url) if self.google_search else None
async def handle_response(finished_response: PlaywrightResponse):
nonlocal final_response
if finished_response.request.resource_type == "document" and finished_response.request.is_navigation_request():
final_response = finished_response
async with async_playwright() as p:
# Creating the browser
if self.cdp_url:
cdp_url = self._cdp_url_logic()
browser = await p.chromium.connect_over_cdp(endpoint_url=cdp_url)
else:
browser = await p.chromium.launch(**self.__launch_kwargs())
context = await browser.new_context(**self.__context_kwargs())
page = await context.new_page()
page.set_default_navigation_timeout(self.timeout)
page.set_default_timeout(self.timeout)
page.on("response", handle_response)
if self.extra_headers:
await page.set_extra_http_headers(self.extra_headers)
if self.disable_resources:
await page.route("**/*", async_intercept_route)
if self.stealth:
for script in self.__stealth_scripts():
await page.add_init_script(path=script)
first_response = await page.goto(url, referer=referer)
await page.wait_for_load_state(state="domcontentloaded")
if self.network_idle:
await page.wait_for_load_state('networkidle')
if self.page_action is not None:
try:
page = await self.page_action(page)
except Exception as e:
log.error(f"Error executing async page_action: {e}")
if self.wait_selector and type(self.wait_selector) is str:
try:
waiter = page.locator(self.wait_selector)
await waiter.first.wait_for(state=self.wait_selector_state)
# Wait again after waiting for the selector, helpful with protections like Cloudflare
await page.wait_for_load_state(state="load")
await page.wait_for_load_state(state="domcontentloaded")
if self.network_idle:
await page.wait_for_load_state('networkidle')
except Exception as e:
log.error(f"Error waiting for selector {self.wait_selector}: {e}")
await page.wait_for_timeout(self.wait)
# In case we didn't catch a document type somehow
final_response = final_response if final_response else first_response
if not final_response:
raise ValueError("Failed to get a response from the page")
# This will be parsed inside `Response`
encoding = final_response.headers.get('content-type', '') or 'utf-8' # default encoding
# PlayWright API sometimes give empty status text for some reason!
status_text = final_response.status_text or StatusText.get(final_response.status)
history = await self._async_process_response_history(first_response)
try:
page_content = await page.content()
except Exception as e:
log.error(f"Error getting page content in async: {e}")
page_content = ""
response = Response(
url=page.url,
text=page_content,
body=page_content.encode('utf-8'),
status=final_response.status,
reason=status_text,
encoding=encoding,
cookies={cookie['name']: cookie['value'] for cookie in await page.context.cookies()},
headers=await first_response.all_headers(),
request_headers=await first_response.request.all_headers(),
history=history,
**self.adaptor_arguments
)
await page.close()
await context.close()
return response
scrapling/engines/static.py
+671 -114
View File
@@ -1,156 +1,713 @@
import httpx
from httpx._models import Response as httpxResponse
from time import sleep as time_sleep
from asyncio import sleep as asyncio_sleep
from scrapling.core._types import Dict, Optional, Tuple, Union
from scrapling.core.utils import log, lru_cache
from curl_cffi.requests.session import CurlError
from curl_cffi import CurlHttpVersion
from curl_cffi.requests.impersonate import DEFAULT_CHROME
from curl_cffi.requests import (
ProxySpec,
CookieTypes,
BrowserTypeLiteral,
Session as CurlSession,
AsyncSession as AsyncCurlSession,
)
from .toolbelt import Response, generate_convincing_referer, generate_headers
from scrapling.core.utils import log
from scrapling.core._types import (
Dict,
Optional,
Tuple,
Mapping,
SUPPORTED_HTTP_METHODS,
Awaitable,
List,
Any,
)
from .toolbelt import (
Response,
generate_convincing_referer,
generate_headers,
ResponseFactory,
__default_useragent__,
)
_UNSET = object()
@lru_cache(2, typed=True) # Singleton easily
class StaticEngine:
class FetcherSession:
"""
A context manager that provides configured Fetcher sessions.
When this manager is used in a 'with' or 'async with' block,
it yields a new session configured with the manager's defaults.
A single instance of this manager should ideally be used for one active
session at a time (or sequentially). Re-entering a context with the
same manager instance while a session is already active is disallowed.
"""
def __init__(
self, url: str, proxy: Optional[str] = None, stealthy_headers: bool = True, follow_redirects: bool = True,
timeout: Optional[Union[int, float]] = None, retries: Optional[int] = 3, adaptor_arguments: Tuple = None
self,
impersonate: Optional[BrowserTypeLiteral] = DEFAULT_CHROME,
http3: Optional[bool] = False,
stealthy_headers: Optional[bool] = True,
proxies: Optional[Dict[str, str]] = None,
proxy: Optional[str] = None,
proxy_auth: Optional[Tuple[str, str]] = None,
timeout: Optional[int | float] = 30,
headers: Optional[Dict[str, str]] = None,
retries: Optional[int] = 3,
retry_delay: Optional[int] = 1,
follow_redirects: bool = True,
max_redirects: int = 30,
verify: bool = True,
cert: Optional[str | Tuple[str, str]] = None,
selector_config: Optional[Dict] = None,
):
"""An engine that utilizes httpx library, check the `Fetcher` class for more documentation.
:param url: Target url.
:param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers and
create a referer header as if this request had came from Google's search of this URL's domain.
:param proxy: A string of a proxy to use for http and https requests, the format accepted is `http://username:password@localhost:8030`
:param follow_redirects: As the name says -- if enabled (default), redirects will be followed.
:param timeout: The time to wait for the request to finish in seconds. The default is 10 seconds.
:param adaptor_arguments: The arguments that will be passed in the end while creating the final Adaptor's class.
"""
self.url = url
self.proxy = proxy
:param impersonate: Browser version to impersonate. Automatically defaults to the latest available Chrome version.
:param http3: Whether to use HTTP3. Defaults to False. It might be problematic if used it with `impersonate`.
:param stealthy_headers: If enabled (default), it creates and adds real browser headers. It also sets the referer header as if this request came from a Google search of URL's domain.
:param proxies: Dict of proxies to use. Format: {"http": proxy_url, "https": proxy_url}.
:param proxy: Proxy URL to use. Format: "http://username:password@localhost:8030".
Cannot be used together with the `proxies` parameter.
:param proxy_auth: HTTP basic auth for proxy, tuple of (username, password).
:param timeout: Number of seconds to wait before timing out.
:param headers: Headers to include in the session with every request.
:param retries: Number of retry attempts. Defaults to 3.
:param retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second.
:param follow_redirects: Whether to follow redirects. Defaults to True.
:param max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited.
:param verify: Whether to verify HTTPS certificates. Defaults to True.
:param cert: Tuple of (cert, key) filenames for the client certificate.
:param selector_config: Arguments passed when creating the final Selector class.
"""
self.default_impersonate = impersonate
self.stealth = stealthy_headers
self.timeout = timeout
self.follow_redirects = bool(follow_redirects)
self.retries = retries
self._extra_headers = generate_headers(browser_mode=False)
# Because we are using `lru_cache` for a slight optimization but both dict/dict_items are not hashable so they can't be cached
# So my solution here was to convert it to tuple then convert it back to dictionary again here as tuples are hashable, ofc `tuple().__hash__()`
self.adaptor_arguments = dict(adaptor_arguments) if adaptor_arguments else {}
self.default_proxies = proxies or {}
self.default_proxy = proxy or None
self.default_proxy_auth = proxy_auth or None
self.default_timeout = timeout
self.default_headers = headers or {}
self.default_retries = retries
self.default_retry_delay = retry_delay
self.default_follow_redirects = follow_redirects
self.default_max_redirects = max_redirects
self.default_verify = verify
self.default_cert = cert
self.default_http3 = http3
self.selector_config = selector_config or {}
def _headers_job(self, headers: Optional[Dict]) -> Dict:
self._curl_session: Optional[CurlSession] = None
self._async_curl_session: Optional[AsyncCurlSession] = None
def _merge_request_args(self, **kwargs) -> Dict[str, Any]:
"""Merge request-specific arguments with default session arguments."""
url = kwargs.pop("url")
request_args = {}
headers = self.get_with_precedence(kwargs, "headers", self.default_headers)
stealth = self.get_with_precedence(kwargs, "stealth", self.stealth)
impersonate = self.get_with_precedence(
kwargs, "impersonate", self.default_impersonate
)
if self.get_with_precedence(
kwargs, "http3", self.default_http3
): # pragma: no cover
request_args["http_version"] = CurlHttpVersion.V3ONLY
if impersonate:
log.warning(
"The argument `http3` might cause errors if used with `impersonate` argument, try switching it off if you encounter any curl errors."
)
request_args.update(
{
"url": url,
# Curl automatically generates the suitable browser headers when you use `impersonate`
"headers": self._headers_job(url, headers, stealth, bool(impersonate)),
"proxies": self.get_with_precedence(
kwargs, "proxies", self.default_proxies
),
"proxy": self.get_with_precedence(kwargs, "proxy", self.default_proxy),
"proxy_auth": self.get_with_precedence(
kwargs, "proxy_auth", self.default_proxy_auth
),
"timeout": self.get_with_precedence(
kwargs, "timeout", self.default_timeout
),
"allow_redirects": self.get_with_precedence(
kwargs, "allow_redirects", self.default_follow_redirects
),
"max_redirects": self.get_with_precedence(
kwargs, "max_redirects", self.default_max_redirects
),
"verify": self.get_with_precedence(
kwargs, "verify", self.default_verify
),
"cert": self.get_with_precedence(kwargs, "cert", self.default_cert),
"impersonate": impersonate,
**{
k: v
for k, v in kwargs.items()
if v
not in (
_UNSET,
None,
)
}, # Add any remaining parameters (after all known ones are popped)
}
)
return request_args
def _headers_job(
self,
url,
headers: Optional[Dict],
stealth: Optional[bool],
impersonate_enabled: bool,
) -> Dict:
"""Adds useragent to headers if it doesn't exist, generates real headers and append it to current headers, and
finally generates a referer header that looks like if this request came from Google's search of the current URL's domain.
:param headers: Current headers in the request if the user passed any
:param stealth: Whether to enable the `stealthy_headers` argument to this request or not. If `None`, it defaults to the session default value.
:param impersonate_enabled: Whether the browser impersonation is enabled or not.
:return: A dictionary of the new headers.
"""
headers = headers or {}
# Handle headers - if it was _UNSET, use default_headers
if headers is _UNSET:
headers = self.default_headers.copy()
else:
# Merge session headers with request headers, request takes precedence
headers = {**self.default_headers, **(headers or {})}
headers_keys = set(map(str.lower, headers.keys()))
if stealth:
if "referer" not in headers_keys:
headers.update({"referer": generate_convincing_referer(url)})
if impersonate_enabled: # Curl will generate the suitable headers
return headers
if self.stealth:
extra_headers = generate_headers(browser_mode=False)
# Don't overwrite user supplied headers
extra_headers = {key: value for key, value in extra_headers.items() if key.lower() not in headers_keys}
# Don't overwrite user-supplied headers
extra_headers = {
key: value
for key, value in extra_headers.items()
if key.lower() not in headers_keys
}
headers.update(extra_headers)
if 'referer' not in headers_keys:
headers.update({'referer': generate_convincing_referer(self.url)})
elif 'user-agent' not in headers_keys:
headers['User-Agent'] = generate_headers(browser_mode=False).get('User-Agent')
log.debug(f"Can't find useragent in headers so '{headers['User-Agent']}' was used.")
elif "user-agent" not in headers_keys and not impersonate_enabled:
headers["User-Agent"] = __default_useragent__
log.debug(
f"Can't find useragent in headers so '{headers['User-Agent']}' was used."
)
return headers
def _prepare_response(self, response: httpxResponse) -> Response:
"""Takes httpx response and generates `Response` object from it.
def __enter__(self):
"""Creates and returns a new synchronous Fetcher Session"""
if self._curl_session:
raise RuntimeError(
"This FetcherSession instance already has an active synchronous session. "
"Create a new FetcherSession instance for a new independent session, "
"or use the current instance sequentially after the previous context has exited."
)
if (
self._async_curl_session
): # Prevent mixing if async is active from this instance
raise RuntimeError(
"This FetcherSession instance has an active asynchronous session. "
"Cannot enter a synchronous context simultaneously with the same manager instance."
)
:param response: httpx response object
:return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
self._curl_session = CurlSession()
return self
def __exit__(self, exc_type, exc_val, exc_tb):
"""Closes the active synchronous session managed by this instance, if any."""
if self._curl_session:
self._curl_session.close()
self._curl_session = None
async def __aenter__(self):
"""Creates and returns a new asynchronous Session."""
if self._async_curl_session:
raise RuntimeError(
"This FetcherSession instance already has an active asynchronous session. "
"Create a new FetcherSession instance for a new independent session, "
"or use the current instance sequentially after the previous context has exited."
)
if self._curl_session: # Prevent mixing if sync is active from this instance
raise RuntimeError(
"This FetcherSession instance has an active synchronous session. "
"Cannot enter an asynchronous context simultaneously with the same manager instance."
)
self._async_curl_session = AsyncCurlSession()
return self
async def __aexit__(self, exc_type, exc_val, exc_tb):
"""Closes the active asynchronous session managed by this instance, if any."""
if self._async_curl_session:
await self._async_curl_session.close()
self._async_curl_session = None
def __make_request(
self,
method: SUPPORTED_HTTP_METHODS,
request_args: Dict[str, Any],
max_retries: int,
retry_delay: int,
selector_config: Optional[Dict] = None,
) -> Response:
"""
return Response(
url=str(response.url),
text=response.text,
body=response.content,
status=response.status_code,
reason=response.reason_phrase,
encoding=response.encoding or 'utf-8',
cookies=dict(response.cookies),
headers=dict(response.headers),
request_headers=dict(response.request.headers),
method=response.request.method,
history=[self._prepare_response(redirection) for redirection in response.history],
**self.adaptor_arguments
Perform an HTTP request using the configured session.
:param method: HTTP method to be used, supported methods are ["GET", "POST", "PUT", "DELETE"]
:param url: Target URL for the request.
:param request_args: Arguments to be passed to the session's `request()` method.
:param max_retries: Maximum number of retries for the request.
:param retry_delay: Number of seconds to wait between retries.
:param selector_config: Arguments passed when creating the final Selector class.
:return: A `Response` object for synchronous requests or an awaitable for asynchronous.
"""
session = self._curl_session
if session is True and not any(
(self.__enter__, self.__exit__, self.__aenter__, self.__aexit__)
):
# For usage inside FetcherClient
# It turns out `curl_cffi` caches impersonation state, so if you turned it off, then on then off, it won't be off on the last time.
session = CurlSession()
if session:
for attempt in range(max_retries):
try:
response = session.request(method, **request_args)
# response.raise_for_status() # Retry responses with a status code between 200-400
return ResponseFactory.from_http_request(response, selector_config)
except CurlError as e: # pragma: no cover
if attempt < max_retries - 1:
log.error(
f"Attempt {attempt + 1} failed: {e}. Retrying in {retry_delay} seconds..."
)
time_sleep(retry_delay)
else:
log.error(f"Failed after {max_retries} attempts: {e}")
raise # Raise the exception if all retries fail
raise RuntimeError("No active session available.") # pragma: no cover
async def __make_async_request(
self,
method: SUPPORTED_HTTP_METHODS,
request_args: Dict[str, Any],
max_retries: int,
retry_delay: int,
selector_config: Optional[Dict] = None,
) -> Response:
"""
Perform an HTTP request using the configured session.
:param method: HTTP method to be used, supported methods are ["GET", "POST", "PUT", "DELETE"]
:param url: Target URL for the request.
:param request_args: Arguments to be passed to the session's `request()` method.
:param max_retries: Maximum number of retries for the request.
:param retry_delay: Number of seconds to wait between retries.
:param selector_config: Arguments passed when creating the final Selector class.
:return: A `Response` object for synchronous requests or an awaitable for asynchronous.
"""
session = self._async_curl_session
if session is True and not any(
(self.__enter__, self.__exit__, self.__aenter__, self.__aexit__)
):
# For usage inside the ` AsyncFetcherClient ` class, and that's for several reasons
# 1. It turns out `curl_cffi` caches impersonation state, so if you turned it off, then on then off, it won't be off on the last time.
# 2. `curl_cffi` doesn't support making async requests without sessions
# 3. Using a single session for many requests at the same time in async doesn't sit well with curl_cffi.
session = AsyncCurlSession()
if session:
for attempt in range(max_retries):
try:
response = await session.request(method, **request_args)
# response.raise_for_status() # Retry responses with a status code between 200-400
return ResponseFactory.from_http_request(response, selector_config)
except CurlError as e: # pragma: no cover
if attempt < max_retries - 1:
log.error(
f"Attempt {attempt + 1} failed: {e}. Retrying in {retry_delay} seconds..."
)
await asyncio_sleep(retry_delay)
else:
log.error(f"Failed after {max_retries} attempts: {e}")
raise # Raise the exception if all retries fail
raise RuntimeError("No active session available.") # pragma: no cover
@staticmethod
def get_with_precedence(kwargs, key, default_value):
"""Get value with request-level priority over session-level"""
request_value = kwargs.pop(key, _UNSET)
return request_value if request_value is not _UNSET else default_value
def __prepare_and_dispatch(
self,
method: SUPPORTED_HTTP_METHODS,
stealth: Optional[bool] = None,
**kwargs,
) -> Response | Awaitable[Response]:
"""
Internal dispatcher. Prepares arguments and calls sync or async request helper.
:param method: HTTP method to be used, supported methods are ["GET", "POST", "PUT", "DELETE"]
:param stealth: Whether to enable the `stealthy_headers` argument to this request or not. If `None`, it defaults to the session default value.
:param url: Target URL for the request.
:param kwargs: Additional request-specific arguments.
:return: A `Response` object for synchronous requests or an awaitable for asynchronous.
"""
stealth = self.stealth if stealth is None else stealth
selector_config = kwargs.pop("selector_config", {}) or self.selector_config
max_retries = self.get_with_precedence(kwargs, "retries", self.default_retries)
retry_delay = self.get_with_precedence(
kwargs, "retry_delay", self.default_retry_delay
)
request_args = self._merge_request_args(stealth=stealth, **kwargs)
if self._curl_session:
return self.__make_request(
method, request_args, max_retries, retry_delay, selector_config
)
elif self._async_curl_session:
# The returned value is a Coroutine
return self.__make_async_request(
method, request_args, max_retries, retry_delay, selector_config
)
raise RuntimeError("No active session available.")
def get(
self,
url: str,
params: Optional[Dict | List | Tuple] = None,
headers: Optional[Mapping[str, Optional[str]]] = _UNSET,
cookies: Optional[CookieTypes] = None,
timeout: Optional[int | float] = _UNSET,
follow_redirects: Optional[bool] = _UNSET,
max_redirects: Optional[int] = _UNSET,
retries: Optional[int] = _UNSET,
retry_delay: Optional[int] = _UNSET,
proxies: Optional[ProxySpec] = _UNSET,
proxy: Optional[str] = _UNSET,
proxy_auth: Optional[Tuple[str, str]] = _UNSET,
auth: Optional[Tuple[str, str]] = None,
verify: Optional[bool] = _UNSET,
cert: Optional[str | Tuple[str, str]] = _UNSET,
impersonate: Optional[BrowserTypeLiteral] = _UNSET,
http3: Optional[bool] = _UNSET,
stealthy_headers: Optional[bool] = _UNSET,
**kwargs,
) -> Response | Awaitable[Response]:
"""
Perform a GET request.
:param url: Target URL for the request.
:param params: Query string parameters for the request.
:param headers: Headers to include in the request.
:param cookies: Cookies to use in the request.
:param timeout: Number of seconds to wait before timing out.
:param follow_redirects: Whether to follow redirects. Defaults to True.
:param max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited.
:param retries: Number of retry attempts. Defaults to 3.
:param retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second.
:param proxies: Dict of proxies to use.
:param proxy: Proxy URL to use. Format: "http://username:password@localhost:8030".
Cannot be used together with the `proxies` parameter.
:param proxy_auth: HTTP basic auth for proxy, tuple of (username, password).
:param auth: HTTP basic auth tuple of (username, password). Only basic auth is supported.
:param verify: Whether to verify HTTPS certificates.
:param cert: Tuple of (cert, key) filenames for the client certificate.
:param impersonate: Browser version to impersonate. Automatically defaults to the latest available Chrome version.
:param http3: Whether to use HTTP3. Defaults to False. It might be problematic if used it with `impersonate`.
:param stealthy_headers: If enabled (default), it creates and adds real browser headers. It also sets the referer header as if this request came from a Google search of URL's domain.
:param kwargs: Additional keyword arguments to pass to the [`curl_cffi.requests.Session().request()`, `curl_cffi.requests.AsyncSession().request()`] method.
:return: A `Response` object or an awaitable for async.
"""
request_args = {
"url": url,
"params": params,
"headers": headers,
"cookies": cookies,
"timeout": timeout,
"retry_delay": retry_delay,
"allow_redirects": follow_redirects,
"max_redirects": max_redirects,
"retries": retries,
"proxies": proxies,
"proxy": proxy,
"proxy_auth": proxy_auth,
"auth": auth,
"verify": verify,
"cert": cert,
"impersonate": impersonate,
"http3": http3,
**kwargs,
}
return self.__prepare_and_dispatch(
"GET", stealth=stealthy_headers, **request_args
)
def _make_request(self, method: str, **kwargs) -> Response:
headers = self._headers_job(kwargs.pop('headers', {}))
with httpx.Client(proxy=self.proxy, transport=httpx.HTTPTransport(retries=self.retries)) as client:
request = getattr(client, method)(url=self.url, headers=headers, follow_redirects=self.follow_redirects, timeout=self.timeout, **kwargs)
return self._prepare_response(request)
async def _async_make_request(self, method: str, **kwargs) -> Response:
headers = self._headers_job(kwargs.pop('headers', {}))
async with httpx.AsyncClient(proxy=self.proxy, transport=httpx.AsyncHTTPTransport(retries=self.retries)) as client:
request = await getattr(client, method)(url=self.url, headers=headers, follow_redirects=self.follow_redirects, timeout=self.timeout, **kwargs)
return self._prepare_response(request)
def get(self, **kwargs: Dict) -> Response:
"""Make basic HTTP GET request for you but with some added flavors.
:param kwargs: Any keyword arguments are passed directly to `httpx.get()` function so check httpx documentation for details.
:return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
def post(
self,
url: str,
data: Optional[Dict | str] = None,
json: Optional[Dict | List] = None,
headers: Optional[Mapping[str, Optional[str]]] = _UNSET,
params: Optional[Dict | List | Tuple] = None,
cookies: Optional[CookieTypes] = None,
timeout: Optional[int | float] = _UNSET,
follow_redirects: Optional[bool] = _UNSET,
max_redirects: Optional[int] = _UNSET,
retries: Optional[int] = _UNSET,
retry_delay: Optional[int] = _UNSET,
proxies: Optional[ProxySpec] = _UNSET,
proxy: Optional[str] = _UNSET,
proxy_auth: Optional[Tuple[str, str]] = _UNSET,
auth: Optional[Tuple[str, str]] = None,
verify: Optional[bool] = _UNSET,
cert: Optional[str | Tuple[str, str]] = _UNSET,
impersonate: Optional[BrowserTypeLiteral] = _UNSET,
http3: Optional[bool] = _UNSET,
stealthy_headers: Optional[bool] = _UNSET,
**kwargs,
) -> Response | Awaitable[Response]:
"""
return self._make_request('get', **kwargs)
Perform a POST request.
async def async_get(self, **kwargs: Dict) -> Response:
"""Make basic async HTTP GET request for you but with some added flavors.
:param kwargs: Any keyword arguments are passed directly to `httpx.get()` function so check httpx documentation for details.
:return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
:param url: Target URL for the request.
:param data: Form data to include in the request body.
:param json: A JSON serializable object to include in the body of the request.
:param headers: Headers to include in the request.
:param params: Query string parameters for the request.
:param cookies: Cookies to use in the request.
:param timeout: Number of seconds to wait before timing out.
:param follow_redirects: Whether to follow redirects. Defaults to True.
:param max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited.
:param retries: Number of retry attempts. Defaults to 3.
:param retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second.
:param proxies: Dict of proxies to use. Format: {"http": proxy_url, "https": proxy_url}.
:param proxy: Proxy URL to use. Format: "http://username:password@localhost:8030".
Cannot be used together with the `proxies` parameter.
:param proxy_auth: HTTP basic auth for proxy, tuple of (username, password).
:param auth: HTTP basic auth tuple of (username, password). Only basic auth is supported.
:param verify: Whether to verify HTTPS certificates. Defaults to True.
:param cert: Tuple of (cert, key) filenames for the client certificate.
:param impersonate: Browser version to impersonate. Automatically defaults to the latest available Chrome version.
:param http3: Whether to use HTTP3. Defaults to False. It might be problematic if used it with `impersonate`.
:param stealthy_headers: If enabled (default), it creates and adds real browser headers. It also sets the referer header as if this request came from a Google search of URL's domain.
:param kwargs: Additional keyword arguments to pass to the [`curl_cffi.requests.Session().request()`, `curl_cffi.requests.AsyncSession().request()`] method.
:return: A `Response` object or an awaitable for async.
"""
return await self._async_make_request('get', **kwargs)
request_args = {
"url": url,
"data": data,
"json": json,
"headers": headers,
"params": params,
"cookies": cookies,
"timeout": timeout,
"retry_delay": retry_delay,
"proxy": proxy,
"impersonate": impersonate,
"allow_redirects": follow_redirects,
"max_redirects": max_redirects,
"retries": retries,
"proxies": proxies,
"proxy_auth": proxy_auth,
"auth": auth,
"verify": verify,
"cert": cert,
"http3": http3,
**kwargs,
}
return self.__prepare_and_dispatch(
"POST", stealth=stealthy_headers, **request_args
)
def post(self, **kwargs: Dict) -> Response:
"""Make basic HTTP POST request for you but with some added flavors.
:param kwargs: Any keyword arguments are passed directly to `httpx.post()` function so check httpx documentation for details.
:return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
def put(
self,
url: str,
data: Optional[Dict | str] = None,
json: Optional[Dict | List] = None,
headers: Optional[Mapping[str, Optional[str]]] = _UNSET,
params: Optional[Dict | List | Tuple] = None,
cookies: Optional[CookieTypes] = None,
timeout: Optional[int | float] = _UNSET,
follow_redirects: Optional[bool] = _UNSET,
max_redirects: Optional[int] = _UNSET,
retries: Optional[int] = _UNSET,
retry_delay: Optional[int] = _UNSET,
proxies: Optional[ProxySpec] = _UNSET,
proxy: Optional[str] = _UNSET,
proxy_auth: Optional[Tuple[str, str]] = _UNSET,
auth: Optional[Tuple[str, str]] = None,
verify: Optional[bool] = _UNSET,
cert: Optional[str | Tuple[str, str]] = _UNSET,
impersonate: Optional[BrowserTypeLiteral] = _UNSET,
http3: Optional[bool] = _UNSET,
stealthy_headers: Optional[bool] = _UNSET,
**kwargs,
) -> Response | Awaitable[Response]:
"""
return self._make_request('post', **kwargs)
Perform a PUT request.
async def async_post(self, **kwargs: Dict) -> Response:
"""Make basic async HTTP POST request for you but with some added flavors.
:param kwargs: Any keyword arguments are passed directly to `httpx.post()` function so check httpx documentation for details.
:return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
:param url: Target URL for the request.
:param data: Form data to include in the request body.
:param json: A JSON serializable object to include in the body of the request.
:param headers: Headers to include in the request.
:param params: Query string parameters for the request.
:param cookies: Cookies to use in the request.
:param timeout: Number of seconds to wait before timing out.
:param follow_redirects: Whether to follow redirects. Defaults to True.
:param max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited.
:param retries: Number of retry attempts. Defaults to 3.
:param retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second.
:param proxies: Dict of proxies to use. Format: {"http": proxy_url, "https": proxy_url}.
:param proxy: Proxy URL to use. Format: "http://username:password@localhost:8030".
Cannot be used together with the `proxies` parameter.
:param proxy_auth: HTTP basic auth for proxy, tuple of (username, password).
:param auth: HTTP basic auth tuple of (username, password). Only basic auth is supported.
:param verify: Whether to verify HTTPS certificates. Defaults to True.
:param cert: Tuple of (cert, key) filenames for the client certificate.
:param impersonate: Browser version to impersonate. Automatically defaults to the latest available Chrome version.
:param http3: Whether to use HTTP3. Defaults to False. It might be problematic if used it with `impersonate`.
:param stealthy_headers: If enabled (default), it creates and adds real browser headers. It also sets the referer header as if this request came from a Google search of URL's domain.
:param kwargs: Additional keyword arguments to pass to the [`curl_cffi.requests.Session().request()`, `curl_cffi.requests.AsyncSession().request()`] method.
:return: A `Response` object or an awaitable for async.
"""
return await self._async_make_request('post', **kwargs)
request_args = {
"url": url,
"data": data,
"json": json,
"headers": headers,
"params": params,
"cookies": cookies,
"timeout": timeout,
"retry_delay": retry_delay,
"proxy": proxy,
"impersonate": impersonate,
"allow_redirects": follow_redirects,
"max_redirects": max_redirects,
"retries": retries,
"proxies": proxies,
"proxy_auth": proxy_auth,
"auth": auth,
"verify": verify,
"cert": cert,
"http3": http3,
**kwargs,
}
return self.__prepare_and_dispatch(
"PUT", stealth=stealthy_headers, **request_args
)
def delete(self, **kwargs: Dict) -> Response:
"""Make basic HTTP DELETE request for you but with some added flavors.
:param kwargs: Any keyword arguments are passed directly to `httpx.delete()` function so check httpx documentation for details.
:return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
def delete(
self,
url: str,
data: Optional[Dict | str] = None,
json: Optional[Dict | List] = None,
headers: Optional[Mapping[str, Optional[str]]] = _UNSET,
params: Optional[Dict | List | Tuple] = None,
cookies: Optional[CookieTypes] = None,
timeout: Optional[int | float] = _UNSET,
follow_redirects: Optional[bool] = _UNSET,
max_redirects: Optional[int] = _UNSET,
retries: Optional[int] = _UNSET,
retry_delay: Optional[int] = _UNSET,
proxies: Optional[ProxySpec] = _UNSET,
proxy: Optional[str] = _UNSET,
proxy_auth: Optional[Tuple[str, str]] = _UNSET,
auth: Optional[Tuple[str, str]] = None,
verify: Optional[bool] = _UNSET,
cert: Optional[str | Tuple[str, str]] = _UNSET,
impersonate: Optional[BrowserTypeLiteral] = _UNSET,
http3: Optional[bool] = _UNSET,
stealthy_headers: Optional[bool] = _UNSET,
**kwargs,
) -> Response | Awaitable[Response]:
"""
return self._make_request('delete', **kwargs)
Perform a DELETE request.
async def async_delete(self, **kwargs: Dict) -> Response:
"""Make basic async HTTP DELETE request for you but with some added flavors.
:param kwargs: Any keyword arguments are passed directly to `httpx.delete()` function so check httpx documentation for details.
:return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
:param url: Target URL for the request.
:param data: Form data to include in the request body.
:param json: A JSON serializable object to include in the body of the request.
:param headers: Headers to include in the request.
:param params: Query string parameters for the request.
:param cookies: Cookies to use in the request.
:param timeout: Number of seconds to wait before timing out.
:param follow_redirects: Whether to follow redirects. Defaults to True.
:param max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited.
:param retries: Number of retry attempts. Defaults to 3.
:param retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second.
:param proxies: Dict of proxies to use. Format: {"http": proxy_url, "https": proxy_url}.
:param proxy: Proxy URL to use. Format: "http://username:password@localhost:8030".
Cannot be used together with the `proxies` parameter.
:param proxy_auth: HTTP basic auth for proxy, tuple of (username, password).
:param auth: HTTP basic auth tuple of (username, password). Only basic auth is supported.
:param verify: Whether to verify HTTPS certificates. Defaults to True.
:param cert: Tuple of (cert, key) filenames for the client certificate.
:param impersonate: Browser version to impersonate. Automatically defaults to the latest available Chrome version.
:param http3: Whether to use HTTP3. Defaults to False. It might be problematic if used it with `impersonate`.
:param stealthy_headers: If enabled (default), it creates and adds real browser headers. It also sets the referer header as if this request came from a Google search of URL's domain.
:param kwargs: Additional keyword arguments to pass to the [`curl_cffi.requests.Session().request()`, `curl_cffi.requests.AsyncSession().request()`] method.
:return: A `Response` object or an awaitable for async.
"""
return await self._async_make_request('delete', **kwargs)
request_args = {
"url": url,
# Careful of sending a body in a DELETE request, it might cause some websites to reject the request as per https://www.rfc-editor.org/rfc/rfc7231#section-4.3.5,
# But some websites accept it, it depends on the implementation used.
"data": data,
"json": json,
"headers": headers,
"params": params,
"cookies": cookies,
"timeout": timeout,
"retry_delay": retry_delay,
"proxy": proxy,
"impersonate": impersonate,
"allow_redirects": follow_redirects,
"max_redirects": max_redirects,
"retries": retries,
"proxies": proxies,
"proxy_auth": proxy_auth,
"auth": auth,
"verify": verify,
"cert": cert,
"http3": http3,
**kwargs,
}
return self.__prepare_and_dispatch(
"DELETE", stealth=stealthy_headers, **request_args
)
def put(self, **kwargs: Dict) -> Response:
"""Make basic HTTP PUT request for you but with some added flavors.
:param kwargs: Any keyword arguments are passed directly to `httpx.put()` function so check httpx documentation for details.
:return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
"""
return self._make_request('put', **kwargs)
class FetcherClient(FetcherSession):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.__enter__ = None
self.__exit__ = None
self.__aenter__ = None
self.__aexit__ = None
self._curl_session = True
async def async_put(self, **kwargs: Dict) -> Response:
"""Make basic async HTTP PUT request for you but with some added flavors.
:param kwargs: Any keyword arguments are passed directly to `httpx.put()` function so check httpx documentation for details.
:return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
"""
return await self._async_make_request('put', **kwargs)
class AsyncFetcherClient(FetcherSession):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.__enter__ = None
self.__exit__ = None
self.__aenter__ = None
self.__aexit__ = None
self._async_curl_session = True
scrapling/engines/toolbelt/__init__.py
+20 -6
View File
@@ -1,6 +1,20 @@
from .custom import (BaseFetcher, Response, StatusText, check_if_engine_usable,
check_type_validity, get_variable_name)
from .fingerprints import (generate_convincing_referer, generate_headers,
get_os_name)
from .navigation import (async_intercept_route, construct_cdp_url,
construct_proxy_dict, intercept_route, js_bypass_path)
from .custom import (
BaseFetcher,
Response,
StatusText,
get_variable_name,
)
from .fingerprints import (
generate_convincing_referer,
generate_headers,
get_os_name,
__default_useragent__,
)
from .navigation import (
async_intercept_route,
construct_cdp_url,
construct_proxy_dict,
intercept_route,
js_bypass_path,
)
from .convertor import ResponseFactory
scrapling/engines/toolbelt/bypasses/pdf_viewer.js
-5
View File
@@ -1,5 +0,0 @@
// PDF viewer enabled
// Bypasses `pdfIsDisabled` test in creepsjs's 'Like Headless' sections
Object.defineProperty(navigator, 'pdfViewerEnabled', {
get: () => true,
});
scrapling/engines/toolbelt/bypasses/playwright_fingerprint.js
+2 -1
View File
@@ -1,2 +1,3 @@
// Remove playwright fingerprint => https://github.com/microsoft/playwright/commit/c9e673c6dca746384338ab6bb0cf63c7e7caa9b2#diff-087773eea292da9db5a3f27de8f1a2940cdb895383ad750c3cd8e01772a35b40R915
delete __pwInitScripts;
delete window.__pwInitScripts;
delete window.__playwright__binding__;
scrapling/engines/toolbelt/convertor.py
+254
View File
@@ -0,0 +1,254 @@
from curl_cffi.requests import Response as CurlResponse
from playwright.sync_api import Page as SyncPage, Response as SyncResponse
from playwright.async_api import Page as AsyncPage, Response as AsyncResponse
from scrapling.core.utils import log
from scrapling.core._types import Dict, Optional
from .custom import Response, StatusText
class ResponseFactory:
"""
Factory class for creating `Response` objects from various sources.
This class provides multiple static and instance methods for building standardized `Response` objects
from diverse input sources such as Playwright responses, asynchronous Playwright responses,
and raw HTTP request responses. It supports handling response histories, constructing the proper
response objects, and managing encoding, headers, cookies, and other attributes.
"""
@classmethod
def _process_response_history(
cls, first_response: SyncResponse, parser_arguments: Dict
) -> list[Response]:
"""Process response history to build a list of `Response` objects"""
history = []
current_request = first_response.request.redirected_from
try:
while current_request:
try:
current_response = current_request.response()
history.insert(
0,
Response(
url=current_request.url,
# using current_response.text() will trigger "Error: Response.text: Response body is unavailable for redirect responses"
content="",
status=current_response.status if current_response else 301,
reason=(
current_response.status_text
or StatusText.get(current_response.status)
)
if current_response
else StatusText.get(301),
encoding=current_response.headers.get("content-type", "")
or "utf-8",
cookies=tuple(),
headers=current_response.all_headers()
if current_response
else {},
request_headers=current_request.all_headers(),
**parser_arguments,
),
)
except Exception as e: # pragma: no cover
log.error(f"Error processing redirect: {e}")
break
current_request = current_request.redirected_from
except Exception as e: # pragma: no cover
log.error(f"Error processing response history: {e}")
return history
@classmethod
def from_playwright_response(
cls,
page: SyncPage,
first_response: SyncResponse,
final_response: Optional[SyncResponse],
parser_arguments: Dict,
) -> Response:
"""
Transforms a Playwright response into an internal `Response` object, encapsulating
the page's content, response status, headers, and relevant metadata.
The function handles potential issues, such as empty or missing final responses,
by falling back to the first response if necessary. Encoding and status text
are also derived from the provided response headers or reasonable defaults.
Additionally, the page content and cookies are extracted for further use.
:param page: A synchronous Playwright `Page` instance that represents the current browser page. Required to retrieve the page's URL, cookies, and content.
:param final_response: The last response received for the given request from the Playwright instance. Typically used as the main response object to derive status, headers, and other metadata.
:param first_response: An earlier or initial Playwright `Response` object that may serve as a fallback response in the absence of the final one.
:param parser_arguments: A dictionary containing additional arguments needed for parsing or further customization of the returned `Response`. These arguments are dynamically unpacked into
the `Response` object.
:return: A fully populated `Response` object containing the page's URL, content, status, headers, cookies, and other derived metadata.
:rtype: Response
"""
# In case we didn't catch a document type somehow
final_response = final_response if final_response else first_response
if not final_response:
raise ValueError("Failed to get a response from the page")
# This will be parsed inside `Response`
encoding = (
final_response.headers.get("content-type", "") or "utf-8"
) # default encoding
# PlayWright API sometimes give empty status text for some reason!
status_text = final_response.status_text or StatusText.get(
final_response.status
)
history = cls._process_response_history(first_response, parser_arguments)
try:
page_content = page.content()
except Exception as e: # pragma: no cover
log.error(f"Error getting page content: {e}")
page_content = ""
return Response(
url=page.url,
content=page_content,
status=final_response.status,
reason=status_text,
encoding=encoding,
cookies=tuple(dict(cookie) for cookie in page.context.cookies()),
headers=first_response.all_headers(),
request_headers=first_response.request.all_headers(),
history=history,
**parser_arguments,
)
@classmethod
async def _async_process_response_history(
cls, first_response: AsyncResponse, parser_arguments: Dict
) -> list[Response]:
"""Process response history to build a list of `Response` objects"""
history = []
current_request = first_response.request.redirected_from
try:
while current_request:
try:
current_response = await current_request.response()
history.insert(
0,
Response(
url=current_request.url,
# using current_response.text() will trigger "Error: Response.text: Response body is unavailable for redirect responses"
content="",
status=current_response.status if current_response else 301,
reason=(
current_response.status_text
or StatusText.get(current_response.status)
)
if current_response
else StatusText.get(301),
encoding=current_response.headers.get("content-type", "")
or "utf-8",
cookies=tuple(),
headers=await current_response.all_headers()
if current_response
else {},
request_headers=await current_request.all_headers(),
**parser_arguments,
),
)
except Exception as e: # pragma: no cover
log.error(f"Error processing redirect: {e}")
break
current_request = current_request.redirected_from
except Exception as e: # pragma: no cover
log.error(f"Error processing response history: {e}")
return history
@classmethod
async def from_async_playwright_response(
cls,
page: AsyncPage,
first_response: AsyncResponse,
final_response: Optional[AsyncResponse],
parser_arguments: Dict,
) -> Response:
"""
Transforms a Playwright response into an internal `Response` object, encapsulating
the page's content, response status, headers, and relevant metadata.
The function handles potential issues, such as empty or missing final responses,
by falling back to the first response if necessary. Encoding and status text
are also derived from the provided response headers or reasonable defaults.
Additionally, the page content and cookies are extracted for further use.
:param page: An asynchronous Playwright `Page` instance that represents the current browser page. Required to retrieve the page's URL, cookies, and content.
:param final_response: The last response received for the given request from the Playwright instance. Typically used as the main response object to derive status, headers, and other metadata.
:param first_response: An earlier or initial Playwright `Response` object that may serve as a fallback response in the absence of the final one.
:param parser_arguments: A dictionary containing additional arguments needed for parsing or further customization of the returned `Response`. These arguments are dynamically unpacked into
the `Response` object.
:return: A fully populated `Response` object containing the page's URL, content, status, headers, cookies, and other derived metadata.
:rtype: Response
"""
# In case we didn't catch a document type somehow
final_response = final_response if final_response else first_response
if not final_response:
raise ValueError("Failed to get a response from the page")
# This will be parsed inside `Response`
encoding = (
final_response.headers.get("content-type", "") or "utf-8"
) # default encoding
# PlayWright API sometimes give empty status text for some reason!
status_text = final_response.status_text or StatusText.get(
final_response.status
)
history = await cls._async_process_response_history(
first_response, parser_arguments
)
try:
page_content = await page.content()
except Exception as e: # pragma: no cover
log.error(f"Error getting page content in async: {e}")
page_content = ""
return Response(
url=page.url,
content=page_content,
status=final_response.status,
reason=status_text,
encoding=encoding,
cookies=tuple(dict(cookie) for cookie in await page.context.cookies()),
headers=await first_response.all_headers(),
request_headers=await first_response.request.all_headers(),
history=history,
**parser_arguments,
)
@staticmethod
def from_http_request(response: CurlResponse, parser_arguments: Dict) -> Response:
"""Takes `curl_cffi` response and generates `Response` object from it.
:param response: `curl_cffi` response object
:param parser_arguments: Additional arguments to be passed to the `Response` object constructor.
:return: A `Response` object that is the same as `Selector` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
"""
return Response(
url=response.url,
content=response.content
if isinstance(response.content, bytes)
else response.content.encode(),
status=response.status_code,
reason=response.reason,
encoding=response.encoding or "utf-8",
cookies=dict(response.cookies),
headers=dict(response.headers),
request_headers=dict(response.request.headers),
method=response.request.method,
history=response.history, # https://github.com/lexiforest/curl_cffi/issues/82
**parser_arguments,
)
scrapling/engines/toolbelt/custom.py
+158 -175
View File
@@ -1,19 +1,29 @@
"""
Functions related to custom types or type checking
"""
import inspect
from email.message import Message
from scrapling.core._types import (Any, Callable, Dict, List, Optional, Tuple,
Type, Union)
from scrapling.core._types import (
Any,
Dict,
List,
Optional,
Tuple,
)
from scrapling.core.custom_types import MappingProxyType
from scrapling.core.utils import log, lru_cache
from scrapling.parser import Adaptor, SQLiteStorageSystem
from scrapling.parser import Selector, SQLiteStorageSystem
class ResponseEncoding:
__DEFAULT_ENCODING = "utf-8"
__ISO_8859_1_CONTENT_TYPES = {"text/plain", "text/html", "text/css", "text/javascript"}
__ISO_8859_1_CONTENT_TYPES = {
"text/plain",
"text/html",
"text/css",
"text/javascript",
}
@classmethod
@lru_cache(maxsize=128)
@@ -27,19 +37,21 @@ class ResponseEncoding:
"""
# Create a Message object and set the Content-Type header then get the content type and parameters
msg = Message()
msg['content-type'] = header_value
msg["content-type"] = header_value
content_type = msg.get_content_type()
params = dict(msg.get_params(failobj=[]))
# Remove the content-type from params if present somehow
params.pop('content-type', None)
params.pop("content-type", None)
return content_type, params
@classmethod
@lru_cache(maxsize=128)
def get_value(cls, content_type: Optional[str], text: Optional[str] = 'test') -> str:
def get_value(
cls, content_type: Optional[str], text: Optional[str] = "test"
) -> str:
"""Determine the appropriate character encoding from a content-type header.
The encoding is determined by these rules in order:
@@ -72,7 +84,9 @@ class ResponseEncoding:
encoding = cls.__DEFAULT_ENCODING
if encoding:
_ = text.encode(encoding) # Validate encoding and validate it can encode the given text
_ = text.encode(
encoding
) # Validate encoding and validate it can encode the given text
return encoding
return cls.__DEFAULT_ENCODING
@@ -81,48 +95,74 @@ class ResponseEncoding:
return cls.__DEFAULT_ENCODING
class Response(Adaptor):
class Response(Selector):
"""This class is returned by all engines as a way to unify response type between different libraries."""
def __init__(self, url: str, text: str, body: bytes, status: int, reason: str, cookies: Dict, headers: Dict, request_headers: Dict,
encoding: str = 'utf-8', method: str = 'GET', history: List = None, **adaptor_arguments: Dict):
automatch_domain = adaptor_arguments.pop('automatch_domain', None)
def __init__(
self,
url: str,
content: str | bytes,
status: int,
reason: str,
cookies: Tuple[Dict[str, str], ...] | Dict[str, str],
headers: Dict,
request_headers: Dict,
encoding: str = "utf-8",
method: str = "GET",
history: List = None,
**selector_config: Dict,
):
adaptive_domain = selector_config.pop("adaptive_domain", None)
self.status = status
self.reason = reason
self.cookies = cookies
self.headers = headers
self.request_headers = request_headers
self.history = history or []
encoding = ResponseEncoding.get_value(encoding, text)
super().__init__(text=text, body=body, url=automatch_domain or url, encoding=encoding, **adaptor_arguments)
# For back-ward compatibility
self.adaptor = self
encoding = ResponseEncoding.get_value(
encoding, content.decode("utf-8") if isinstance(content, bytes) else content
)
super().__init__(
content=content,
url=adaptive_domain or url,
encoding=encoding,
**selector_config,
)
# For easier debugging while working from a Python shell
log.info(f'Fetched ({status}) <{method} {url}> (referer: {request_headers.get("referer")})')
# def __repr__(self):
# return f'<{self.__class__.__name__} [{self.status} {self.reason}]>'
log.info(
f"Fetched ({status}) <{method} {url}> (referer: {request_headers.get('referer')})"
)
class BaseFetcher:
__slots__ = ()
huge_tree: bool = True
auto_match: Optional[bool] = False
adaptive: Optional[bool] = False
storage: Any = SQLiteStorageSystem
keep_cdata: Optional[bool] = False
storage_args: Optional[Dict] = None
keep_comments: Optional[bool] = False
automatch_domain: Optional[str] = None
parser_keywords: Tuple = ('huge_tree', 'auto_match', 'storage', 'keep_cdata', 'storage_args', 'keep_comments', 'automatch_domain',) # Left open for the user
adaptive_domain: Optional[str] = None
parser_keywords: Tuple = (
"huge_tree",
"adaptive",
"storage",
"keep_cdata",
"storage_args",
"keep_comments",
"adaptive_domain",
) # Left open for the user
def __init__(self, *args, **kwargs):
# For backward-compatibility before 0.2.99
args_str = ", ".join(args) or ''
kwargs_str = ", ".join(f'{k}={v}' for k, v in kwargs.items()) or ''
args_str = ", ".join(args) or ""
kwargs_str = ", ".join(f"{k}={v}" for k, v in kwargs.items()) or ""
if args_str:
args_str += ', '
args_str += ", "
log.warning(f'This logic is deprecated now, and have no effect; It will be removed with v0.3. Use `{self.__class__.__name__}.configure({args_str}{kwargs_str})` instead before fetching')
log.warning(
f"This logic is deprecated now, and have no effect; It will be removed with v0.3. Use `{self.__class__.__name__}.configure({args_str}{kwargs_str})` instead before fetching"
)
pass
@classmethod
@@ -131,17 +171,17 @@ class BaseFetcher:
huge_tree=cls.huge_tree,
keep_comments=cls.keep_comments,
keep_cdata=cls.keep_cdata,
auto_match=cls.auto_match,
adaptive=cls.adaptive,
storage=cls.storage,
storage_args=cls.storage_args,
automatch_domain=cls.automatch_domain,
adaptive_domain=cls.adaptive_domain,
)
@classmethod
def configure(cls, **kwargs):
"""Set multiple arguments for the parser at once globally
:param kwargs: The keywords can be any arguments of the following: huge_tree, keep_comments, keep_cdata, auto_match, storage, storage_args, automatch_domain
:param kwargs: The keywords can be any arguments of the following: huge_tree, keep_comments, keep_cdata, adaptive, storage, storage_args, adaptive_domain
"""
for key, value in kwargs.items():
key = key.strip().lower()
@@ -150,30 +190,38 @@ class BaseFetcher:
setattr(cls, key, value)
else:
# Yup, no fun allowed LOL
raise AttributeError(f'Unknown parser argument: "{key}"; maybe you meant {cls.parser_keywords}?')
raise AttributeError(
f'Unknown parser argument: "{key}"; maybe you meant {cls.parser_keywords}?'
)
else:
raise ValueError(f'Unknown parser argument: "{key}"; maybe you meant {cls.parser_keywords}?')
raise ValueError(
f'Unknown parser argument: "{key}"; maybe you meant {cls.parser_keywords}?'
)
if not kwargs:
raise AttributeError(f'You must pass a keyword to configure, current keywords: {cls.parser_keywords}?')
raise AttributeError(
f"You must pass a keyword to configure, current keywords: {cls.parser_keywords}?"
)
@classmethod
def _generate_parser_arguments(cls) -> Dict:
# Adaptor class parameters
# I won't validate Adaptor's class parameters here again, I will leave it to be validated later
# Selector class parameters
# I won't validate Selector's class parameters here again, I will leave it to be validated later
parser_arguments = dict(
huge_tree=cls.huge_tree,
keep_comments=cls.keep_comments,
keep_cdata=cls.keep_cdata,
auto_match=cls.auto_match,
adaptive=cls.adaptive,
storage=cls.storage,
storage_args=cls.storage_args
storage_args=cls.storage_args,
)
if cls.automatch_domain:
if type(cls.automatch_domain) is not str:
log.warning('[Ignored] The argument "automatch_domain" must be of string type')
if cls.adaptive_domain:
if not isinstance(cls.adaptive_domain, str):
log.warning(
'[Ignored] The argument "adaptive_domain" must be of string type'
)
else:
parser_arguments.update({'automatch_domain': cls.automatch_domain})
parser_arguments.update({"adaptive_domain": cls.adaptive_domain})
return parser_arguments
@@ -181,72 +229,75 @@ class BaseFetcher:
class StatusText:
"""A class that gets the status text of response status code.
Reference: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status
Reference: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status
"""
_phrases = MappingProxyType({
100: "Continue",
101: "Switching Protocols",
102: "Processing",
103: "Early Hints",
200: "OK",
201: "Created",
202: "Accepted",
203: "Non-Authoritative Information",
204: "No Content",
205: "Reset Content",
206: "Partial Content",
207: "Multi-Status",
208: "Already Reported",
226: "IM Used",
300: "Multiple Choices",
301: "Moved Permanently",
302: "Found",
303: "See Other",
304: "Not Modified",
305: "Use Proxy",
307: "Temporary Redirect",
308: "Permanent Redirect",
400: "Bad Request",
401: "Unauthorized",
402: "Payment Required",
403: "Forbidden",
404: "Not Found",
405: "Method Not Allowed",
406: "Not Acceptable",
407: "Proxy Authentication Required",
408: "Request Timeout",
409: "Conflict",
410: "Gone",
411: "Length Required",
412: "Precondition Failed",
413: "Payload Too Large",
414: "URI Too Long",
415: "Unsupported Media Type",
416: "Range Not Satisfiable",
417: "Expectation Failed",
418: "I'm a teapot",
421: "Misdirected Request",
422: "Unprocessable Entity",
423: "Locked",
424: "Failed Dependency",
425: "Too Early",
426: "Upgrade Required",
428: "Precondition Required",
429: "Too Many Requests",
431: "Request Header Fields Too Large",
451: "Unavailable For Legal Reasons",
500: "Internal Server Error",
501: "Not Implemented",
502: "Bad Gateway",
503: "Service Unavailable",
504: "Gateway Timeout",
505: "HTTP Version Not Supported",
506: "Variant Also Negotiates",
507: "Insufficient Storage",
508: "Loop Detected",
510: "Not Extended",
511: "Network Authentication Required"
})
_phrases = MappingProxyType(
{
100: "Continue",
101: "Switching Protocols",
102: "Processing",
103: "Early Hints",
200: "OK",
201: "Created",
202: "Accepted",
203: "Non-Authoritative Information",
204: "No Content",
205: "Reset Content",
206: "Partial Content",
207: "Multi-Status",
208: "Already Reported",
226: "IM Used",
300: "Multiple Choices",
301: "Moved Permanently",
302: "Found",
303: "See Other",
304: "Not Modified",
305: "Use Proxy",
307: "Temporary Redirect",
308: "Permanent Redirect",
400: "Bad Request",
401: "Unauthorized",
402: "Payment Required",
403: "Forbidden",
404: "Not Found",
405: "Method Not Allowed",
406: "Not Acceptable",
407: "Proxy Authentication Required",
408: "Request Timeout",
409: "Conflict",
410: "Gone",
411: "Length Required",
412: "Precondition Failed",
413: "Payload Too Large",
414: "URI Too Long",
415: "Unsupported Media Type",
416: "Range Not Satisfiable",
417: "Expectation Failed",
418: "I'm a teapot",
421: "Misdirected Request",
422: "Unprocessable Entity",
423: "Locked",
424: "Failed Dependency",
425: "Too Early",
426: "Upgrade Required",
428: "Precondition Required",
429: "Too Many Requests",
431: "Request Header Fields Too Large",
451: "Unavailable For Legal Reasons",
500: "Internal Server Error",
501: "Not Implemented",
502: "Bad Gateway",
503: "Service Unavailable",
504: "Gateway Timeout",
505: "HTTP Version Not Supported",
506: "Variant Also Negotiates",
507: "Insufficient Storage",
508: "Loop Detected",
510: "Not Extended",
511: "Network Authentication Required",
}
)
@classmethod
@lru_cache(maxsize=128)
@@ -255,32 +306,6 @@ class StatusText:
return cls._phrases.get(status_code, "Unknown Status Code")
def check_if_engine_usable(engine: Callable) -> Union[Callable, None]:
"""This function check if the passed engine can be used by a Fetcher-type class or not.
:param engine: The engine class itself
:return: The engine class again if all checks out, otherwise raises error
:raise TypeError: If engine class don't have fetch method, If engine class have fetch attribute not method, or If engine class have fetch function but it doesn't take arguments
"""
# if isinstance(engine, type):
# raise TypeError("Expected an engine instance, not a class definition of the engine")
if hasattr(engine, 'fetch'):
fetch_function = getattr(engine, "fetch")
if callable(fetch_function):
if len(inspect.signature(fetch_function).parameters) > 0:
return engine
else:
# raise TypeError("Engine class instance must have a callable method 'fetch' with the first argument used for the url.")
raise TypeError("Engine class must have a callable method 'fetch' with the first argument used for the url.")
else:
# raise TypeError("Invalid engine instance! Engine class must have a callable method 'fetch'")
raise TypeError("Invalid engine class! Engine class must have a callable method 'fetch'")
else:
# raise TypeError("Invalid engine instance! Engine class must have the method 'fetch'")
raise TypeError("Invalid engine class! Engine class must have the method 'fetch'")
def get_variable_name(var: Any) -> Optional[str]:
"""Get the name of a variable using global and local scopes.
:param var: The variable to find the name for
@@ -291,45 +316,3 @@ def get_variable_name(var: Any) -> Optional[str]:
if value is var:
return name
return None
def check_type_validity(variable: Any, valid_types: Union[List[Type], None], default_value: Any = None, critical: bool = False, param_name: Optional[str] = None) -> Any:
"""Check if a variable matches the specified type constraints.
:param variable: The variable to check
:param valid_types: List of valid types for the variable
:param default_value: Value to return if type check fails
:param critical: If True, raises TypeError instead of logging error
:param param_name: Optional parameter name for error messages
:return: The original variable if valid, default_value if invalid
:raise TypeError: If critical=True and type check fails
"""
# Use provided param_name or try to get it automatically
var_name = param_name or get_variable_name(variable) or "Unknown"
# Convert valid_types to a list if None
valid_types = valid_types or []
# Handle None value
if variable is None:
if type(None) in valid_types:
return variable
error_msg = f'Argument "{var_name}" cannot be None'
if critical:
raise TypeError(error_msg)
log.error(f'[Ignored] {error_msg}')
return default_value
# If no valid_types specified and variable has a value, return it
if not valid_types:
return variable
# Check if variable type matches any of the valid types
if not any(isinstance(variable, t) for t in valid_types):
type_names = [t.__name__ for t in valid_types]
error_msg = f'Argument "{var_name}" must be of type {" or ".join(type_names)}'
if critical:
raise TypeError(error_msg)
log.error(f'[Ignored] {error_msg}')
return default_value
return variable
scrapling/engines/toolbelt/fingerprints.py
+32 -46
View File
@@ -2,19 +2,20 @@
Functions related to generating headers and fingerprints generally
"""
import platform
from platform import system as platform_system
from browserforge.fingerprints import Fingerprint, FingerprintGenerator
from browserforge.headers import Browser, HeaderGenerator
from tldextract import extract
from browserforge.headers import Browser, HeaderGenerator
from scrapling.core._types import Dict, Union
from scrapling.core._types import Dict, Optional
from scrapling.core.utils import lru_cache
__OS_NAME__ = platform_system()
@lru_cache(10, typed=True)
def generate_convincing_referer(url: str) -> str:
"""Takes the domain from the URL without the subdomain/suffix and make it look like you were searching google for this website
"""Takes the domain from the URL without the subdomain/suffix and make it look like you were searching Google for this website
>>> generate_convincing_referer('https://www.somewebsite.com/blah')
'https://www.google.com/search?q=somewebsite'
@@ -23,59 +24,44 @@ def generate_convincing_referer(url: str) -> str:
:return: Google's search URL of the domain name
"""
website_name = extract(url).domain
return f'https://www.google.com/search?q={website_name}'
return f"https://www.google.com/search?q={website_name}"
@lru_cache(1, typed=True)
def get_os_name() -> Union[str, None]:
def get_os_name() -> Optional[str]:
"""Get the current OS name in the same format needed for browserforge
:return: Current OS name or `None` otherwise
"""
#
os_name = platform.system()
return {
'Linux': 'linux',
'Darwin': 'macos',
'Windows': 'windows',
# For the future? because why not
'iOS': 'ios',
}.get(os_name)
def generate_suitable_fingerprint() -> Fingerprint:
"""Generates a browserforge's fingerprint that matches current OS, desktop device, and Chrome with version 128 at least.
This function was originally created to test Browserforge's injector.
:return: `Fingerprint` object
"""
return FingerprintGenerator(
browser=[Browser(name='chrome', min_version=128)],
os=get_os_name(), # None is ignored
device='desktop'
).generate()
"Linux": "linux",
"Darwin": "macos",
"Windows": "windows",
# For the future? because why not?
"iOS": "ios",
}.get(__OS_NAME__)
def generate_headers(browser_mode: bool = False) -> Dict:
"""Generate real browser-like headers using browserforge's generator
:param browser_mode: If enabled, the headers created are used for playwright so it have to match everything
:param browser_mode: If enabled, the headers created are used for playwright, so it has to match everything
:return: A dictionary of the generated headers
"""
if browser_mode:
# In this mode we don't care about anything other than matching the OS and the browser type with the browser we are using
# So we don't raise any inconsistency red flags while websites fingerprinting us
os_name = get_os_name()
return HeaderGenerator(
browser=[Browser(name='chrome', min_version=130)],
os=os_name, # None is ignored
device='desktop'
).generate()
else:
# Here it's used for normal requests that aren't done through browsers so we can take it lightly
browsers = [
Browser(name='chrome', min_version=120),
Browser(name='firefox', min_version=120),
Browser(name='edge', min_version=120),
]
return HeaderGenerator(browser=browsers, device='desktop').generate()
# In the browser mode, we don't care about anything other than matching the OS and the browser type with the browser we are using,
# So we don't raise any inconsistency red flags while websites fingerprinting us
os_name = get_os_name()
browsers = [Browser(name="chrome", min_version=130)]
if not browser_mode:
os_name = ("windows", "macos", "linux")
browsers.extend(
[
Browser(name="firefox", min_version=130),
Browser(name="edge", min_version=130),
]
)
return HeaderGenerator(browser=browsers, os=os_name, device="desktop").generate()
__default_useragent__ = generate_headers(browser_mode=False).get("User-Agent")
scrapling/engines/toolbelt/navigation.py
+65 -36
View File
@@ -1,74 +1,97 @@
"""
Functions related to files and URLs
"""
import os
from pathlib import Path
from functools import lru_cache
from urllib.parse import urlencode, urlparse
from playwright.async_api import Route as async_Route
from msgspec import Struct, structs, convert, ValidationError
from playwright.sync_api import Route
from scrapling.core._types import Dict, Optional, Union
from scrapling.core.utils import log, lru_cache
from scrapling.core.utils import log
from scrapling.core._types import Dict, Optional, Tuple
from scrapling.engines.constants import DEFAULT_DISABLED_RESOURCES
__BYPASSES_DIR__ = Path(__file__).parent / "bypasses"
class ProxyDict(Struct):
server: str
username: str = ""
password: str = ""
def intercept_route(route: Route):
"""This is just a route handler but it drops requests that its type falls in `DEFAULT_DISABLED_RESOURCES`
"""This is just a route handler, but it drops requests that its type falls in `DEFAULT_DISABLED_RESOURCES`
:param route: PlayWright `Route` object of the current page
:return: PlayWright `Route` object
"""
if route.request.resource_type in DEFAULT_DISABLED_RESOURCES:
log.debug(f'Blocking background resource "{route.request.url}" of type "{route.request.resource_type}"')
log.debug(
f'Blocking background resource "{route.request.url}" of type "{route.request.resource_type}"'
)
route.abort()
else:
route.continue_()
async def async_intercept_route(route: async_Route):
"""This is just a route handler but it drops requests that its type falls in `DEFAULT_DISABLED_RESOURCES`
"""This is just a route handler, but it drops requests that its type falls in `DEFAULT_DISABLED_RESOURCES`
:param route: PlayWright `Route` object of the current page
:return: PlayWright `Route` object
"""
if route.request.resource_type in DEFAULT_DISABLED_RESOURCES:
log.debug(f'Blocking background resource "{route.request.url}" of type "{route.request.resource_type}"')
log.debug(
f'Blocking background resource "{route.request.url}" of type "{route.request.resource_type}"'
)
await route.abort()
else:
await route.continue_()
def construct_proxy_dict(proxy_string: Union[str, Dict[str, str]]) -> Union[Dict, None]:
def construct_proxy_dict(
proxy_string: str | Dict[str, str], as_tuple=False
) -> Optional[Dict | Tuple]:
"""Validate a proxy and return it in the acceptable format for Playwright
Reference: https://playwright.dev/python/docs/network#http-proxy
:param proxy_string: A string or a dictionary representation of the proxy.
:param as_tuple: Return the proxy dictionary as a tuple to be cachable
:return:
"""
if proxy_string:
if isinstance(proxy_string, str):
proxy = urlparse(proxy_string)
try:
return {
'server': f'{proxy.scheme}://{proxy.hostname}:{proxy.port}',
'username': proxy.username or '',
'password': proxy.password or '',
}
except ValueError:
# Urllib will say that one of the parameters above can't be casted to the correct type like `int` for port etc...
raise TypeError('The proxy argument\'s string is in invalid format!')
if isinstance(proxy_string, str):
proxy = urlparse(proxy_string)
if (
proxy.scheme not in ("http", "https", "socks4", "socks5")
or not proxy.hostname
):
raise ValueError("Invalid proxy string!")
elif isinstance(proxy_string, dict):
valid_keys = ('server', 'username', 'password', )
if all(key in valid_keys for key in proxy_string.keys()) and not any(key not in valid_keys for key in proxy_string.keys()):
return proxy_string
else:
raise TypeError(f'A proxy dictionary must have only these keys: {valid_keys}')
try:
result = {
"server": f"{proxy.scheme}://{proxy.hostname}",
"username": proxy.username or "",
"password": proxy.password or "",
}
if proxy.port:
result["server"] += f":{proxy.port}"
return tuple(result.items()) if as_tuple else result
except ValueError:
# Urllib will say that one of the parameters above can't be casted to the correct type like `int` for port etc...
raise ValueError("The proxy argument's string is in invalid format!")
else:
raise TypeError(f'Invalid type of proxy ({type(proxy_string)}), the proxy argument must be a string or a dictionary!')
elif isinstance(proxy_string, dict):
try:
validated = convert(proxy_string, ProxyDict)
result_dict = structs.asdict(validated)
return tuple(result_dict.items()) if as_tuple else result_dict
except ValidationError as e:
raise TypeError(f"Invalid proxy dictionary: {e}")
# The default value for proxy in Playwright's source is `None`
return None
@@ -84,17 +107,24 @@ def construct_cdp_url(cdp_url: str, query_params: Optional[Dict] = None) -> str:
parsed = urlparse(cdp_url)
# Check scheme
if parsed.scheme not in ('ws', 'wss'):
if parsed.scheme not in ("ws", "wss"):
raise ValueError("CDP URL must use 'ws://' or 'wss://' scheme")
# Validate hostname and port
if not parsed.netloc:
raise ValueError("Invalid hostname for the CDP URL")
# Ensure path starts with /
try:
# Checking if the port is valid (if available)
_ = parsed.port
except ValueError:
# urlparse will raise `ValueError` if the port can't be casted to integer
raise ValueError("Invalid port for the CDP URL")
# Ensure the path starts with /
path = parsed.path
if not path.startswith('/'):
path = '/' + path
if not path.startswith("/"):
path = "/" + path
# Reconstruct the base URL with validated parts
validated_base = f"{parsed.scheme}://{parsed.netloc}{path}"
@@ -112,10 +142,9 @@ def construct_cdp_url(cdp_url: str, query_params: Optional[Dict] = None) -> str:
@lru_cache(10, typed=True)
def js_bypass_path(filename: str) -> str:
"""Takes the base filename of JS file inside the `bypasses` folder then return the full path of it
"""Takes the base filename of a JS file inside the `bypasses` folder, then return the full path of it
:param filename: The base filename of the JS file.
:return: The full path of the JS file.
"""
current_directory = os.path.dirname(__file__)
return os.path.join(current_directory, 'bypasses', filename)
return str(__BYPASSES_DIR__ / filename)
scrapling/fetchers.py
+227 -333
View File
@@ -1,284 +1,127 @@
from scrapling.core._types import (Callable, Dict, List, Literal, Optional,
SelectorWaitStates, Union)
from scrapling.engines import (CamoufoxEngine, PlaywrightEngine, StaticEngine,
check_if_engine_usable)
from scrapling.core._types import (
Callable,
Dict,
List,
Optional,
SelectorWaitStates,
Iterable,
)
from scrapling.engines import (
FetcherSession,
StealthySession,
AsyncStealthySession,
DynamicSession,
AsyncDynamicSession,
FetcherClient as _FetcherClient,
AsyncFetcherClient as _AsyncFetcherClient,
)
from scrapling.engines.toolbelt import BaseFetcher, Response
__FetcherClientInstance__ = _FetcherClient()
__AsyncFetcherClientInstance__ = _AsyncFetcherClient()
class Fetcher(BaseFetcher):
"""A basic `Fetcher` class type that can only do basic GET, POST, PUT, and DELETE HTTP requests based on httpx.
"""A basic `Fetcher` class type that can only do basic GET, POST, PUT, and DELETE HTTP requests based on `curl_cffi`."""
Any additional keyword arguments passed to the methods below are passed to the respective httpx's method directly.
"""
@classmethod
def get(
cls, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = 10, stealthy_headers: bool = True,
proxy: Optional[str] = None, retries: Optional[int] = 3, custom_config: Dict = None, **kwargs: Dict) -> Response:
"""Make basic HTTP GET request for you but with some added flavors.
:param url: Target url.
:param follow_redirects: As the name says -- if enabled (default), redirects will be followed.
:param timeout: The time to wait for the request to finish in seconds. The default is 10 seconds.
:param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers and
create a referer header as if this request had came from Google's search of this URL's domain.
:param proxy: A string of a proxy to use for http and https requests, the format accepted is `http://username:password@localhost:8030`
:param retries: The number of retries to do through httpx if the request failed for any reason. The default is 3 retries.
:param custom_config: A dictionary of custom parser arguments to use with this request. Any argument passed will override any class parameters values.
:param kwargs: Any additional keyword arguments are passed directly to `httpx.get()` function so check httpx documentation for details.
:return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
"""
if not custom_config:
custom_config = {}
elif not isinstance(custom_config, dict):
ValueError(f"The custom parser config must be of type dictionary, got {cls.__class__}")
adaptor_arguments = tuple({**cls._generate_parser_arguments(), **custom_config}.items())
response_object = StaticEngine(url, proxy, stealthy_headers, follow_redirects, timeout, retries, adaptor_arguments=adaptor_arguments).get(**kwargs)
return response_object
@classmethod
def post(
cls, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = 10, stealthy_headers: bool = True,
proxy: Optional[str] = None, retries: Optional[int] = 3, custom_config: Dict = None, **kwargs: Dict) -> Response:
"""Make basic HTTP POST request for you but with some added flavors.
:param url: Target url.
:param follow_redirects: As the name says -- if enabled (default), redirects will be followed.
:param timeout: The time to wait for the request to finish in seconds. The default is 10 seconds.
:param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers and
create a referer header as if this request came from Google's search of this URL's domain.
:param proxy: A string of a proxy to use for http and https requests, the format accepted is `http://username:password@localhost:8030`
:param retries: The number of retries to do through httpx if the request failed for any reason. The default is 3 retries.
:param custom_config: A dictionary of custom parser arguments to use with this request. Any argument passed will override any class parameters values.
:param kwargs: Any additional keyword arguments are passed directly to `httpx.post()` function so check httpx documentation for details.
:return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
"""
if not custom_config:
custom_config = {}
elif not isinstance(custom_config, dict):
ValueError(f"The custom parser config must be of type dictionary, got {cls.__class__}")
adaptor_arguments = tuple({**cls._generate_parser_arguments(), **custom_config}.items())
response_object = StaticEngine(url, proxy, stealthy_headers, follow_redirects, timeout, retries, adaptor_arguments=adaptor_arguments).post(**kwargs)
return response_object
@classmethod
def put(
cls, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = 10, stealthy_headers: bool = True,
proxy: Optional[str] = None, retries: Optional[int] = 3, custom_config: Dict = None, **kwargs: Dict) -> Response:
"""Make basic HTTP PUT request for you but with some added flavors.
:param url: Target url
:param follow_redirects: As the name says -- if enabled (default), redirects will be followed.
:param timeout: The time to wait for the request to finish in seconds. The default is 10 seconds.
:param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers and
create a referer header as if this request came from Google's search of this URL's domain.
:param proxy: A string of a proxy to use for http and https requests, the format accepted is `http://username:password@localhost:8030`
:param retries: The number of retries to do through httpx if the request failed for any reason. The default is 3 retries.
:param custom_config: A dictionary of custom parser arguments to use with this request. Any argument passed will override any class parameters values.
:param kwargs: Any additional keyword arguments are passed directly to `httpx.put()` function so check httpx documentation for details.
:return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
"""
if not custom_config:
custom_config = {}
elif not isinstance(custom_config, dict):
ValueError(f"The custom parser config must be of type dictionary, got {cls.__class__}")
adaptor_arguments = tuple({**cls._generate_parser_arguments(), **custom_config}.items())
response_object = StaticEngine(url, proxy, stealthy_headers, follow_redirects, timeout, retries, adaptor_arguments=adaptor_arguments).put(**kwargs)
return response_object
@classmethod
def delete(
cls, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = 10, stealthy_headers: bool = True,
proxy: Optional[str] = None, retries: Optional[int] = 3, custom_config: Dict = None, **kwargs: Dict) -> Response:
"""Make basic HTTP DELETE request for you but with some added flavors.
:param url: Target url
:param follow_redirects: As the name says -- if enabled (default), redirects will be followed.
:param timeout: The time to wait for the request to finish in seconds. The default is 10 seconds.
:param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers and
create a referer header as if this request came from Google's search of this URL's domain.
:param proxy: A string of a proxy to use for http and https requests, the format accepted is `http://username:password@localhost:8030`
:param retries: The number of retries to do through httpx if the request failed for any reason. The default is 3 retries.
:param custom_config: A dictionary of custom parser arguments to use with this request. Any argument passed will override any class parameters values.
:param kwargs: Any additional keyword arguments are passed directly to `httpx.delete()` function so check httpx documentation for details.
:return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
"""
if not custom_config:
custom_config = {}
elif not isinstance(custom_config, dict):
ValueError(f"The custom parser config must be of type dictionary, got {cls.__class__}")
adaptor_arguments = tuple({**cls._generate_parser_arguments(), **custom_config}.items())
response_object = StaticEngine(url, proxy, stealthy_headers, follow_redirects, timeout, retries, adaptor_arguments=adaptor_arguments).delete(**kwargs)
return response_object
get = __FetcherClientInstance__.get
post = __FetcherClientInstance__.post
put = __FetcherClientInstance__.put
delete = __FetcherClientInstance__.delete
class AsyncFetcher(Fetcher):
@classmethod
async def get(
cls, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = 10, stealthy_headers: bool = True,
proxy: Optional[str] = None, retries: Optional[int] = 3, custom_config: Dict = None, **kwargs: Dict) -> Response:
"""Make basic HTTP GET request for you but with some added flavors.
class AsyncFetcher(BaseFetcher):
"""A basic `Fetcher` class type that can only do basic GET, POST, PUT, and DELETE HTTP requests based on `curl_cffi`."""
:param url: Target url.
:param follow_redirects: As the name says -- if enabled (default), redirects will be followed.
:param timeout: The time to wait for the request to finish in seconds. The default is 10 seconds.
:param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers and
create a referer header as if this request had came from Google's search of this URL's domain.
:param proxy: A string of a proxy to use for http and https requests, the format accepted is `http://username:password@localhost:8030`
:param retries: The number of retries to do through httpx if the request failed for any reason. The default is 3 retries.
:param custom_config: A dictionary of custom parser arguments to use with this request. Any argument passed will override any class parameters values.
:param kwargs: Any additional keyword arguments are passed directly to `httpx.get()` function so check httpx documentation for details.
:return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
"""
if not custom_config:
custom_config = {}
elif not isinstance(custom_config, dict):
ValueError(f"The custom parser config must be of type dictionary, got {cls.__class__}")
adaptor_arguments = tuple({**cls._generate_parser_arguments(), **custom_config}.items())
response_object = await StaticEngine(url, proxy, stealthy_headers, follow_redirects, timeout, retries=retries, adaptor_arguments=adaptor_arguments).async_get(**kwargs)
return response_object
@classmethod
async def post(
cls, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = 10, stealthy_headers: bool = True,
proxy: Optional[str] = None, retries: Optional[int] = 3, custom_config: Dict = None, **kwargs: Dict) -> Response:
"""Make basic HTTP POST request for you but with some added flavors.
:param url: Target url.
:param follow_redirects: As the name says -- if enabled (default), redirects will be followed.
:param timeout: The time to wait for the request to finish in seconds. The default is 10 seconds.
:param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers and
create a referer header as if this request came from Google's search of this URL's domain.
:param proxy: A string of a proxy to use for http and https requests, the format accepted is `http://username:password@localhost:8030`
:param retries: The number of retries to do through httpx if the request failed for any reason. The default is 3 retries.
:param custom_config: A dictionary of custom parser arguments to use with this request. Any argument passed will override any class parameters values.
:param kwargs: Any additional keyword arguments are passed directly to `httpx.post()` function so check httpx documentation for details.
:return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
"""
if not custom_config:
custom_config = {}
elif not isinstance(custom_config, dict):
ValueError(f"The custom parser config must be of type dictionary, got {cls.__class__}")
adaptor_arguments = tuple({**cls._generate_parser_arguments(), **custom_config}.items())
response_object = await StaticEngine(url, proxy, stealthy_headers, follow_redirects, timeout, retries=retries, adaptor_arguments=adaptor_arguments).async_post(**kwargs)
return response_object
@classmethod
async def put(
cls, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = 10, stealthy_headers: bool = True,
proxy: Optional[str] = None, retries: Optional[int] = 3, custom_config: Dict = None, **kwargs: Dict) -> Response:
"""Make basic HTTP PUT request for you but with some added flavors.
:param url: Target url
:param follow_redirects: As the name says -- if enabled (default), redirects will be followed.
:param timeout: The time to wait for the request to finish in seconds. The default is 10 seconds.
:param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers and
create a referer header as if this request came from Google's search of this URL's domain.
:param proxy: A string of a proxy to use for http and https requests, the format accepted is `http://username:password@localhost:8030`
:param retries: The number of retries to do through httpx if the request failed for any reason. The default is 3 retries.
:param custom_config: A dictionary of custom parser arguments to use with this request. Any argument passed will override any class parameters values.
:param kwargs: Any additional keyword arguments are passed directly to `httpx.put()` function so check httpx documentation for details.
:return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
"""
if not custom_config:
custom_config = {}
elif not isinstance(custom_config, dict):
ValueError(f"The custom parser config must be of type dictionary, got {cls.__class__}")
adaptor_arguments = tuple({**cls._generate_parser_arguments(), **custom_config}.items())
response_object = await StaticEngine(url, proxy, stealthy_headers, follow_redirects, timeout, retries=retries, adaptor_arguments=adaptor_arguments).async_put(**kwargs)
return response_object
@classmethod
async def delete(
cls, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = 10, stealthy_headers: bool = True,
proxy: Optional[str] = None, retries: Optional[int] = 3, custom_config: Dict = None, **kwargs: Dict) -> Response:
"""Make basic HTTP DELETE request for you but with some added flavors.
:param url: Target url
:param follow_redirects: As the name says -- if enabled (default), redirects will be followed.
:param timeout: The time to wait for the request to finish in seconds. The default is 10 seconds.
:param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers and
create a referer header as if this request came from Google's search of this URL's domain.
:param proxy: A string of a proxy to use for http and https requests, the format accepted is `http://username:password@localhost:8030`
:param retries: The number of retries to do through httpx if the request failed for any reason. The default is 3 retries.
:param custom_config: A dictionary of custom parser arguments to use with this request. Any argument passed will override any class parameters values.
:param kwargs: Any additional keyword arguments are passed directly to `httpx.delete()` function so check httpx documentation for details.
:return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
"""
if not custom_config:
custom_config = {}
elif not isinstance(custom_config, dict):
ValueError(f"The custom parser config must be of type dictionary, got {cls.__class__}")
adaptor_arguments = tuple({**cls._generate_parser_arguments(), **custom_config}.items())
response_object = await StaticEngine(url, proxy, stealthy_headers, follow_redirects, timeout, retries=retries, adaptor_arguments=adaptor_arguments).async_delete(**kwargs)
return response_object
get = __AsyncFetcherClientInstance__.get
post = __AsyncFetcherClientInstance__.post
put = __AsyncFetcherClientInstance__.put
delete = __AsyncFetcherClientInstance__.delete
class StealthyFetcher(BaseFetcher):
"""A `Fetcher` class type that is completely stealthy fetcher that uses a modified version of Firefox.
"""A `Fetcher` class type that is a completely stealthy fetcher that uses a modified version of Firefox.
It works as real browsers passing almost all online tests/protections based on Camoufox.
Other added flavors include setting the faked OS fingerprints to match the user's OS and the referer of every request is set as if this request came from Google's search of this URL's domain.
It works as real browsers passing almost all online tests/protections based on Camoufox.
Other added flavors include setting the faked OS fingerprints to match the user's OS, and the referer of every request is set as if this request came from Google's search of this URL's domain.
"""
@classmethod
def fetch(
cls, url: str, headless: Union[bool, Literal['virtual']] = True, block_images: bool = False, disable_resources: bool = False,
block_webrtc: bool = False, allow_webgl: bool = True, network_idle: bool = False, addons: Optional[List[str]] = None, wait: Optional[int] = 0,
timeout: Optional[float] = 30000, page_action: Callable = None, wait_selector: Optional[str] = None, humanize: Optional[Union[bool, float]] = True,
wait_selector_state: SelectorWaitStates = 'attached', google_search: bool = True, extra_headers: Optional[Dict[str, str]] = None,
proxy: Optional[Union[str, Dict[str, str]]] = None, os_randomize: bool = False, disable_ads: bool = False, geoip: bool = False,
custom_config: Dict = None, additional_arguments: Dict = None
cls,
url: str,
headless: bool = True, # noqa: F821
block_images: bool = False,
disable_resources: bool = False,
block_webrtc: bool = False,
allow_webgl: bool = True,
network_idle: bool = False,
humanize: bool | float = True,
solve_cloudflare: bool = False,
wait: int | float = 0,
timeout: int | float = 30000,
page_action: Optional[Callable] = None,
wait_selector: Optional[str] = None,
addons: Optional[List[str]] = None,
wait_selector_state: SelectorWaitStates = "attached",
cookies: Optional[List[Dict]] = None,
google_search: bool = True,
extra_headers: Optional[Dict[str, str]] = None,
proxy: Optional[str | Dict[str, str]] = None,
os_randomize: bool = False,
disable_ads: bool = False,
geoip: bool = False,
custom_config: Optional[Dict] = None,
additional_args: Optional[Dict] = None,
) -> Response:
"""
Opens up a browser and do your request based on your chosen options below.
:param url: Target url.
:param headless: Run the browser in headless/hidden (default), 'virtual' screen mode, or headful/visible mode.
:param headless: Run the browser in headless/hidden (default), or headful/visible mode.
:param block_images: Prevent the loading of images through Firefox preferences.
This can help save your proxy usage but be careful with this option as it makes some websites never finish loading.
:param disable_resources: Drop requests of unnecessary resources for a speed boost. It depends but it made requests ~25% faster in my tests for some websites.
:param disable_resources: Drop requests of unnecessary resources for a speed boost. It depends, but it made requests ~25% faster in my tests for some websites.
Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
This can help save your proxy usage but be careful with this option as it makes some websites never finish loading.
:param block_webrtc: Blocks WebRTC entirely.
:param cookies: Set cookies for the next request.
:param addons: List of Firefox addons to use. Must be paths to extracted addons.
:param disable_ads: Disabled by default, this installs `uBlock Origin` addon on the browser if enabled.
:param humanize: Humanize the cursor movement. Takes either True or the MAX duration in seconds of the cursor movement. The cursor typically takes up to 1.5 seconds to move across the window.
:param allow_webgl: Enabled by default. Disabling it WebGL not recommended as many WAFs now checks if WebGL is enabled.
:param geoip: Recommended to use with proxies; Automatically use IP's longitude, latitude, timezone, country, locale, & spoof the WebRTC IP address.
It will also calculate and spoof the browser's language based on the distribution of language speakers in the target region.
:param solve_cloudflare: Solves all 3 types of the Cloudflare's Turnstile wait page before returning the response to you.
:param allow_webgl: Enabled by default. Disabling WebGL is not recommended as many WAFs now check if WebGL is enabled.
:param network_idle: Wait for the page until there are no network connections for at least 500 ms.
:param disable_ads: Disabled by default, this installs the `uBlock Origin` addon on the browser if enabled.
:param os_randomize: If enabled, Scrapling will randomize the OS fingerprints used. The default is Scrapling matching the fingerprints with the current OS.
:param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30000.
:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning `Response` object.
:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
:param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
:param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again.
:param wait_selector: Wait for a specific css selector to be in a specific state.
:param wait_selector_state: The state to wait for the selector given with `wait_selector`. Default state is `attached`.
:param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search for this website's domain name.
:param wait_selector: Wait for a specific CSS selector to be in a specific state.
:param geoip: Recommended to use with proxies; Automatically use IP's longitude, latitude, timezone, country, locale, and spoof the WebRTC IP address.
It will also calculate and spoof the browser's language based on the distribution of language speakers in the target region.
:param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.
:param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name.
:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
:param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.
:param custom_config: A dictionary of custom parser arguments to use with this request. Any argument passed will override any class parameters values.
:param additional_arguments: Additional arguments to be passed to Camoufox as additional settings and it takes higher priority than Scrapling's settings.
:return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
:param additional_args: Additional arguments to be passed to Camoufox as additional settings, and it takes higher priority than Scrapling's settings.
:return: A `Response` object.
"""
if not custom_config:
custom_config = {}
elif not isinstance(custom_config, dict):
ValueError(f"The custom parser config must be of type dictionary, got {cls.__class__}")
ValueError(
f"The custom parser config must be of type dictionary, got {cls.__class__}"
)
engine = CamoufoxEngine(
with StealthySession(
wait=wait,
max_pages=1,
proxy=proxy,
geoip=geoip,
addons=addons,
timeout=timeout,
cookies=cookies,
headless=headless,
humanize=humanize,
disable_ads=disable_ads,
@@ -291,64 +134,90 @@ class StealthyFetcher(BaseFetcher):
wait_selector=wait_selector,
google_search=google_search,
extra_headers=extra_headers,
solve_cloudflare=solve_cloudflare,
disable_resources=disable_resources,
wait_selector_state=wait_selector_state,
adaptor_arguments={**cls._generate_parser_arguments(), **custom_config},
additional_arguments=additional_arguments or {}
)
return engine.fetch(url)
selector_config={**cls._generate_parser_arguments(), **custom_config},
additional_args=additional_args or {},
) as engine:
return engine.fetch(url)
@classmethod
async def async_fetch(
cls, url: str, headless: Union[bool, Literal['virtual']] = True, block_images: bool = False, disable_resources: bool = False,
block_webrtc: bool = False, allow_webgl: bool = True, network_idle: bool = False, addons: Optional[List[str]] = None, wait: Optional[int] = 0,
timeout: Optional[float] = 30000, page_action: Callable = None, wait_selector: Optional[str] = None, humanize: Optional[Union[bool, float]] = True,
wait_selector_state: SelectorWaitStates = 'attached', google_search: bool = True, extra_headers: Optional[Dict[str, str]] = None,
proxy: Optional[Union[str, Dict[str, str]]] = None, os_randomize: bool = False, disable_ads: bool = False, geoip: bool = False,
custom_config: Dict = None, additional_arguments: Dict = None
cls,
url: str,
headless: bool = True, # noqa: F821
block_images: bool = False,
disable_resources: bool = False,
block_webrtc: bool = False,
allow_webgl: bool = True,
network_idle: bool = False,
humanize: bool | float = True,
solve_cloudflare: bool = False,
wait: int | float = 0,
timeout: int | float = 30000,
page_action: Optional[Callable] = None,
wait_selector: Optional[str] = None,
addons: Optional[List[str]] = None,
wait_selector_state: SelectorWaitStates = "attached",
cookies: Optional[List[Dict]] = None,
google_search: bool = True,
extra_headers: Optional[Dict[str, str]] = None,
proxy: Optional[str | Dict[str, str]] = None,
os_randomize: bool = False,
disable_ads: bool = False,
geoip: bool = False,
custom_config: Optional[Dict] = None,
additional_args: Optional[Dict] = None,
) -> Response:
"""
Opens up a browser and do your request based on your chosen options below.
:param url: Target url.
:param headless: Run the browser in headless/hidden (default), 'virtual' screen mode, or headful/visible mode.
:param headless: Run the browser in headless/hidden (default), or headful/visible mode.
:param block_images: Prevent the loading of images through Firefox preferences.
This can help save your proxy usage but be careful with this option as it makes some websites never finish loading.
:param disable_resources: Drop requests of unnecessary resources for a speed boost. It depends but it made requests ~25% faster in my tests for some websites.
:param disable_resources: Drop requests of unnecessary resources for a speed boost. It depends, but it made requests ~25% faster in my tests for some websites.
Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
This can help save your proxy usage but be careful with this option as it makes some websites never finish loading.
:param block_webrtc: Blocks WebRTC entirely.
:param cookies: Set cookies for the next request.
:param addons: List of Firefox addons to use. Must be paths to extracted addons.
:param disable_ads: Disabled by default, this installs `uBlock Origin` addon on the browser if enabled.
:param humanize: Humanize the cursor movement. Takes either True or the MAX duration in seconds of the cursor movement. The cursor typically takes up to 1.5 seconds to move across the window.
:param allow_webgl: Enabled by default. Disabling it WebGL not recommended as many WAFs now checks if WebGL is enabled.
:param geoip: Recommended to use with proxies; Automatically use IP's longitude, latitude, timezone, country, locale, & spoof the WebRTC IP address.
It will also calculate and spoof the browser's language based on the distribution of language speakers in the target region.
:param solve_cloudflare: Solves all 3 types of the Cloudflare's Turnstile wait page before returning the response to you.
:param allow_webgl: Enabled by default. Disabling WebGL is not recommended as many WAFs now check if WebGL is enabled.
:param network_idle: Wait for the page until there are no network connections for at least 500 ms.
:param disable_ads: Disabled by default, this installs the `uBlock Origin` addon on the browser if enabled.
:param os_randomize: If enabled, Scrapling will randomize the OS fingerprints used. The default is Scrapling matching the fingerprints with the current OS.
:param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30000
:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning `Response` object.
:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
:param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
:param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again.
:param wait_selector: Wait for a specific css selector to be in a specific state.
:param wait_selector_state: The state to wait for the selector given with `wait_selector`. Default state is `attached`.
:param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search for this website's domain name.
:param wait_selector: Wait for a specific CSS selector to be in a specific state.
:param geoip: Recommended to use with proxies; Automatically use IP's longitude, latitude, timezone, country, locale, and spoof the WebRTC IP address.
It will also calculate and spoof the browser's language based on the distribution of language speakers in the target region.
:param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.
:param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name.
:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
:param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.
:param custom_config: A dictionary of custom parser arguments to use with this request. Any argument passed will override any class parameters values.
:param additional_arguments: Additional arguments to be passed to Camoufox as additional settings and it takes higher priority than Scrapling's settings.
:return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
:param additional_args: Additional arguments to be passed to Camoufox as additional settings, and it takes higher priority than Scrapling's settings.
:return: A `Response` object.
"""
if not custom_config:
custom_config = {}
elif not isinstance(custom_config, dict):
ValueError(f"The custom parser config must be of type dictionary, got {cls.__class__}")
ValueError(
f"The custom parser config must be of type dictionary, got {cls.__class__}"
)
engine = CamoufoxEngine(
async with AsyncStealthySession(
wait=wait,
max_pages=1,
proxy=proxy,
geoip=geoip,
addons=addons,
timeout=timeout,
cookies=cookies,
headless=headless,
humanize=humanize,
disable_ads=disable_ads,
@@ -361,82 +230,99 @@ class StealthyFetcher(BaseFetcher):
wait_selector=wait_selector,
google_search=google_search,
extra_headers=extra_headers,
solve_cloudflare=solve_cloudflare,
disable_resources=disable_resources,
wait_selector_state=wait_selector_state,
adaptor_arguments={**cls._generate_parser_arguments(), **custom_config},
additional_arguments=additional_arguments or {}
)
return await engine.async_fetch(url)
selector_config={**cls._generate_parser_arguments(), **custom_config},
additional_args=additional_args or {},
) as engine:
return await engine.fetch(url)
class PlayWrightFetcher(BaseFetcher):
class DynamicFetcher(BaseFetcher):
"""A `Fetcher` class type that provide many options, all of them are based on PlayWright.
Using this Fetcher class, you can do requests with:
- Vanilla Playwright without any modifications other than the ones you chose.
- Stealthy Playwright with the stealth mode I wrote for it. It's still a work in progress but it bypasses many online tests like bot.sannysoft.com
- Stealthy Playwright with the stealth mode I wrote for it. It's still a work in progress, but it bypasses many online tests like bot.sannysoft.com
Some of the things stealth mode does include:
1) Patches the CDP runtime fingerprint.
2) Mimics some of the real browsers' properties by injecting several JS files and using custom options.
3) Using custom flags on launch to hide Playwright even more and make it faster.
4) Generates real browser's headers of the same type and same user OS then append it to the request.
- Real browsers by passing the `real_chrome` argument or the CDP URL of your browser to be controlled by the Fetcher and most of the options can be enabled on it.
4) Generates real browser's headers of the same type and same user OS, then append it to the request.
- Real browsers by passing the `real_chrome` argument or the CDP URL of your browser to be controlled by the Fetcher, and most of the options can be enabled on it.
- NSTBrowser's docker browserless option by passing the CDP URL and enabling `nstbrowser_mode` option.
> Note that these are the main options with PlayWright but it can be mixed together.
> Note that these are the main options with PlayWright, but it can be mixed.
"""
@classmethod
def fetch(
cls, url: str, headless: Union[bool, str] = True, disable_resources: bool = None,
useragent: Optional[str] = None, network_idle: bool = False, timeout: Optional[float] = 30000, wait: Optional[int] = 0,
page_action: Optional[Callable] = None, wait_selector: Optional[str] = None, wait_selector_state: SelectorWaitStates = 'attached',
hide_canvas: bool = False, disable_webgl: bool = False, extra_headers: Optional[Dict[str, str]] = None, google_search: bool = True,
proxy: Optional[Union[str, Dict[str, str]]] = None, locale: Optional[str] = 'en-US',
stealth: bool = False, real_chrome: bool = False,
cdp_url: Optional[str] = None,
nstbrowser_mode: bool = False, nstbrowser_config: Optional[Dict] = None,
custom_config: Dict = None
cls,
url: str,
headless: bool = True,
google_search: bool = True,
hide_canvas: bool = False,
disable_webgl: bool = False,
real_chrome: bool = False,
stealth: bool = False,
wait: int | float = 0,
page_action: Optional[Callable] = None,
proxy: Optional[str | Dict[str, str]] = None,
locale: str = "en-US",
extra_headers: Optional[Dict[str, str]] = None,
useragent: Optional[str] = None,
cdp_url: Optional[str] = None,
timeout: int | float = 30000,
disable_resources: bool = False,
wait_selector: Optional[str] = None,
cookies: Optional[Iterable[Dict]] = None,
network_idle: bool = False,
wait_selector_state: SelectorWaitStates = "attached",
custom_config: Optional[Dict] = None,
) -> Response:
"""Opens up a browser and do your request based on your chosen options below.
:param url: Target url.
:param headless: Run the browser in headless/hidden (default), or headful/visible mode.
:param disable_resources: Drop requests of unnecessary resources for speed boost. It depends but it made requests ~25% faster in my tests for some websites.
:param disable_resources: Drop requests of unnecessary resources for a speed boost. It depends, but it made requests ~25% faster in my tests for some websites.
Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
This can help save your proxy usage but be careful with this option as it makes some websites never finish loading.
:param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it.
:param cookies: Set cookies for the next request.
:param network_idle: Wait for the page until there are no network connections for at least 500 ms.
:param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30000.
:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning `Response` object.
:param locale: Set the locale for the browser if wanted. The default value is `en-US`.
:param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
:param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again.
:param wait_selector: Wait for a specific css selector to be in a specific state.
:param wait_selector_state: The state to wait for the selector given with `wait_selector`. Default state is `attached`.
:param wait_selector: Wait for a specific CSS selector to be in a specific state.
:param locale: Set the locale for the browser if wanted. The default value is `en-US`.
:param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.
:param stealth: Enables stealth mode, check the documentation to see what stealth mode does currently.
:param real_chrome: If you have chrome browser installed on your device, enable this and the Fetcher will launch an instance of your browser and use it.
:param real_chrome: If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch an instance of your browser and use it.
:param hide_canvas: Add random noise to canvas operations to prevent fingerprinting.
:param disable_webgl: Disables WebGL and WebGL 2.0 support entirely.
:param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search for this website's domain name.
:param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers/NSTBrowser through CDP.
:param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name.
:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
:param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.
:param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers/NSTBrowser through CDP.
:param nstbrowser_mode: Enables NSTBrowser mode, it have to be used with `cdp_url` argument or it will get completely ignored.
:param nstbrowser_config: The config you want to send with requests to the NSTBrowser. If left empty, Scrapling defaults to an optimized NSTBrowser's docker browserless config.
:param custom_config: A dictionary of custom parser arguments to use with this request. Any argument passed will override any class parameters values.
:return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
:return: A `Response` object.
"""
if not custom_config:
custom_config = {}
elif not isinstance(custom_config, dict):
ValueError(f"The custom parser config must be of type dictionary, got {cls.__class__}")
raise ValueError(
f"The custom parser config must be of type dictionary, got {cls.__class__}"
)
engine = PlaywrightEngine(
with DynamicSession(
wait=wait,
proxy=proxy,
locale=locale,
timeout=timeout,
stealth=stealth,
cdp_url=cdp_url,
cookies=cookies,
headless=headless,
useragent=useragent,
real_chrome=real_chrome,
@@ -447,68 +333,82 @@ class PlayWrightFetcher(BaseFetcher):
extra_headers=extra_headers,
wait_selector=wait_selector,
disable_webgl=disable_webgl,
nstbrowser_mode=nstbrowser_mode,
nstbrowser_config=nstbrowser_config,
disable_resources=disable_resources,
wait_selector_state=wait_selector_state,
adaptor_arguments={**cls._generate_parser_arguments(), **custom_config},
)
return engine.fetch(url)
selector_config={**cls._generate_parser_arguments(), **custom_config},
) as session:
return session.fetch(url)
@classmethod
async def async_fetch(
cls, url: str, headless: Union[bool, str] = True, disable_resources: bool = None,
useragent: Optional[str] = None, network_idle: bool = False, timeout: Optional[float] = 30000, wait: Optional[int] = 0,
page_action: Optional[Callable] = None, wait_selector: Optional[str] = None, wait_selector_state: SelectorWaitStates = 'attached',
hide_canvas: bool = False, disable_webgl: bool = False, extra_headers: Optional[Dict[str, str]] = None, google_search: bool = True,
proxy: Optional[Union[str, Dict[str, str]]] = None, locale: Optional[str] = 'en-US',
stealth: bool = False, real_chrome: bool = False,
cdp_url: Optional[str] = None,
nstbrowser_mode: bool = False, nstbrowser_config: Optional[Dict] = None,
custom_config: Dict = None
cls,
url: str,
headless: bool = True,
google_search: bool = True,
hide_canvas: bool = False,
disable_webgl: bool = False,
real_chrome: bool = False,
stealth: bool = False,
wait: int | float = 0,
page_action: Optional[Callable] = None,
proxy: Optional[str | Dict[str, str]] = None,
locale: str = "en-US",
extra_headers: Optional[Dict[str, str]] = None,
useragent: Optional[str] = None,
cdp_url: Optional[str] = None,
timeout: int | float = 30000,
disable_resources: bool = False,
wait_selector: Optional[str] = None,
cookies: Optional[Iterable[Dict]] = None,
network_idle: bool = False,
wait_selector_state: SelectorWaitStates = "attached",
custom_config: Optional[Dict] = None,
) -> Response:
"""Opens up a browser and do your request based on your chosen options below.
:param url: Target url.
:param headless: Run the browser in headless/hidden (default), or headful/visible mode.
:param disable_resources: Drop requests of unnecessary resources for speed boost. It depends but it made requests ~25% faster in my tests for some websites.
:param disable_resources: Drop requests of unnecessary resources for a speed boost. It depends, but it made requests ~25% faster in my tests for some websites.
Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
This can help save your proxy usage but be careful with this option as it makes some websites never finish loading.
:param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it.
:param cookies: Set cookies for the next request.
:param network_idle: Wait for the page until there are no network connections for at least 500 ms.
:param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30000.
:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning `Response` object.
:param locale: Set the locale for the browser if wanted. The default value is `en-US`.
:param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
:param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again.
:param wait_selector: Wait for a specific css selector to be in a specific state.
:param wait_selector_state: The state to wait for the selector given with `wait_selector`. Default state is `attached`.
:param wait_selector: Wait for a specific CSS selector to be in a specific state.
:param locale: Set the locale for the browser if wanted. The default value is `en-US`.
:param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.
:param stealth: Enables stealth mode, check the documentation to see what stealth mode does currently.
:param real_chrome: If you have chrome browser installed on your device, enable this and the Fetcher will launch an instance of your browser and use it.
:param real_chrome: If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch an instance of your browser and use it.
:param hide_canvas: Add random noise to canvas operations to prevent fingerprinting.
:param disable_webgl: Disables WebGL and WebGL 2.0 support entirely.
:param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search for this website's domain name.
:param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers/NSTBrowser through CDP.
:param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name.
:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
:param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.
:param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers/NSTBrowser through CDP.
:param nstbrowser_mode: Enables NSTBrowser mode, it have to be used with `cdp_url` argument or it will get completely ignored.
:param nstbrowser_config: The config you want to send with requests to the NSTBrowser. If left empty, Scrapling defaults to an optimized NSTBrowser's docker browserless config.
:param custom_config: A dictionary of custom parser arguments to use with this request. Any argument passed will override any class parameters values.
:return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
:return: A `Response` object.
"""
if not custom_config:
custom_config = {}
elif not isinstance(custom_config, dict):
ValueError(f"The custom parser config must be of type dictionary, got {cls.__class__}")
raise ValueError(
f"The custom parser config must be of type dictionary, got {cls.__class__}"
)
engine = PlaywrightEngine(
async with AsyncDynamicSession(
wait=wait,
proxy=proxy,
locale=locale,
timeout=timeout,
stealth=stealth,
cdp_url=cdp_url,
cookies=cookies,
headless=headless,
useragent=useragent,
max_pages=1,
real_chrome=real_chrome,
page_action=page_action,
hide_canvas=hide_canvas,
@@ -517,17 +417,11 @@ class PlayWrightFetcher(BaseFetcher):
extra_headers=extra_headers,
wait_selector=wait_selector,
disable_webgl=disable_webgl,
nstbrowser_mode=nstbrowser_mode,
nstbrowser_config=nstbrowser_config,
disable_resources=disable_resources,
wait_selector_state=wait_selector_state,
adaptor_arguments={**cls._generate_parser_arguments(), **custom_config},
)
return await engine.async_fetch(url)
selector_config={**cls._generate_parser_arguments(), **custom_config},
) as session:
return await session.fetch(url)
class CustomFetcher(BaseFetcher):
@classmethod
def fetch(cls, url: str, browser_engine, **kwargs) -> Response:
engine = check_if_engine_usable(browser_engine)(adaptor_arguments=cls._generate_parser_arguments(), **kwargs)
return engine.fetch(url)
PlayWrightFetcher = DynamicFetcher # For backward-compatibility
scrapling/parser.py
+783 -451
View File
File diff suppressed because it is too large Load Diff
setup.cfg
+2 -2
View File
@@ -1,8 +1,8 @@
[metadata]
name = scrapling
version = 0.2.99
version = 0.3
author = Karim Shoair
author_email = karim.shoair@pm.me
description = Scrapling is an undetectable, powerful, flexible, high-performance Python library that makes Web Scraping easy again!
description = Scrapling is an undetectable, powerful, flexible, high-performance Python library that makes Web Scraping easy and effortless as it should be!
license = BSD
home_page = https://github.com/D4Vinci/Scrapling
setup.py
-72
View File
@@ -1,72 +0,0 @@
from setuptools import find_packages, setup
with open("README.md", "r", encoding="utf-8") as fh:
long_description = fh.read()
setup(
name="scrapling",
version="0.2.99",
description="""Scrapling is an undetectable, powerful, flexible, high-performance Python library that makes Web Scraping easy again! In an internet filled with complications,
it simplifies web scraping, even when websites' design changes, while providing impressive speed that surpasses almost all alternatives.""",
long_description=long_description,
long_description_content_type="text/markdown",
author="Karim Shoair",
author_email="karim.shoair@pm.me",
license="BSD",
packages=find_packages(),
zip_safe=False,
package_dir={
"scrapling": "scrapling",
},
entry_points={
'console_scripts': [
'scrapling=scrapling.cli:main'
],
},
include_package_data=True,
classifiers=[
"Operating System :: OS Independent",
"Development Status :: 4 - Beta",
# "Development Status :: 5 - Production/Stable",
# "Development Status :: 6 - Mature",
# "Development Status :: 7 - Inactive",
"Intended Audience :: Developers",
"License :: OSI Approved :: BSD License",
"Natural Language :: English",
"Topic :: Internet :: WWW/HTTP",
"Topic :: Text Processing :: Markup",
"Topic :: Internet :: WWW/HTTP :: Browsers",
"Topic :: Text Processing :: Markup :: HTML",
"Topic :: Software Development :: Libraries :: Python Modules",
"Programming Language :: Python :: 3",
"Programming Language :: Python :: 3 :: Only",
"Programming Language :: Python :: 3.9",
"Programming Language :: Python :: 3.10",
"Programming Language :: Python :: 3.11",
"Programming Language :: Python :: 3.12",
"Programming Language :: Python :: 3.13",
"Programming Language :: Python :: Implementation :: CPython",
"Typing :: Typed",
],
# Instead of using requirements file to dodge possible errors from tox?
install_requires=[
"lxml>=5.0",
"cssselect>=1.2",
'click',
"w3lib",
"orjson>=3",
"tldextract",
'httpx[brotli,zstd, socks]',
'playwright>=1.49.1',
'rebrowser-playwright>=1.49.1',
'camoufox[geoip]>=0.4.11'
],
python_requires=">=3.9",
url="https://github.com/D4Vinci/Scrapling",
project_urls={
"Documentation": "https://scrapling.readthedocs.io/en/latest/",
"Source": "https://github.com/D4Vinci/Scrapling",
"Tracker": "https://github.com/D4Vinci/Scrapling/issues",
}
)
tests/ai/__init__.py
View File
tests/ai/test_ai_mcp.py
+70
View File
@@ -0,0 +1,70 @@
import pytest
import pytest_httpbin
from unittest.mock import Mock, patch
from scrapling.core.ai import ScraplingMCPServer, ResponseModel
@pytest_httpbin.use_class_based_httpbin
class TestMCPServer:
"""Test MCP server functionality"""
@pytest.fixture(scope="class")
def test_url(self, httpbin):
return f"{httpbin.url}/html"
@pytest.fixture
def server(self):
return ScraplingMCPServer()
def test_server_creation(self, server):
"""Test server instance creation"""
assert server._server is not None
assert server._server.name == "Scrapling"
def test_get_tool(self, server, test_url):
"""Test the get tool method"""
result = server.get(url=test_url, extraction_type="markdown")
assert isinstance(result, ResponseModel)
assert result.status == 200
assert result.url == test_url
@pytest.mark.asyncio
async def test_bulk_get_tool(self, server, test_url):
"""Test the bulk_get tool method"""
results = await server.bulk_get(urls=(test_url, test_url), extraction_type="html")
assert len(results) == 2
assert all(isinstance(r, ResponseModel) for r in results)
@pytest.mark.asyncio
async def test_fetch_tool(self, server, test_url):
"""Test the fetch tool method"""
result = await server.fetch(url=test_url, headless=True)
assert isinstance(result, ResponseModel)
assert result.status == 200
@pytest.mark.asyncio
async def test_bulk_fetch_tool(self, server, test_url):
"""Test the bulk_fetch tool method"""
result = await server.bulk_fetch(urls=(test_url, test_url), headless=True)
assert all(isinstance(r, ResponseModel) for r in result)
@pytest.mark.asyncio
async def test_stealthy_fetch_tool(self, server, test_url):
"""Test the stealthy_fetch tool method"""
result = await server.stealthy_fetch(url=test_url, headless=True)
assert isinstance(result, ResponseModel)
assert result.status == 200
@pytest.mark.asyncio
async def test_bulk_stealthy_fetch_tool(self, server, test_url):
"""Test the bulk_stealthy_fetch tool method"""
result = await server.bulk_stealthy_fetch(urls=(test_url, test_url), headless=True)
assert all(isinstance(r, ResponseModel) for r in result)
def test_serve_method(self, server):
"""Test the serve method"""
with patch.object(server._server, 'run') as mock_run:
server.serve()
mock_run.assert_called_once_with(transport="stdio")
tests/cli/__init__.py
View File
tests/cli/test_cli.py
+193
View File
@@ -0,0 +1,193 @@
import pytest
from click.testing import CliRunner
from unittest.mock import patch, MagicMock
import pytest_httpbin
from scrapling.cli import (
shell, mcp, get, post, put, delete, fetch, stealthy_fetch
)
@pytest_httpbin.use_class_based_httpbin
class TestCLI:
"""Test CLI functionality"""
@pytest.fixture
def html_url(self, httpbin):
return f"{httpbin.url}/html"
@pytest.fixture
def runner(self):
return CliRunner()
def test_shell_command(self, runner):
"""Test shell command"""
with patch('scrapling.core.shell.CustomShell') as mock_shell:
mock_instance = MagicMock()
mock_shell.return_value = mock_instance
result = runner.invoke(shell)
assert result.exit_code == 0
mock_instance.start.assert_called_once()
def test_mcp_command(self, runner):
"""Test MCP command"""
with patch('scrapling.core.ai.ScraplingMCPServer') as mock_server:
mock_instance = MagicMock()
mock_server.return_value = mock_instance
result = runner.invoke(mcp)
assert result.exit_code == 0
mock_instance.serve.assert_called_once()
def test_extract_get_command(self, runner, tmp_path, html_url):
"""Test extract `get` command"""
output_file = tmp_path / "output.md"
with patch('scrapling.fetchers.Fetcher.get') as mock_get:
mock_response = MagicMock()
mock_response.status = 200
mock_get.return_value = mock_response
with patch('scrapling.cli.Convertor.write_content_to_file'):
result = runner.invoke(
get,
[html_url, str(output_file)]
)
assert result.exit_code == 0
# Test with various options
with patch('scrapling.fetchers.Fetcher.get') as mock_get:
mock_get.return_value = mock_response
with patch('scrapling.cli.Convertor.write_content_to_file'):
result = runner.invoke(
get,
[
html_url,
str(output_file),
'-H', 'User-Agent: Test',
'--cookies', 'session=abc123',
'--timeout', '60',
'--proxy', 'http://proxy:8080',
'-s', '.content',
'-p', 'page=1'
]
)
assert result.exit_code == 0
def test_extract_post_command(self, runner, tmp_path, html_url):
"""Test extract `post` command"""
output_file = tmp_path / "output.html"
with patch('scrapling.fetchers.Fetcher.post') as mock_post:
mock_response = MagicMock()
mock_post.return_value = mock_response
with patch('scrapling.cli.Convertor.write_content_to_file'):
result = runner.invoke(
post,
[
html_url,
str(output_file),
'-d', 'key=value',
'-j', '{"data": "test"}'
]
)
assert result.exit_code == 0
def test_extract_put_command(self, runner, tmp_path, html_url):
"""Test extract `put` command"""
output_file = tmp_path / "output.html"
with patch('scrapling.fetchers.Fetcher.put') as mock_put:
mock_response = MagicMock()
mock_put.return_value = mock_response
with patch('scrapling.cli.Convertor.write_content_to_file'):
result = runner.invoke(
put,
[
html_url,
str(output_file),
'-d', 'key=value',
'-j', '{"data": "test"}'
]
)
assert result.exit_code == 0
def test_extract_delete_command(self, runner, tmp_path, html_url):
"""Test extract `delete` command"""
output_file = tmp_path / "output.html"
with patch('scrapling.fetchers.Fetcher.delete') as mock_delete:
mock_response = MagicMock()
mock_delete.return_value = mock_response
with patch('scrapling.cli.Convertor.write_content_to_file'):
result = runner.invoke(
delete,
[
html_url,
str(output_file)
]
)
assert result.exit_code == 0
def test_extract_fetch_command(self, runner, tmp_path, html_url):
"""Test extract fetch command"""
output_file = tmp_path / "output.txt"
with patch('scrapling.fetchers.DynamicFetcher.fetch') as mock_fetch:
mock_response = MagicMock()
mock_fetch.return_value = mock_response
with patch('scrapling.cli.Convertor.write_content_to_file'):
result = runner.invoke(
fetch,
[
html_url,
str(output_file),
'--headless',
'--stealth',
'--timeout', '60000'
]
)
assert result.exit_code == 0
def test_extract_stealthy_fetch_command(self, runner, tmp_path, html_url):
"""Test extract fetch command"""
output_file = tmp_path / "output.md"
with patch('scrapling.fetchers.StealthyFetcher.fetch') as mock_fetch:
mock_response = MagicMock()
mock_fetch.return_value = mock_response
with patch('scrapling.cli.Convertor.write_content_to_file'):
result = runner.invoke(
stealthy_fetch,
[
html_url,
str(output_file),
'--headless',
'--css-selector', 'body',
'--timeout', '60000'
]
)
assert result.exit_code == 0
def test_invalid_arguments(self, runner, html_url):
"""Test invalid arguments handling"""
# Missing required arguments
result = runner.invoke(get)
assert result.exit_code != 0
# Invalid output file extension
with patch('scrapling.cli.Convertor.write_content_to_file') as mock_write:
mock_write.side_effect = ValueError("Unknown file type")
_ = runner.invoke(
get,
[html_url, 'output.invalid']
)
# Should handle the error gracefully
tests/cli/test_shell_functionality.py
+200
View File
@@ -0,0 +1,200 @@
import pytest
from unittest.mock import patch, MagicMock
from scrapling.parser import Selector
from scrapling.core.shell import CustomShell, CurlParser, Convertor
class TestCurlParser:
"""Test curl command parsing"""
@pytest.fixture
def parser(self):
return CurlParser()
def test_basic_curl_parse(self, parser):
"""Test parsing basic curl commands"""
# Simple GET
curl_cmd = 'curl https://example.com'
request = parser.parse(curl_cmd)
assert request.url == 'https://example.com'
assert request.method == 'get'
assert request.data is None
def test_curl_with_headers(self, parser):
"""Test parsing curl with headers"""
curl_cmd = '''curl https://example.com \
-H "User-Agent: Mozilla/5.0" \
-H "Accept: application/json"'''
request = parser.parse(curl_cmd)
assert request.headers['User-Agent'] == 'Mozilla/5.0'
assert request.headers['Accept'] == 'application/json'
def test_curl_with_data(self, parser):
"""Test parsing curl with data"""
# Form data
curl_cmd = 'curl https://example.com -X POST -d "key=value&foo=bar"'
request = parser.parse(curl_cmd)
assert request.method == 'post'
assert request.data == 'key=value&foo=bar'
# JSON data
curl_cmd = """curl https://example.com -X POST --data-raw '{"key": "value"}'"""
request = parser.parse(curl_cmd)
assert request.json_data == {"key": "value"}
def test_curl_with_cookies(self, parser):
"""Test parsing curl with cookies"""
curl_cmd = '''curl https://example.com \
-H "Cookie: session=abc123; user=john" \
-b "extra=cookie"'''
request = parser.parse(curl_cmd)
assert request.cookies['session'] == 'abc123'
assert request.cookies['user'] == 'john'
assert request.cookies['extra'] == 'cookie'
def test_curl_with_proxy(self, parser):
"""Test parsing curl with proxy"""
curl_cmd = 'curl https://example.com -x http://proxy:8080 -U user:pass'
request = parser.parse(curl_cmd)
assert 'http://user:pass@proxy:8080' in request.proxy['http']
def test_curl2fetcher(self, parser):
"""Test converting curl to fetcher request"""
with patch('scrapling.fetchers.Fetcher.get') as mock_get:
mock_response = MagicMock()
mock_get.return_value = mock_response
curl_cmd = 'curl https://example.com'
_ = parser.convert2fetcher(curl_cmd)
mock_get.assert_called_once()
def test_invalid_curl_commands(self, parser):
"""Test handling invalid curl commands"""
# Invalid format
with pytest.raises(AttributeError):
parser.parse('not a curl command')
class TestConvertor:
"""Test content conversion functionality"""
@pytest.fixture
def sample_html(self):
return """
<html>
<body>
<div class="content">
<h1>Title</h1>
<p>Some text content</p>
</div>
</body>
</html>
"""
def test_extract_markdown(self, sample_html):
"""Test extracting content as Markdown"""
page = Selector(sample_html)
content = list(Convertor._extract_content(page, "markdown"))
assert len(content) > 0
assert "Title\n=====" in content[0] # Markdown conversion
def test_extract_html(self, sample_html):
"""Test extracting content as HTML"""
page = Selector(sample_html)
content = list(Convertor._extract_content(page, "html"))
assert len(content) > 0
assert "<h1>Title</h1>" in content[0]
def test_extract_text(self, sample_html):
"""Test extracting content as plain text"""
page = Selector(sample_html)
content = list(Convertor._extract_content(page, "text"))
assert len(content) > 0
assert "Title" in content[0]
assert "Some text content" in content[0]
def test_extract_with_selector(self, sample_html):
"""Test extracting with CSS selector"""
page = Selector(sample_html)
content = list(Convertor._extract_content(
page,
"text",
css_selector=".content"
))
assert len(content) > 0
def test_write_to_file(self, sample_html, tmp_path):
"""Test writing content to files"""
page = Selector(sample_html)
# Test markdown
md_file = tmp_path / "output.md"
Convertor.write_content_to_file(page, str(md_file))
assert md_file.exists()
# Test HTML
html_file = tmp_path / "output.html"
Convertor.write_content_to_file(page, str(html_file))
assert html_file.exists()
# Test text
txt_file = tmp_path / "output.txt"
Convertor.write_content_to_file(page, str(txt_file))
assert txt_file.exists()
def test_invalid_operations(self, sample_html):
"""Test error handling in convertor"""
page = Selector(sample_html)
# Invalid extraction type
with pytest.raises(ValueError):
list(Convertor._extract_content(page, "invalid"))
# Invalid filename
with pytest.raises(ValueError):
Convertor.write_content_to_file(page, "")
# Unknown file extension
with pytest.raises(ValueError):
Convertor.write_content_to_file(page, "output.xyz")
class TestCustomShell:
"""Test interactive shell functionality"""
def test_shell_initialization(self):
"""Test shell initialization"""
with patch('scrapling.core.shell.InteractiveShellEmbed'):
shell = CustomShell(code="", log_level="debug")
assert shell.log_level == 10 # DEBUG level
assert shell.page is None
assert len(shell.pages) == 0
def test_shell_namespace(self):
"""Test shell namespace creation"""
with patch('scrapling.core.shell.InteractiveShellEmbed'):
shell = CustomShell(code="")
namespace = shell.get_namespace()
# Check all expected functions/classes are available
assert 'get' in namespace
assert 'post' in namespace
assert 'Fetcher' in namespace
assert 'DynamicFetcher' in namespace
assert 'view' in namespace
assert 'uncurl' in namespace
tests/core/__init__.py
View File
tests/core/test_shell_core.py
+243
View File
@@ -0,0 +1,243 @@
import pytest
from scrapling.core.shell import (
_CookieParser,
_ParseHeaders,
Request,
_known_logging_levels,
)
class TestCookieParser:
"""Test cookie parsing functionality"""
def test_simple_cookie_parsing(self):
"""Test parsing a simple cookie"""
cookie_string = "session_id=abc123"
cookies = list(_CookieParser(cookie_string))
assert len(cookies) == 1
assert cookies[0] == ("session_id", "abc123")
def test_multiple_cookies_parsing(self):
"""Test parsing multiple cookies"""
cookie_string = "session_id=abc123; theme=dark; lang=en"
cookies = list(_CookieParser(cookie_string))
assert len(cookies) == 3
cookie_dict = dict(cookies)
assert cookie_dict["session_id"] == "abc123"
assert cookie_dict["theme"] == "dark"
assert cookie_dict["lang"] == "en"
def test_cookie_with_attributes(self):
"""Test parsing cookies with attributes"""
cookie_string = "session_id=abc123; Path=/; HttpOnly; Secure"
cookies = list(_CookieParser(cookie_string))
assert len(cookies) == 1
assert cookies[0] == ("session_id", "abc123")
def test_empty_cookie_string(self):
"""Test parsing empty cookie string"""
cookies = list(_CookieParser(""))
assert len(cookies) == 0
def test_malformed_cookie_handling(self):
"""Test handling of malformed cookies"""
# Should not raise exception but may return an empty list
cookies = list(_CookieParser("invalid_cookie_format"))
assert isinstance(cookies, list)
class TestParseHeaders:
"""Test header parsing functionality"""
def test_simple_headers(self):
"""Test parsing simple headers"""
header_lines = [
"Content-Type: text/html",
"Content-Length: 1234",
"User-Agent: TestAgent/1.0"
]
headers, cookies = _ParseHeaders(header_lines)
assert headers["Content-Type"] == "text/html"
assert headers["Content-Length"] == "1234"
assert headers["User-Agent"] == "TestAgent/1.0"
assert len(cookies) == 0
def test_headers_with_cookies(self):
"""Test parsing headers with cookie headers"""
header_lines = [
"Content-Type: text/html",
"Set-Cookie: session_id=abc123",
"Set-Cookie: theme=dark; Path=/",
]
headers, cookies = _ParseHeaders(header_lines)
assert headers["Content-Type"] == "text/html"
assert "Set-Cookie" in headers # Should contain the first Set-Cookie
# Cookie parsing behavior depends on implementation
def test_headers_without_colons(self):
"""Test headers without colons"""
header_lines = [
"Content-Type: text/html",
"InvalidHeader;", # Header ending with semicolon
]
headers, cookies = _ParseHeaders(header_lines)
assert headers["Content-Type"] == "text/html"
assert "InvalidHeader" in headers
assert headers["InvalidHeader"] == ""
def test_invalid_header_format(self):
"""Test invalid header format raises error"""
header_lines = [
"Content-Type: text/html",
"InvalidHeaderWithoutColon", # No colon, no semicolon
]
with pytest.raises(ValueError, match="Could not parse header without colon"):
_ParseHeaders(header_lines)
def test_headers_with_multiple_colons(self):
"""Test headers with multiple colons"""
header_lines = [
"Authorization: Bearer: token123",
"X-Custom: value:with:colons",
]
headers, cookies = _ParseHeaders(header_lines)
assert headers["Authorization"] == "Bearer: token123"
assert headers["X-Custom"] == "value:with:colons"
def test_headers_with_whitespace(self):
"""Test headers with extra whitespace"""
header_lines = [
" Content-Type : text/html ",
"\tUser-Agent\t:\tTestAgent/1.0\t",
]
headers, cookies = _ParseHeaders(header_lines)
# Should handle whitespace correctly
assert "Content-Type" in headers or " Content-Type " in headers
assert "text/html" in str(headers.values()) or " text/html " in str(headers.values())
def test_parse_cookies_disabled(self):
"""Test parsing with cookies disabled"""
header_lines = [
"Content-Type: text/html",
"Set-Cookie: session_id=abc123",
]
headers, cookies = _ParseHeaders(header_lines, parse_cookies=False)
assert headers["Content-Type"] == "text/html"
# Cookie parsing behavior when disabled
assert len(cookies) == 0 or "Set-Cookie" in headers
def test_empty_header_lines(self):
"""Test parsing empty header lines"""
headers, cookies = _ParseHeaders([])
assert len(headers) == 0
assert len(cookies) == 0
class TestRequestNamedTuple:
"""Test Request namedtuple functionality"""
def test_request_creation(self):
"""Test creating Request namedtuple"""
request = Request(
method="GET",
url="https://example.com",
params={"q": "test"},
data=None,
json_data=None,
headers={"User-Agent": "Test"},
cookies={"session": "abc123"},
proxy=None,
follow_redirects=True
)
assert request.method == "GET"
assert request.url == "https://example.com"
assert request.params == {"q": "test"}
assert request.headers == {"User-Agent": "Test"}
assert request.follow_redirects is True
def test_request_defaults(self):
"""Test Request with default/None values"""
request = Request(
method="POST",
url="https://api.example.com",
params=None,
data='{"key": "value"}',
json_data={"key": "value"},
headers={},
cookies={},
proxy="http://proxy:8080",
follow_redirects=False
)
assert request.method == "POST"
assert request.data == '{"key": "value"}'
assert request.json_data == {"key": "value"}
assert request.proxy == "http://proxy:8080"
assert request.follow_redirects is False
def test_request_field_access(self):
"""Test accessing Request fields"""
request = Request(
"GET", "https://example.com", {}, None, None, {}, {}, None, True
)
# Test field access by name
assert hasattr(request, 'method')
assert hasattr(request, 'url')
assert hasattr(request, 'params')
assert hasattr(request, 'data')
assert hasattr(request, 'json_data')
assert hasattr(request, 'headers')
assert hasattr(request, 'cookies')
assert hasattr(request, 'proxy')
assert hasattr(request, 'follow_redirects')
# Test field access by index
assert request[0] == "GET"
assert request[1] == "https://example.com"
class TestLoggingLevels:
"""Test logging level constants"""
def test_known_logging_levels(self):
"""Test that all known logging levels are defined"""
expected_levels = ["debug", "info", "warning", "error", "critical", "fatal"]
for level in expected_levels:
assert level in _known_logging_levels
assert isinstance(_known_logging_levels[level], int)
def test_logging_level_values(self):
"""Test logging level values are correct"""
from logging import DEBUG, INFO, WARNING, ERROR, CRITICAL, FATAL
assert _known_logging_levels["debug"] == DEBUG
assert _known_logging_levels["info"] == INFO
assert _known_logging_levels["warning"] == WARNING
assert _known_logging_levels["error"] == ERROR
assert _known_logging_levels["critical"] == CRITICAL
assert _known_logging_levels["fatal"] == FATAL
def test_level_hierarchy(self):
"""Test that logging levels have correct hierarchy"""
levels = [
_known_logging_levels["debug"],
_known_logging_levels["info"],
_known_logging_levels["warning"],
_known_logging_levels["error"],
_known_logging_levels["critical"],
]
# Levels should be in ascending order
for i in range(len(levels) - 1):
assert levels[i] < levels[i + 1]
tests/core/test_storage_core.py
+37
View File
@@ -0,0 +1,37 @@
import tempfile
import os
from scrapling.core.storage import SQLiteStorageSystem
class TestSQLiteStorageSystem:
"""Test SQLiteStorageSystem functionality"""
def test_sqlite_storage_creation(self):
"""Test SQLite storage system creation"""
# Use an in-memory database for testing
storage = SQLiteStorageSystem(storage_file=":memory:")
assert storage is not None
def test_sqlite_storage_with_file(self):
"""Test SQLite storage with an actual file"""
with tempfile.NamedTemporaryFile(suffix='.db', delete=False) as tmp_file:
db_path = tmp_file.name
try:
storage = SQLiteStorageSystem(storage_file=db_path)
assert storage is not None
assert os.path.exists(db_path)
finally:
if os.path.exists(db_path):
os.unlink(db_path)
def test_sqlite_storage_initialization_args(self):
"""Test SQLite storage with various initialization arguments"""
# Test with URL parameter
storage = SQLiteStorageSystem(
storage_file=":memory:",
url="https://example.com"
)
assert storage is not None
assert storage.url == "https://example.com"
tests/fetchers/async/test_camoufox.py
+57 -60
View File
@@ -3,7 +3,7 @@ import pytest_httpbin
from scrapling import StealthyFetcher
StealthyFetcher.auto_match = True
StealthyFetcher.adaptive = True
@pytest_httpbin.use_class_based_httpbin
@@ -17,46 +17,34 @@ class TestStealthyFetcher:
def urls(self, httpbin):
url = httpbin.url
return {
'status_200': f'{url}/status/200',
'status_404': f'{url}/status/404',
'status_501': f'{url}/status/501',
'basic_url': f'{url}/get',
'html_url': f'{url}/html',
'delayed_url': f'{url}/delay/10', # 10 Seconds delay response
'cookies_url': f"{url}/cookies/set/test/value"
"status_200": f"{url}/status/200",
"status_404": f"{url}/status/404",
"status_501": f"{url}/status/501",
"basic_url": f"{url}/get",
"html_url": f"{url}/html",
"delayed_url": f"{url}/delay/10", # 10 Seconds delay response
"cookies_url": f"{url}/cookies/set/test/value",
"cloudflare_url": "https://nopecha.com/demo/cloudflare", # Interactive turnstile page
}
async def test_cloudflare_fetch(self, fetcher, urls):
"""Test if Cloudflare bypass is working"""
assert (await fetcher.async_fetch(urls["cloudflare_url"], solve_cloudflare=True)).status == 200
async def test_basic_fetch(self, fetcher, urls):
"""Test doing basic fetch request with multiple statuses"""
assert (await fetcher.async_fetch(urls['status_200'])).status == 200
assert (await fetcher.async_fetch(urls['status_404'])).status == 404
assert (await fetcher.async_fetch(urls['status_501'])).status == 501
async def test_networkidle(self, fetcher, urls):
"""Test if waiting for `networkidle` make page does not finish loading or not"""
assert (await fetcher.async_fetch(urls['basic_url'], network_idle=True)).status == 200
async def test_blocking_resources(self, fetcher, urls):
"""Test if blocking resources make page does not finish loading or not"""
assert (await fetcher.async_fetch(urls['basic_url'], block_images=True)).status == 200
assert (await fetcher.async_fetch(urls['basic_url'], disable_resources=True)).status == 200
async def test_waiting_selector(self, fetcher, urls):
"""Test if waiting for a selector make page does not finish loading or not"""
assert (await fetcher.async_fetch(urls['html_url'], wait_selector='h1')).status == 200
assert (await fetcher.async_fetch(
urls['html_url'],
wait_selector='h1',
wait_selector_state='visible'
)).status == 200
"""Test doing a basic fetch request with multiple statuses"""
assert (await fetcher.async_fetch(urls["status_200"])).status == 200
assert (await fetcher.async_fetch(urls["status_404"])).status == 404
assert (await fetcher.async_fetch(urls["status_501"])).status == 501
async def test_cookies_loading(self, fetcher, urls):
"""Test if cookies are set after the request"""
response = await fetcher.async_fetch(urls['cookies_url'])
assert response.cookies == {'test': 'value'}
response = await fetcher.async_fetch(urls["cookies_url"])
cookies = {response.cookies[0]['name']: response.cookies[0]['value']}
assert cookies == {"test": "value"}
async def test_automation(self, fetcher, urls):
"""Test if automation break the code or not"""
"""Test if automation breaks the code or not"""
async def scroll_page(page):
await page.mouse.wheel(10, 0)
@@ -64,34 +52,43 @@ class TestStealthyFetcher:
await page.mouse.up()
return page
assert (await fetcher.async_fetch(urls['html_url'], page_action=scroll_page)).status == 200
assert (
await fetcher.async_fetch(urls["html_url"], page_action=scroll_page, humanize=True)
).status == 200
async def test_properties(self, fetcher, urls):
"""Test if different arguments breaks the code or not"""
assert (await fetcher.async_fetch(
urls['html_url'],
block_webrtc=True,
allow_webgl=True
)).status == 200
assert (await fetcher.async_fetch(
urls['html_url'],
block_webrtc=False,
allow_webgl=True
)).status == 200
assert (await fetcher.async_fetch(
urls['html_url'],
block_webrtc=True,
allow_webgl=False
)).status == 200
assert (await fetcher.async_fetch(
urls['html_url'],
extra_headers={'ayo': ''},
os_randomize=True
)).status == 200
@pytest.mark.parametrize(
"kwargs",
[
{"block_webrtc": True, "allow_webgl": True, "disable_ads": False},
{"block_webrtc": False, "allow_webgl": True, "block_images": True},
{"block_webrtc": True, "allow_webgl": False, "disable_resources": True},
{"block_images": True, "disable_resources": True, },
{"wait_selector": "h1", "wait_selector_state": "attached"},
{"wait_selector": "h1", "wait_selector_state": "visible"},
{
"network_idle": True,
"wait": 10,
"cookies": [{"name": "test", "value": "123", "domain": "example.com", "path": "/"}],
"google_search": True,
"extra_headers": {"ayo": ""},
"os_randomize": True,
"disable_ads": True,
# "geoip": True,
"custom_config": {"keep_comments": False, "keep_cdata": False},
"additional_args": {"window": (1920, 1080)},
},
],
)
async def test_properties(self, fetcher, urls, kwargs):
"""Test if different arguments break the code or not"""
response = await fetcher.async_fetch(
urls["html_url"],
**kwargs
)
assert response.status == 200
async def test_infinite_timeout(self, fetcher, urls):
"""Test if infinite timeout breaks the code or not"""
assert (await fetcher.async_fetch(urls['delayed_url'], timeout=None)).status == 200
assert (
await fetcher.async_fetch(urls["delayed_url"], timeout=0)
).status == 200
tests/fetchers/async/test_camoufox_session.py
+85
View File
@@ -0,0 +1,85 @@
import pytest
import asyncio
import pytest_httpbin
from scrapling.engines import AsyncStealthySession
@pytest_httpbin.use_class_based_httpbin
@pytest.mark.asyncio
class TestAsyncStealthySession:
"""Test AsyncStealthySession"""
# The `AsyncStealthySession` is inheriting from `StealthySession` class so no need to repeat all the tests
@pytest.fixture
def urls(self, httpbin):
return {
"basic": f"{httpbin.url}/get",
"html": f"{httpbin.url}/html",
}
async def test_concurrent_async_requests(self, urls):
"""Test concurrent requests with async session"""
async with AsyncStealthySession(max_pages=3) as session:
# Launch multiple concurrent requests
tasks = [
session.fetch(urls["basic"]),
session.fetch(urls["html"]),
session.fetch(urls["basic"])
]
assert session.max_pages == 3
assert session.page_pool.max_pages == 3
assert session.context is not None
responses = await asyncio.gather(*tasks)
# All should succeed
assert all(r.status == 200 for r in responses)
# Check pool stats
stats = session.get_pool_stats()
assert stats["total_pages"] <= 3
# After exit, should be closed
assert session._closed is True
# Should raise RuntimeError when used after closing
with pytest.raises(RuntimeError):
await session.fetch(urls["basic"])
async def test_page_pool_management(self, urls):
"""Test page pool creation and reuse"""
async with AsyncStealthySession() as session:
# The first request creates a page
_ = await session.fetch(urls["basic"])
assert session.page_pool.pages_count == 1
# The second request should reuse the page
_ = await session.fetch(urls["html"])
assert session.page_pool.pages_count == 1
# Check pool stats
stats = session.get_pool_stats()
assert stats["total_pages"] == 1
assert stats["max_pages"] == 1
async def test_stealthy_session_with_options(self, urls):
"""Test AsyncStealthySession with various options"""
async with AsyncStealthySession(
max_pages=1,
block_images=True,
disable_ads=True,
humanize=True
) as session:
response = await session.fetch(urls["html"])
assert response.status == 200
async def test_error_handling_in_fetch(self, urls):
"""Test error handling during fetch"""
async with AsyncStealthySession() as session:
# Test with invalid URL
with pytest.raises(Exception):
await session.fetch("invalid://url")
tests/fetchers/async/test_dynamic.py
+98
View File
@@ -0,0 +1,98 @@
import pytest
import pytest_httpbin
from scrapling import DynamicFetcher
DynamicFetcher.adaptive = True
@pytest_httpbin.use_class_based_httpbin
class TestDynamicFetcherAsync:
@pytest.fixture
def fetcher(self):
return DynamicFetcher
@pytest.fixture
def urls(self, httpbin):
return {
"status_200": f"{httpbin.url}/status/200",
"status_404": f"{httpbin.url}/status/404",
"status_501": f"{httpbin.url}/status/501",
"basic_url": f"{httpbin.url}/get",
"html_url": f"{httpbin.url}/html",
"delayed_url": f"{httpbin.url}/delay/10",
"cookies_url": f"{httpbin.url}/cookies/set/test/value",
}
@pytest.mark.asyncio
async def test_basic_fetch(self, fetcher, urls):
"""Test doing a basic fetch request with multiple statuses"""
response = await fetcher.async_fetch(urls["status_200"])
assert response.status == 200
@pytest.mark.asyncio
async def test_cookies_loading(self, fetcher, urls):
"""Test if cookies are set after the request"""
response = await fetcher.async_fetch(urls["cookies_url"])
cookies = {response.cookies[0]['name']: response.cookies[0]['value']}
assert cookies == {"test": "value"}
@pytest.mark.asyncio
async def test_automation(self, fetcher, urls):
"""Test if automation breaks the code or not"""
async def scroll_page(page):
await page.mouse.wheel(10, 0)
await page.mouse.move(100, 400)
await page.mouse.up()
return page
response = await fetcher.async_fetch(urls["html_url"], page_action=scroll_page)
assert response.status == 200
@pytest.mark.parametrize(
"kwargs",
[
{"disable_webgl": True, "hide_canvas": False},
{"disable_webgl": False, "hide_canvas": True, "disable_resources": True},
{"stealth": True}, # causes issues with GitHub Actions
{"wait_selector": "h1", "wait_selector_state": "attached"},
{"wait_selector": "h1", "wait_selector_state": "visible"},
{
"google_search": True,
"real_chrome": True,
"wait": 10,
"locale": "en-US",
"extra_headers": {"ayo": ""},
"useragent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:131.0) Gecko/20100101 Firefox/131.0",
"cookies": [{"name": "test", "value": "123", "domain": "example.com", "path": "/"}],
"network_idle": True,
"custom_config": {"keep_comments": False, "keep_cdata": False},
},
],
)
@pytest.mark.asyncio
async def test_properties(self, fetcher, urls, kwargs):
"""Test if different arguments break the code or not"""
response = await fetcher.async_fetch(urls["html_url"], **kwargs)
assert response.status == 200
@pytest.mark.asyncio
async def test_cdp_url_invalid(self, fetcher, urls):
"""Test if invalid CDP URLs raise appropriate exceptions"""
with pytest.raises(TypeError):
await fetcher.async_fetch(urls["html_url"], cdp_url="blahblah")
with pytest.raises(TypeError):
await fetcher.async_fetch(
urls["html_url"], cdp_url="blahblah", nstbrowser_mode=True
)
with pytest.raises(Exception):
await fetcher.async_fetch(urls["html_url"], cdp_url="ws://blahblah")
@pytest.mark.asyncio
async def test_infinite_timeout(self, fetcher, urls):
"""Test if infinite timeout breaks the code or not"""
response = await fetcher.async_fetch(urls["delayed_url"], timeout=0)
assert response.status == 200
tests/fetchers/async/test_dynamic_session.py
+84
View File
@@ -0,0 +1,84 @@
import pytest
import asyncio
import pytest_httpbin
from scrapling.engines import AsyncDynamicSession
@pytest_httpbin.use_class_based_httpbin
@pytest.mark.asyncio
class TestAsyncDynamicSession:
"""Test AsyncDynamicSession"""
# The `AsyncDynamicSession` is inheriting from `DynamicSession` class so no need to repeat all the tests
@pytest.fixture
def urls(self, httpbin):
return {
"basic": f"{httpbin.url}/get",
"html": f"{httpbin.url}/html",
}
async def test_concurrent_async_requests(self, urls):
"""Test concurrent requests with async session"""
async with AsyncDynamicSession(max_pages=3) as session:
# Launch multiple concurrent requests
tasks = [
session.fetch(urls["basic"]),
session.fetch(urls["html"]),
session.fetch(urls["basic"])
]
assert session.max_pages == 3
assert session.page_pool.max_pages == 3
assert session.context is not None
responses = await asyncio.gather(*tasks)
# All should succeed
assert all(r.status == 200 for r in responses)
# Check pool stats
stats = session.get_pool_stats()
assert stats["total_pages"] <= 3
# After exit, should be closed
assert session._closed is True
# Should raise RuntimeError when used after closing
with pytest.raises(RuntimeError):
await session.fetch(urls["basic"])
async def test_page_pool_management(self, urls):
"""Test page pool creation and reuse"""
async with AsyncDynamicSession() as session:
# The first request creates a page
_ = await session.fetch(urls["basic"])
assert session.page_pool.pages_count == 1
# The second request should reuse the page
_ = await session.fetch(urls["html"])
assert session.page_pool.pages_count == 1
# Check pool stats
stats = session.get_pool_stats()
assert stats["total_pages"] == 1
assert stats["max_pages"] == 1
async def test_dynamic_session_with_options(self, urls):
"""Test AsyncDynamicSession with various options"""
async with AsyncDynamicSession(
headless=False,
stealth=True,
disable_resources=True,
extra_headers={"X-Test": "value"}
) as session:
response = await session.fetch(urls["html"])
assert response.status == 200
async def test_error_handling_in_fetch(self, urls):
"""Test error handling during fetch"""
async with AsyncDynamicSession() as session:
# Test with invalid URL
with pytest.raises(Exception):
await session.fetch("invalid://url")
tests/fetchers/async/test_httpx.py
-85
View File
@@ -1,85 +0,0 @@
import pytest
import pytest_httpbin
from scrapling.fetchers import AsyncFetcher
AsyncFetcher.auto_match = True
@pytest_httpbin.use_class_based_httpbin
@pytest.mark.asyncio
class TestAsyncFetcher:
@pytest.fixture(scope="class")
def fetcher(self):
return AsyncFetcher
@pytest.fixture(scope="class")
def urls(self, httpbin):
return {
'status_200': f'{httpbin.url}/status/200',
'status_404': f'{httpbin.url}/status/404',
'status_501': f'{httpbin.url}/status/501',
'basic_url': f'{httpbin.url}/get',
'post_url': f'{httpbin.url}/post',
'put_url': f'{httpbin.url}/put',
'delete_url': f'{httpbin.url}/delete',
'html_url': f'{httpbin.url}/html'
}
async def test_basic_get(self, fetcher, urls):
"""Test doing basic get request with multiple statuses"""
assert (await fetcher.get(urls['status_200'])).status == 200
assert (await fetcher.get(urls['status_404'])).status == 404
assert (await fetcher.get(urls['status_501'])).status == 501
async def test_get_properties(self, fetcher, urls):
"""Test if different arguments with GET request breaks the code or not"""
assert (await fetcher.get(urls['status_200'], stealthy_headers=True)).status == 200
assert (await fetcher.get(urls['status_200'], follow_redirects=True)).status == 200
assert (await fetcher.get(urls['status_200'], timeout=None)).status == 200
assert (await fetcher.get(
urls['status_200'],
stealthy_headers=True,
follow_redirects=True,
timeout=None
)).status == 200
async def test_post_properties(self, fetcher, urls):
"""Test if different arguments with POST request breaks the code or not"""
assert (await fetcher.post(urls['post_url'], data={'key': 'value'})).status == 200
assert (await fetcher.post(urls['post_url'], data={'key': 'value'}, stealthy_headers=True)).status == 200
assert (await fetcher.post(urls['post_url'], data={'key': 'value'}, follow_redirects=True)).status == 200
assert (await fetcher.post(urls['post_url'], data={'key': 'value'}, timeout=None)).status == 200
assert (await fetcher.post(
urls['post_url'],
data={'key': 'value'},
stealthy_headers=True,
follow_redirects=True,
timeout=None
)).status == 200
async def test_put_properties(self, fetcher, urls):
"""Test if different arguments with PUT request breaks the code or not"""
assert (await fetcher.put(urls['put_url'], data={'key': 'value'})).status in [200, 405]
assert (await fetcher.put(urls['put_url'], data={'key': 'value'}, stealthy_headers=True)).status in [200, 405]
assert (await fetcher.put(urls['put_url'], data={'key': 'value'}, follow_redirects=True)).status in [200, 405]
assert (await fetcher.put(urls['put_url'], data={'key': 'value'}, timeout=None)).status in [200, 405]
assert (await fetcher.put(
urls['put_url'],
data={'key': 'value'},
stealthy_headers=True,
follow_redirects=True,
timeout=None
)).status in [200, 405]
async def test_delete_properties(self, fetcher, urls):
"""Test if different arguments with DELETE request breaks the code or not"""
assert (await fetcher.delete(urls['delete_url'], stealthy_headers=True)).status == 200
assert (await fetcher.delete(urls['delete_url'], follow_redirects=True)).status == 200
assert (await fetcher.delete(urls['delete_url'], timeout=None)).status == 200
assert (await fetcher.delete(
urls['delete_url'],
stealthy_headers=True,
follow_redirects=True,
timeout=None
)).status == 200
tests/fetchers/async/test_playwright.py
-101
View File
@@ -1,101 +0,0 @@
import pytest
import pytest_httpbin
from scrapling import PlayWrightFetcher
PlayWrightFetcher.auto_match = True
@pytest_httpbin.use_class_based_httpbin
class TestPlayWrightFetcherAsync:
@pytest.fixture
def fetcher(self):
return PlayWrightFetcher
@pytest.fixture
def urls(self, httpbin):
return {
'status_200': f'{httpbin.url}/status/200',
'status_404': f'{httpbin.url}/status/404',
'status_501': f'{httpbin.url}/status/501',
'basic_url': f'{httpbin.url}/get',
'html_url': f'{httpbin.url}/html',
'delayed_url': f'{httpbin.url}/delay/10',
'cookies_url': f"{httpbin.url}/cookies/set/test/value"
}
@pytest.mark.asyncio
async def test_basic_fetch(self, fetcher, urls):
"""Test doing basic fetch request with multiple statuses"""
response = await fetcher.async_fetch(urls['status_200'])
assert response.status == 200
@pytest.mark.asyncio
async def test_networkidle(self, fetcher, urls):
"""Test if waiting for `networkidle` make page does not finish loading or not"""
response = await fetcher.async_fetch(urls['basic_url'], network_idle=True)
assert response.status == 200
@pytest.mark.asyncio
async def test_blocking_resources(self, fetcher, urls):
"""Test if blocking resources make page does not finish loading or not"""
response = await fetcher.async_fetch(urls['basic_url'], disable_resources=True)
assert response.status == 200
@pytest.mark.asyncio
async def test_waiting_selector(self, fetcher, urls):
"""Test if waiting for a selector make page does not finish loading or not"""
response1 = await fetcher.async_fetch(urls['html_url'], wait_selector='h1')
assert response1.status == 200
response2 = await fetcher.async_fetch(urls['html_url'], wait_selector='h1', wait_selector_state='visible')
assert response2.status == 200
@pytest.mark.asyncio
async def test_cookies_loading(self, fetcher, urls):
"""Test if cookies are set after the request"""
response = await fetcher.async_fetch(urls['cookies_url'])
assert response.cookies == {'test': 'value'}
@pytest.mark.asyncio
async def test_automation(self, fetcher, urls):
"""Test if automation break the code or not"""
async def scroll_page(page):
await page.mouse.wheel(10, 0)
await page.mouse.move(100, 400)
await page.mouse.up()
return page
response = await fetcher.async_fetch(urls['html_url'], page_action=scroll_page)
assert response.status == 200
@pytest.mark.parametrize("kwargs", [
{"disable_webgl": True, "hide_canvas": False},
{"disable_webgl": False, "hide_canvas": True},
# {"stealth": True}, # causes issues with Github Actions
{"useragent": 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:131.0) Gecko/20100101 Firefox/131.0'},
{"extra_headers": {'ayo': ''}}
])
@pytest.mark.asyncio
async def test_properties(self, fetcher, urls, kwargs):
"""Test if different arguments breaks the code or not"""
response = await fetcher.async_fetch(urls['html_url'], **kwargs)
assert response.status == 200
@pytest.mark.asyncio
async def test_cdp_url_invalid(self, fetcher, urls):
"""Test if invalid CDP URLs raise appropriate exceptions"""
with pytest.raises(ValueError):
await fetcher.async_fetch(urls['html_url'], cdp_url='blahblah')
with pytest.raises(ValueError):
await fetcher.async_fetch(urls['html_url'], cdp_url='blahblah', nstbrowser_mode=True)
with pytest.raises(Exception):
await fetcher.async_fetch(urls['html_url'], cdp_url='ws://blahblah')
@pytest.mark.asyncio
async def test_infinite_timeout(self, fetcher, urls):
"""Test if infinite timeout breaks the code or not"""
response = await fetcher.async_fetch(urls['delayed_url'], timeout=None)
assert response.status == 200
tests/fetchers/async/test_requests.py
+126
View File
@@ -0,0 +1,126 @@
import pytest
import pytest_httpbin
from scrapling.fetchers import AsyncFetcher
AsyncFetcher.adaptive = True
@pytest_httpbin.use_class_based_httpbin
@pytest.mark.asyncio
class TestAsyncFetcher:
@pytest.fixture(scope="class")
def fetcher(self):
return AsyncFetcher
@pytest.fixture(scope="class")
def urls(self, httpbin):
return {
"status_200": f"{httpbin.url}/status/200",
"status_404": f"{httpbin.url}/status/404",
"status_501": f"{httpbin.url}/status/501",
"basic_url": f"{httpbin.url}/get",
"post_url": f"{httpbin.url}/post",
"put_url": f"{httpbin.url}/put",
"delete_url": f"{httpbin.url}/delete",
"html_url": f"{httpbin.url}/html",
}
async def test_basic_get(self, fetcher, urls):
"""Test doing basic get request with multiple statuses"""
assert (await fetcher.get(urls["status_200"])).status == 200
assert (await fetcher.get(urls["status_404"])).status == 404
assert (await fetcher.get(urls["status_501"])).status == 501
async def test_get_properties(self, fetcher, urls):
"""Test if different arguments with the GET request break the code or not"""
assert (
await fetcher.get(urls["status_200"], stealthy_headers=True)
).status == 200
assert (
await fetcher.get(urls["status_200"], follow_redirects=True)
).status == 200
assert (await fetcher.get(urls["status_200"], timeout=None)).status == 200
assert (
await fetcher.get(
urls["status_200"],
stealthy_headers=True,
follow_redirects=True,
timeout=None,
)
).status == 200
async def test_post_properties(self, fetcher, urls):
"""Test if different arguments with the POST request break the code or not"""
assert (
await fetcher.post(urls["post_url"], data={"key": "value"})
).status == 200
assert (
await fetcher.post(
urls["post_url"], data={"key": "value"}, stealthy_headers=True
)
).status == 200
assert (
await fetcher.post(
urls["post_url"], data={"key": "value"}, follow_redirects=True
)
).status == 200
assert (
await fetcher.post(urls["post_url"], data={"key": "value"}, timeout=None)
).status == 200
assert (
await fetcher.post(
urls["post_url"],
data={"key": "value"},
stealthy_headers=True,
follow_redirects=True,
timeout=None,
)
).status == 200
async def test_put_properties(self, fetcher, urls):
"""Test if different arguments with a PUT request break the code or not"""
assert (await fetcher.put(urls["put_url"], data={"key": "value"})).status in [
200,
405,
]
assert (
await fetcher.put(
urls["put_url"], data={"key": "value"}, stealthy_headers=True
)
).status in [200, 405]
assert (
await fetcher.put(
urls["put_url"], data={"key": "value"}, follow_redirects=True
)
).status in [200, 405]
assert (
await fetcher.put(urls["put_url"], data={"key": "value"}, timeout=None)
).status in [200, 405]
assert (
await fetcher.put(
urls["put_url"],
data={"key": "value"},
stealthy_headers=True,
follow_redirects=True,
timeout=None,
)
).status in [200, 405]
async def test_delete_properties(self, fetcher, urls):
"""Test if different arguments with the DELETE request break the code or not"""
assert (
await fetcher.delete(urls["delete_url"], stealthy_headers=True)
).status == 200
assert (
await fetcher.delete(urls["delete_url"], follow_redirects=True)
).status == 200
assert (await fetcher.delete(urls["delete_url"], timeout=None)).status == 200
assert (
await fetcher.delete(
urls["delete_url"],
stealthy_headers=True,
follow_redirects=True,
timeout=None,
)
).status == 200
tests/fetchers/async/test_requests_session.py
+17
View File
@@ -0,0 +1,17 @@
import pytest
from scrapling.engines.static import AsyncFetcherClient
class TestFetcherSession:
"""Test FetcherSession functionality"""
def test_async_fetcher_client_creation(self):
"""Test AsyncFetcherClient creation"""
client = AsyncFetcherClient()
# Should not have context manager methods
assert client.__aenter__ is None
assert client.__aexit__ is None
assert client._async_curl_session is True # Special marker
tests/fetchers/sync/test_camoufox.py
+50 -32
View File
@@ -2,8 +2,7 @@ import pytest
import pytest_httpbin
from scrapling import StealthyFetcher
StealthyFetcher.auto_match = True
StealthyFetcher.adaptive = True
@pytest_httpbin.use_class_based_httpbin
@@ -16,40 +15,34 @@ class TestStealthyFetcher:
@pytest.fixture(autouse=True)
def setup_urls(self, httpbin):
"""Fixture to set up URLs for testing"""
self.status_200 = f'{httpbin.url}/status/200'
self.status_404 = f'{httpbin.url}/status/404'
self.status_501 = f'{httpbin.url}/status/501'
self.basic_url = f'{httpbin.url}/get'
self.html_url = f'{httpbin.url}/html'
self.delayed_url = f'{httpbin.url}/delay/10' # 10 Seconds delay response
self.status_200 = f"{httpbin.url}/status/200"
self.status_404 = f"{httpbin.url}/status/404"
self.status_501 = f"{httpbin.url}/status/501"
self.basic_url = f"{httpbin.url}/get"
self.html_url = f"{httpbin.url}/html"
self.delayed_url = f"{httpbin.url}/delay/10" # 10 Seconds delay response
self.cookies_url = f"{httpbin.url}/cookies/set/test/value"
self.cloudflare_url = "https://nopecha.com/demo/cloudflare" # Interactive turnstile page
def test_cloudflare_fetch(self, fetcher):
"""Test if Cloudflare bypass is working"""
assert fetcher.fetch(self.cloudflare_url, solve_cloudflare=True).status == 200
def test_basic_fetch(self, fetcher):
"""Test doing basic fetch request with multiple statuses"""
"""Test doing a basic fetch request with multiple statuses"""
assert fetcher.fetch(self.status_200).status == 200
assert fetcher.fetch(self.status_404).status == 404
assert fetcher.fetch(self.status_501).status == 501
def test_networkidle(self, fetcher):
"""Test if waiting for `networkidle` make page does not finish loading or not"""
assert fetcher.fetch(self.basic_url, network_idle=True).status == 200
def test_blocking_resources(self, fetcher):
"""Test if blocking resources make page does not finish loading or not"""
assert fetcher.fetch(self.basic_url, block_images=True).status == 200
assert fetcher.fetch(self.basic_url, disable_resources=True).status == 200
def test_waiting_selector(self, fetcher):
"""Test if waiting for a selector make page does not finish loading or not"""
assert fetcher.fetch(self.html_url, wait_selector='h1').status == 200
assert fetcher.fetch(self.html_url, wait_selector='h1', wait_selector_state='visible').status == 200
def test_cookies_loading(self, fetcher):
"""Test if cookies are set after the request"""
assert fetcher.fetch(self.cookies_url).cookies == {'test': 'value'}
response = fetcher.fetch(self.cookies_url)
cookies = {response.cookies[0]['name']: response.cookies[0]['value']}
assert cookies == {"test": "value"}
def test_automation(self, fetcher):
"""Test if automation break the code or not"""
"""Test if automation breaks the code or not"""
def scroll_page(page):
page.mouse.wheel(10, 0)
page.mouse.move(100, 400)
@@ -58,13 +51,38 @@ class TestStealthyFetcher:
assert fetcher.fetch(self.html_url, page_action=scroll_page).status == 200
def test_properties(self, fetcher):
"""Test if different arguments breaks the code or not"""
assert fetcher.fetch(self.html_url, block_webrtc=True, allow_webgl=True).status == 200
assert fetcher.fetch(self.html_url, block_webrtc=False, allow_webgl=True).status == 200
assert fetcher.fetch(self.html_url, block_webrtc=True, allow_webgl=False).status == 200
assert fetcher.fetch(self.html_url, extra_headers={'ayo': ''}, os_randomize=True).status == 200
@pytest.mark.parametrize(
"kwargs",
[
{"block_webrtc": True, "allow_webgl": True, "disable_ads": False},
{"block_webrtc": False, "allow_webgl": True, "block_images": True},
{"block_webrtc": True, "allow_webgl": False, "disable_resources": True},
{"block_images": True, "disable_resources": True, },
{"wait_selector": "h1", "wait_selector_state": "attached"},
{"wait_selector": "h1", "wait_selector_state": "visible"},
{
"network_idle": True,
"wait": 10,
"timeout": 30_000,
"cookies": [{"name": "test", "value": "123", "domain": "example.com", "path": "/"}],
"google_search": True,
"extra_headers": {"ayo": ""},
"os_randomize": True,
"disable_ads": True,
# "geoip": True,
"custom_config": {"keep_comments": False, "keep_cdata": False},
"additional_args": {"window": (1920, 1080)},
},
],
)
def test_properties(self, fetcher, kwargs):
"""Test if different arguments break the code or not"""
response = fetcher.fetch(
self.html_url,
**kwargs
)
assert response.status == 200
def test_infinite_timeout(self, fetcher):
"""Test if infinite timeout breaks the code or not"""
assert fetcher.fetch(self.delayed_url, timeout=None).status == 200
assert fetcher.fetch(self.delayed_url, timeout=0).status == 200
tests/fetchers/sync/test_camoufox_session.py
+97
View File
@@ -0,0 +1,97 @@
import re
import pytest
import pytest_httpbin
from scrapling.engines._browsers._camoufox import StealthySession, __CF_PATTERN__
class TestCamoufoxConstants:
"""Test Camoufox constants and patterns"""
def test_cf_pattern_regex(self):
"""Test __CF_PATTERN__ regex compilation"""
assert isinstance(__CF_PATTERN__, re.Pattern)
# Test matching URLs
test_urls = [
"https://challenges.cloudflare.com/cdn-cgi/challenge-platform/h/123456",
"https://challenges.cloudflare.com/cdn-cgi/challenge-platform/orchestrate/jsch/v1",
"http://challenges.cloudflare.com/cdn-cgi/challenge-platform/scripts/abc"
]
for url in test_urls:
assert __CF_PATTERN__.search(url) is not None
# Test non-matching URLs
non_matching_urls = [
"https://example.com/challenge",
"https://cloudflare.com/something",
"https://challenges.cloudflare.com/other-path"
]
for url in non_matching_urls:
assert __CF_PATTERN__.search(url) is None
@pytest_httpbin.use_class_based_httpbin
class TestStealthySession:
"""All the code is tested in the async version tests, so no need to repeat it here. The async class inherits from this one."""
@pytest.fixture(autouse=True)
def setup_urls(self, httpbin):
"""Fixture to set up URLs for testing"""
self.status_200 = f"{httpbin.url}/status/200"
self.status_404 = f"{httpbin.url}/status/404"
self.status_501 = f"{httpbin.url}/status/501"
self.basic_url = f"{httpbin.url}/get"
self.html_url = f"{httpbin.url}/html"
self.delayed_url = f"{httpbin.url}/delay/10" # 10 Seconds delay response
self.cookies_url = f"{httpbin.url}/cookies/set/test/value"
def test_session_creation(self):
"""Test if the session is created correctly"""
with StealthySession(
max_pages=3,
headless=True,
block_images=True,
disable_resources=True,
solve_cloudflare=True,
wait=1000,
timeout=60000,
cookies=[{"name": "test", "value": "123", "domain": "example.com", "path": "/"}],
) as session:
assert session.max_pages == 3
assert session.headless is True
assert session.block_images is True
assert session.disable_resources is True
assert session.solve_cloudflare is True
assert session.wait == 1000
assert session.timeout == 60000
assert session.context is not None
# Test Cloudflare detection
for cloudflare_type in ('managed', 'interactive', 'non-interactive'):
page_content = f"""
<html>
<script>
cType: '{cloudflare_type}'
</script>
</html>
"""
result = session._detect_cloudflare(page_content)
assert result == cloudflare_type
page_content = """
<html>
<body>
<p>Regular page content</p>
</body>
</html>
"""
result = StealthySession._detect_cloudflare(page_content)
assert result is None
assert session.fetch(self.status_200).status == 200
tests/fetchers/sync/test_dynamic.py
+86
View File
@@ -0,0 +1,86 @@
import pytest
import pytest_httpbin
from scrapling import DynamicFetcher
DynamicFetcher.adaptive = True
@pytest_httpbin.use_class_based_httpbin
class TestDynamicFetcher:
@pytest.fixture(scope="class")
def fetcher(self):
"""Fixture to create a StealthyFetcher instance for the entire test class"""
return DynamicFetcher
@pytest.fixture(autouse=True)
def setup_urls(self, httpbin):
"""Fixture to set up URLs for testing"""
self.status_200 = f"{httpbin.url}/status/200"
self.status_404 = f"{httpbin.url}/status/404"
self.status_501 = f"{httpbin.url}/status/501"
self.basic_url = f"{httpbin.url}/get"
self.html_url = f"{httpbin.url}/html"
self.delayed_url = f"{httpbin.url}/delay/10" # 10 Seconds delay response
self.cookies_url = f"{httpbin.url}/cookies/set/test/value"
def test_basic_fetch(self, fetcher):
"""Test doing a basic fetch request with multiple statuses"""
assert fetcher.fetch(self.status_200).status == 200
# There's a bug with playwright makes it crashes if a URL returns status code 4xx/5xx without body, let's disable this till they reply to my issue report
# assert fetcher.fetch(self.status_404).status == 404
# assert fetcher.fetch(self.status_501).status == 501
def test_cookies_loading(self, fetcher):
"""Test if cookies are set after the request"""
response = fetcher.fetch(self.cookies_url)
cookies = {response.cookies[0]['name']: response.cookies[0]['value']}
assert cookies == {"test": "value"}
def test_automation(self, fetcher):
"""Test if automation breaks the code or not"""
def scroll_page(page):
page.mouse.wheel(10, 0)
page.mouse.move(100, 400)
page.mouse.up()
return page
assert fetcher.fetch(self.html_url, page_action=scroll_page).status == 200
@pytest.mark.parametrize(
"kwargs",
[
{"disable_webgl": True, "hide_canvas": False},
{"disable_webgl": False, "hide_canvas": True, "disable_resources": True},
{"stealth": True}, # causes issues with GitHub Actions
{"wait_selector": "h1", "wait_selector_state": "attached"},
{"wait_selector": "h1", "wait_selector_state": "visible"},
{
"google_search": True,
"real_chrome": True,
"wait": 10,
"locale": "en-US",
"extra_headers": {"ayo": ""},
"useragent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:131.0) Gecko/20100101 Firefox/131.0",
"cookies": [{"name": "test", "value": "123", "domain": "example.com", "path": "/"}],
"network_idle": True,
"custom_config": {"keep_comments": False, "keep_cdata": False},
},
],
)
def test_properties(self, fetcher, kwargs):
"""Test if different arguments break the code or not"""
response = fetcher.fetch(self.html_url, **kwargs)
assert response.status == 200
def test_cdp_url_invalid(self, fetcher):
"""Test if invalid CDP URLs raise appropriate exceptions"""
with pytest.raises(TypeError):
fetcher.fetch(self.html_url, cdp_url="blahblah")
with pytest.raises(TypeError):
fetcher.fetch(self.html_url, cdp_url="blahblah", nstbrowser_mode=True)
with pytest.raises(Exception):
fetcher.fetch(self.html_url, cdp_url="ws://blahblah")
tests/fetchers/sync/test_httpx.py
-84
View File
@@ -1,84 +0,0 @@
import pytest
import pytest_httpbin
from scrapling import Fetcher
Fetcher.auto_match = True
@pytest_httpbin.use_class_based_httpbin
class TestFetcher:
@pytest.fixture(scope="class")
def fetcher(self):
"""Fixture to create a Fetcher instance for the entire test class"""
return Fetcher
@pytest.fixture(autouse=True)
def setup_urls(self, httpbin):
"""Fixture to set up URLs for testing"""
self.status_200 = f'{httpbin.url}/status/200'
self.status_404 = f'{httpbin.url}/status/404'
self.status_501 = f'{httpbin.url}/status/501'
self.basic_url = f'{httpbin.url}/get'
self.post_url = f'{httpbin.url}/post'
self.put_url = f'{httpbin.url}/put'
self.delete_url = f'{httpbin.url}/delete'
self.html_url = f'{httpbin.url}/html'
def test_basic_get(self, fetcher):
"""Test doing basic get request with multiple statuses"""
assert fetcher.get(self.status_200).status == 200
assert fetcher.get(self.status_404).status == 404
assert fetcher.get(self.status_501).status == 501
def test_get_properties(self, fetcher):
"""Test if different arguments with GET request breaks the code or not"""
assert fetcher.get(self.status_200, stealthy_headers=True).status == 200
assert fetcher.get(self.status_200, follow_redirects=True).status == 200
assert fetcher.get(self.status_200, timeout=None).status == 200
assert fetcher.get(
self.status_200,
stealthy_headers=True,
follow_redirects=True,
timeout=None
).status == 200
def test_post_properties(self, fetcher):
"""Test if different arguments with POST request breaks the code or not"""
assert fetcher.post(self.post_url, data={'key': 'value'}).status == 200
assert fetcher.post(self.post_url, data={'key': 'value'}, stealthy_headers=True).status == 200
assert fetcher.post(self.post_url, data={'key': 'value'}, follow_redirects=True).status == 200
assert fetcher.post(self.post_url, data={'key': 'value'}, timeout=None).status == 200
assert fetcher.post(
self.post_url,
data={'key': 'value'},
stealthy_headers=True,
follow_redirects=True,
timeout=None
).status == 200
def test_put_properties(self, fetcher):
"""Test if different arguments with PUT request breaks the code or not"""
assert fetcher.put(self.put_url, data={'key': 'value'}).status == 200
assert fetcher.put(self.put_url, data={'key': 'value'}, stealthy_headers=True).status == 200
assert fetcher.put(self.put_url, data={'key': 'value'}, follow_redirects=True).status == 200
assert fetcher.put(self.put_url, data={'key': 'value'}, timeout=None).status == 200
assert fetcher.put(
self.put_url,
data={'key': 'value'},
stealthy_headers=True,
follow_redirects=True,
timeout=None
).status == 200
def test_delete_properties(self, fetcher):
"""Test if different arguments with DELETE request breaks the code or not"""
assert fetcher.delete(self.delete_url, stealthy_headers=True).status == 200
assert fetcher.delete(self.delete_url, follow_redirects=True).status == 200
assert fetcher.delete(self.delete_url, timeout=None).status == 200
assert fetcher.delete(
self.delete_url,
stealthy_headers=True,
follow_redirects=True,
timeout=None
).status == 200
tests/fetchers/sync/test_playwright.py
-89
View File
@@ -1,89 +0,0 @@
import pytest
import pytest_httpbin
from scrapling import PlayWrightFetcher
PlayWrightFetcher.auto_match = True
@pytest_httpbin.use_class_based_httpbin
class TestPlayWrightFetcher:
@pytest.fixture(scope="class")
def fetcher(self):
"""Fixture to create a StealthyFetcher instance for the entire test class"""
return PlayWrightFetcher
@pytest.fixture(autouse=True)
def setup_urls(self, httpbin):
"""Fixture to set up URLs for testing"""
self.status_200 = f'{httpbin.url}/status/200'
self.status_404 = f'{httpbin.url}/status/404'
self.status_501 = f'{httpbin.url}/status/501'
self.basic_url = f'{httpbin.url}/get'
self.html_url = f'{httpbin.url}/html'
self.delayed_url = f'{httpbin.url}/delay/10' # 10 Seconds delay response
self.cookies_url = f"{httpbin.url}/cookies/set/test/value"
def test_basic_fetch(self, fetcher):
"""Test doing basic fetch request with multiple statuses"""
assert fetcher.fetch(self.status_200).status == 200
# There's a bug with playwright makes it crashes if a URL returns status code 4xx/5xx without body, let's disable this till they reply to my issue report
# assert fetcher.fetch(self.status_404).status == 404
# assert fetcher.fetch(self.status_501).status == 501
def test_networkidle(self, fetcher):
"""Test if waiting for `networkidle` make page does not finish loading or not"""
assert fetcher.fetch(self.basic_url, network_idle=True).status == 200
def test_blocking_resources(self, fetcher):
"""Test if blocking resources make page does not finish loading or not"""
assert fetcher.fetch(self.basic_url, disable_resources=True).status == 200
def test_waiting_selector(self, fetcher):
"""Test if waiting for a selector make page does not finish loading or not"""
assert fetcher.fetch(self.html_url, wait_selector='h1').status == 200
assert fetcher.fetch(self.html_url, wait_selector='h1', wait_selector_state='visible').status == 200
def test_cookies_loading(self, fetcher):
"""Test if cookies are set after the request"""
assert fetcher.fetch(self.cookies_url).cookies == {'test': 'value'}
def test_automation(self, fetcher):
"""Test if automation break the code or not"""
def scroll_page(page):
page.mouse.wheel(10, 0)
page.mouse.move(100, 400)
page.mouse.up()
return page
assert fetcher.fetch(self.html_url, page_action=scroll_page).status == 200
@pytest.mark.parametrize("kwargs", [
{"disable_webgl": True, "hide_canvas": False},
{"disable_webgl": False, "hide_canvas": True},
# {"stealth": True}, # causes issues with Github Actions
{"useragent": 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:131.0) Gecko/20100101 Firefox/131.0'},
{"extra_headers": {'ayo': ''}}
])
def test_properties(self, fetcher, kwargs):
"""Test if different arguments breaks the code or not"""
response = fetcher.fetch(self.html_url, **kwargs)
assert response.status == 200
def test_cdp_url_invalid(self, fetcher):
"""Test if invalid CDP URLs raise appropriate exceptions"""
with pytest.raises(ValueError):
fetcher.fetch(self.html_url, cdp_url='blahblah')
with pytest.raises(ValueError):
fetcher.fetch(self.html_url, cdp_url='blahblah', nstbrowser_mode=True)
with pytest.raises(Exception):
fetcher.fetch(self.html_url, cdp_url='ws://blahblah')
def test_infinite_timeout(self, fetcher, ):
"""Test if infinite timeout breaks the code or not"""
response = fetcher.fetch(self.delayed_url, timeout=None)
assert response.status == 200
tests/fetchers/sync/test_requests.py
+121
View File
@@ -0,0 +1,121 @@
import pytest
import pytest_httpbin
from scrapling import Fetcher
Fetcher.adaptive = True
@pytest_httpbin.use_class_based_httpbin
class TestFetcher:
@pytest.fixture(scope="class")
def fetcher(self):
"""Fixture to create a Fetcher instance for the entire test class"""
return Fetcher
@pytest.fixture(autouse=True)
def setup_urls(self, httpbin):
"""Fixture to set up URLs for testing"""
self.status_200 = f"{httpbin.url}/status/200"
self.status_404 = f"{httpbin.url}/status/404"
self.status_501 = f"{httpbin.url}/status/501"
self.basic_url = f"{httpbin.url}/get"
self.post_url = f"{httpbin.url}/post"
self.put_url = f"{httpbin.url}/put"
self.delete_url = f"{httpbin.url}/delete"
self.html_url = f"{httpbin.url}/html"
def test_basic_get(self, fetcher):
"""Test doing basic get request with multiple statuses"""
assert fetcher.get(self.status_200).status == 200
assert fetcher.get(self.status_404).status == 404
assert fetcher.get(self.status_501).status == 501
def test_get_properties(self, fetcher):
"""Test if different arguments with the GET request break the code or not"""
assert fetcher.get(self.status_200, stealthy_headers=True).status == 200
assert fetcher.get(self.status_200, follow_redirects=True).status == 200
assert fetcher.get(self.status_200, timeout=None).status == 200
assert (
fetcher.get(
self.status_200,
stealthy_headers=True,
follow_redirects=True,
timeout=None,
).status
== 200
)
def test_post_properties(self, fetcher):
"""Test if different arguments with the POST request break the code or not"""
assert fetcher.post(self.post_url, data={"key": "value"}).status == 200
assert (
fetcher.post(
self.post_url, data={"key": "value"}, stealthy_headers=True
).status
== 200
)
assert (
fetcher.post(
self.post_url, data={"key": "value"}, follow_redirects=True
).status
== 200
)
assert (
fetcher.post(self.post_url, data={"key": "value"}, timeout=None).status
== 200
)
assert (
fetcher.post(
self.post_url,
data={"key": "value"},
stealthy_headers=True,
follow_redirects=True,
timeout=None,
).status
== 200
)
def test_put_properties(self, fetcher):
"""Test if different arguments with a PUT request break the code or not"""
assert fetcher.put(self.put_url, data={"key": "value"}).status == 200
assert (
fetcher.put(
self.put_url, data={"key": "value"}, stealthy_headers=True
).status
== 200
)
assert (
fetcher.put(
self.put_url, data={"key": "value"}, follow_redirects=True
).status
== 200
)
assert (
fetcher.put(self.put_url, data={"key": "value"}, timeout=None).status == 200
)
assert (
fetcher.put(
self.put_url,
data={"key": "value"},
stealthy_headers=True,
follow_redirects=True,
timeout=None,
).status
== 200
)
def test_delete_properties(self, fetcher):
"""Test if different arguments with the DELETE request break the code or not"""
assert fetcher.delete(self.delete_url, stealthy_headers=True).status == 200
assert fetcher.delete(self.delete_url, follow_redirects=True).status == 200
assert fetcher.delete(self.delete_url, timeout=None).status == 200
assert (
fetcher.delete(
self.delete_url,
stealthy_headers=True,
follow_redirects=True,
timeout=None,
).status
== 200
)
tests/fetchers/sync/test_requests_session.py
+47
View File
@@ -0,0 +1,47 @@
import pytest
from scrapling.engines.static import FetcherSession, FetcherClient
class TestFetcherSession:
"""Test FetcherSession functionality"""
def test_fetcher_session_creation(self):
"""Test FetcherSession creation"""
session = FetcherSession(
timeout=30,
retries=3,
stealthy_headers=True
)
assert session.default_timeout == 30
assert session.default_retries == 3
assert session.stealth is True
def test_fetcher_session_context_manager(self):
"""Test FetcherSession as a context manager"""
session = FetcherSession()
with session as s:
assert s == session
assert session._curl_session is not None
# Session should be cleaned up
def test_fetcher_session_double_enter(self):
"""Test error on double entering"""
session = FetcherSession()
with session:
with pytest.raises(RuntimeError):
session.__enter__()
def test_fetcher_client_creation(self):
"""Test FetcherClient creation"""
client = FetcherClient()
# Should not have context manager methods
assert client.__enter__ is None
assert client.__exit__ is None
assert client._curl_session is True # Special marker
tests/fetchers/test_base.py
+84
View File
@@ -0,0 +1,84 @@
import pytest
from scrapling.engines.toolbelt.custom import BaseFetcher
class TestBaseFetcher:
"""Test BaseFetcher configuration functionality"""
def test_default_configuration(self):
"""Test default configuration values"""
config = BaseFetcher.display_config()
assert config['huge_tree'] is True
assert config['adaptive'] is False
assert config['keep_comments'] is False
assert config['keep_cdata'] is False
def test_configure_single_parameter(self):
"""Test configuring single parameter"""
BaseFetcher.configure(adaptive=True)
config = BaseFetcher.display_config()
assert config['adaptive'] is True
# Reset
BaseFetcher.configure(adaptive=False)
def test_configure_multiple_parameters(self):
"""Test configuring multiple parameters"""
BaseFetcher.configure(
huge_tree=False,
keep_comments=True,
adaptive=True
)
config = BaseFetcher.display_config()
assert config['huge_tree'] is False
assert config['keep_comments'] is True
assert config['adaptive'] is True
# Reset
BaseFetcher.configure(
huge_tree=True,
keep_comments=False,
adaptive=False
)
def test_configure_invalid_parameter(self):
"""Test configuring invalid parameter"""
with pytest.raises(ValueError):
BaseFetcher.configure(invalid_param=True)
def test_configure_no_parameters(self):
"""Test configure with no parameters"""
with pytest.raises(AttributeError):
BaseFetcher.configure()
def test_configure_non_parser_keyword(self):
"""Test configuring non-parser keyword"""
with pytest.raises(AttributeError):
# Assuming there's some attribute that's not in parser_keywords
BaseFetcher.some_other_attr = "test"
BaseFetcher.configure(some_other_attr="new_value")
def test_generate_parser_arguments(self):
"""Test parser arguments generation"""
BaseFetcher.configure(
huge_tree=False,
adaptive=True,
adaptive_domain="example.com"
)
args = BaseFetcher._generate_parser_arguments()
assert args['huge_tree'] is False
assert args['adaptive'] is True
assert args['adaptive_domain'] == "example.com"
# Reset
BaseFetcher.configure(
huge_tree=True,
adaptive=False
)
BaseFetcher.adaptive_domain = None
tests/fetchers/test_constants.py
+28
View File
@@ -0,0 +1,28 @@
from scrapling.engines.constants import (
DEFAULT_DISABLED_RESOURCES,
DEFAULT_STEALTH_FLAGS,
HARMFUL_DEFAULT_ARGS,
DEFAULT_FLAGS,
)
class TestConstants:
"""Test constant values"""
def test_default_disabled_resources(self):
"""Test default disabled resources"""
assert "image" in DEFAULT_DISABLED_RESOURCES
assert "font" in DEFAULT_DISABLED_RESOURCES
assert "stylesheet" in DEFAULT_DISABLED_RESOURCES
assert "media" in DEFAULT_DISABLED_RESOURCES
def test_harmful_default_args(self):
"""Test harmful default arguments"""
assert "--enable-automation" in HARMFUL_DEFAULT_ARGS
assert "--disable-popup-blocking" in HARMFUL_DEFAULT_ARGS
def test_flags(self):
"""Test default stealth flags"""
assert "--no-pings" in DEFAULT_FLAGS
assert "--incognito" in DEFAULT_STEALTH_FLAGS
assert "--disable-blink-features=AutomationControlled" in DEFAULT_STEALTH_FLAGS
tests/fetchers/test_pages.py
+154
View File
@@ -0,0 +1,154 @@
import pytest
from unittest.mock import Mock
from scrapling.engines._browsers._page import PageInfo, PagePool
class TestPageInfo:
"""Test PageInfo functionality"""
def test_page_info_creation(self):
"""Test PageInfo creation"""
mock_page = Mock()
page_info = PageInfo(mock_page, "ready", "https://example.com")
assert page_info.page == mock_page
assert page_info.state == "ready"
assert page_info.url == "https://example.com"
def test_page_info_marking(self):
"""Test marking page"""
mock_page = Mock()
page_info = PageInfo(mock_page, "ready", None)
page_info.mark_busy("https://example.com")
assert page_info.state == "busy"
assert page_info.url == "https://example.com"
page_info.mark_ready()
assert page_info.state == "ready"
assert page_info.url == ""
page_info.mark_error()
assert page_info.state == "error"
def test_page_info_equality(self):
"""Test PageInfo equality comparison"""
mock_page1 = Mock()
mock_page2 = Mock()
page_info1 = PageInfo(mock_page1, "ready", None)
page_info2 = PageInfo(mock_page1, "busy", None) # Same page, different state
page_info3 = PageInfo(mock_page2, "ready", None) # Different page
assert page_info1 == page_info2 # Same page
assert page_info1 != page_info3 # Different page
assert page_info1 != "not a page info" # Different type
def test_page_info_repr(self):
"""Test PageInfo string representation"""
mock_page = Mock()
page_info = PageInfo(mock_page, "ready", "https://example.com")
repr_str = repr(page_info)
assert "ready" in repr_str
assert "https://example.com" in repr_str
class TestPagePool:
"""Test PagePool functionality"""
def test_page_pool_creation(self):
"""Test PagePool creation"""
pool = PagePool(max_pages=5)
assert pool.max_pages == 5
assert pool.pages_count == 0
assert pool.ready_count == 0
assert pool.busy_count == 0
def test_add_page(self):
"""Test adding page to pool"""
pool = PagePool(max_pages=2)
mock_page = Mock()
page_info = pool.add_page(mock_page)
assert isinstance(page_info, PageInfo)
assert page_info.page == mock_page
assert page_info.state == "ready"
assert pool.pages_count == 1
def test_add_page_limit_exceeded(self):
"""Test adding page when limit exceeded"""
pool = PagePool(max_pages=1)
# Add first page
pool.add_page(Mock())
# Try to add a second page
with pytest.raises(RuntimeError):
pool.add_page(Mock())
def test_get_ready_page(self):
"""Test getting ready page"""
pool = PagePool(max_pages=3)
# Add pages
page1 = pool.add_page(Mock())
page2 = pool.add_page(Mock())
# Mark one as busy
page1.mark_busy("https://example.com")
# Should get the ready page
ready_page = pool.get_ready_page()
assert ready_page == page2
def test_get_ready_page_none_available(self):
"""Test getting ready page when none available"""
pool = PagePool(max_pages=2)
# Add pages and mark all as busy
page1 = pool.add_page(Mock())
page2 = pool.add_page(Mock())
page1.mark_busy("https://example1.com")
page2.mark_busy("https://example2.com")
# Should return None
ready_page = pool.get_ready_page()
assert ready_page is None
def test_page_counts(self):
"""Test page count properties"""
pool = PagePool(max_pages=3)
# Add pages with different states
page1 = pool.add_page(Mock())
page2 = pool.add_page(Mock())
page3 = pool.add_page(Mock())
page1.mark_busy("https://example.com")
page3.mark_error()
assert pool.pages_count == 3
assert pool.ready_count == 1
assert pool.busy_count == 1
def test_cleanup_error_pages(self):
"""Test cleaning up error pages"""
pool = PagePool(max_pages=3)
# Add pages
page1 = pool.add_page(Mock())
page2 = pool.add_page(Mock())
page3 = pool.add_page(Mock())
# Mark some as error
page1.mark_error()
page3.mark_error()
assert pool.pages_count == 3
pool.cleanup_error_pages()
assert pool.pages_count == 1 # Only page2 should remain
tests/fetchers/test_response_handling.py
+109
View File
@@ -0,0 +1,109 @@
from unittest.mock import Mock
from scrapling.parser import Selector
from scrapling.engines.toolbelt import ResponseFactory, Response
from scrapling.engines.toolbelt.custom import ResponseEncoding
class TestResponseFactory:
"""Test ResponseFactory functionality"""
def test_response_from_curl(self):
"""Test creating response from curl_cffi response"""
# Mock curl response
mock_curl_response = Mock()
mock_curl_response.url = "https://example.com"
mock_curl_response.content = b"<html><body>Test</body></html>"
mock_curl_response.status_code = 200
mock_curl_response.reason = "OK"
mock_curl_response.encoding = "utf-8"
mock_curl_response.cookies = {"session": "abc"}
mock_curl_response.headers = {"Content-Type": "text/html"}
mock_curl_response.request.headers = {"User-Agent": "Test"}
mock_curl_response.request.method = "GET"
mock_curl_response.history = []
response = ResponseFactory.from_http_request(
mock_curl_response,
{"adaptive": False}
)
assert response.status == 200
assert response.url == "https://example.com"
assert isinstance(response, Response)
def test_response_encoding_edge_cases(self):
"""Test response encoding handling"""
# Test various content types
test_cases = [
(None, "utf-8"),
("", "utf-8"),
("text/html; charset=invalid", "utf-8"),
("application/octet-stream", "utf-8"),
]
for content_type, expected in test_cases:
encoding = ResponseEncoding.get_value(content_type)
assert encoding == expected
def test_response_history_processing(self):
"""Test processing response history"""
# Mock responses with redirects
mock_final = Mock()
mock_final.status = 200
mock_final.status_text = "OK"
mock_final.all_headers = Mock(return_value={})
mock_redirect = Mock()
mock_redirect.url = "https://example.com/redirect"
mock_redirect.response = Mock(return_value=mock_final)
mock_redirect.all_headers = Mock(return_value={})
mock_redirect.redirected_from = None
mock_first = Mock()
mock_first.request.redirected_from = mock_redirect
# Process history
history = ResponseFactory._process_response_history(
mock_first,
{}
)
assert len(history) >= 0 # Should process redirects
class TestErrorScenarios:
"""Test various error scenarios"""
def test_invalid_html_handling(self):
"""Test handling of malformed HTML"""
malformed_html = """
<html>
<body>
<div>Unclosed div
<p>Paragraph without closing tag
<span>Nested unclosed
</body>
"""
# Should handle gracefully
page = Selector(malformed_html)
assert page is not None
# Should still be able to select elements
divs = page.css("div")
assert len(divs) > 0
def test_empty_responses(self):
"""Test handling of empty responses"""
# Empty HTML
page = Selector("")
assert page is not None
# Whitespace only
page = Selector(" \n\t ")
assert page is not None
# Null bytes
page = Selector("Hello\x00World")
assert "Hello" in page.get_all_text()

Some files were not shown because too many files have changed in this diff Show More