From ed694f0a79b784284f2d4ae1419f801c9a2fbe14 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Tue, 8 Apr 2025 03:44:57 +0200 Subject: [PATCH 01/34] First version of Scrapling full documentation --- docs/Core/using scrapling custom types.md | 21 - docs/Examples/selectorless_stackoverflow.py | 25 - .../writing storage system.md | 17 - docs/api-reference/adaptor.md | 20 + docs/api-reference/custom-types.md | 21 + docs/api-reference/fetchers.md | 25 + docs/benchmarks.md | 44 ++ docs/contributing.md | 102 ++++ docs/development/automatch_storage_system.md | 66 +++ docs/development/scrapling_custom_types.md | 21 + docs/donate.md | 27 + docs/fetching/choosing.md | 77 +++ docs/fetching/dynamic.md | 248 ++++++++ docs/fetching/static.md | 300 ++++++++++ docs/fetching/stealthy.md | 218 +++++++ docs/index.md | 109 +++- docs/overview.md | 328 +++++++++++ docs/parsing/automatch.md | 220 +++++++ docs/parsing/main_classes.md | 539 ++++++++++++++++++ docs/parsing/selection.md | 512 +++++++++++++++++ docs/stylesheets/extra.css | 3 + .../tutorials/migrating_from_beautifulsoup.md | 98 ++++ docs/tutorials/replacing_ai.md | 1 + mkdocs.yml | 142 +++++ 24 files changed, 3119 insertions(+), 65 deletions(-) delete mode 100644 docs/Core/using scrapling custom types.md delete mode 100644 docs/Examples/selectorless_stackoverflow.py delete mode 100644 docs/Extending Scrapling/writing storage system.md create mode 100644 docs/api-reference/adaptor.md create mode 100644 docs/api-reference/custom-types.md create mode 100644 docs/api-reference/fetchers.md create mode 100644 docs/benchmarks.md create mode 100644 docs/contributing.md create mode 100644 docs/development/automatch_storage_system.md create mode 100644 docs/development/scrapling_custom_types.md create mode 100644 docs/donate.md create mode 100644 docs/fetching/choosing.md create mode 100644 docs/fetching/dynamic.md create mode 100644 docs/fetching/static.md create mode 100644 docs/fetching/stealthy.md create mode 100644 docs/overview.md create mode 100644 docs/parsing/automatch.md create mode 100644 docs/parsing/main_classes.md create mode 100644 docs/parsing/selection.md create mode 100644 docs/stylesheets/extra.css create mode 100644 docs/tutorials/migrating_from_beautifulsoup.md create mode 100644 docs/tutorials/replacing_ai.md create mode 100644 mkdocs.yml diff --git a/docs/Core/using scrapling custom types.md b/docs/Core/using scrapling custom types.md deleted file mode 100644 index 202c91b..0000000 --- a/docs/Core/using scrapling custom types.md +++ /dev/null @@ -1,21 +0,0 @@ -> You can take advantage from the custom-made types for Scrapling and use it outside the library if you want. It's better than copying their code after all :) - -### All current types can be imported alone like below -```python ->>> from scrapling.core.custom_types import TextHandler, AttributesHandler - ->>> somestring = TextHandler('{}') ->>> somestring.json() -'{}' ->>> somedict_1 = AttributesHandler({'a': 1}) ->>> somedict_2 = AttributesHandler(a=1) -``` - -Note `TextHandler` is a sub-class of Python's `str` so all normal operations/methods that work with Python strings will work. -If you want to check for the type in your code, it's better to depend on Python built-in function `issubclass`. - -The class `AttributesHandler` is a sub-class of `collections.abc.Mapping` so it's immutable (read-only) and all operations are inherited from it. The data passed can be accessed later though the `._data` method but careful it's of type `types.MappingProxyType` so it's immutable (read-only) as well (faster than `collections.abc.Mapping` by fractions of seconds). - -So basically to make it simple to you if you are new to Python, the same operations and methods from Python standard `dict` type will all work with class `AttributesHandler` except the ones that try to modify the actual data. - -If you want to modify the data inside `AttributesHandler`, you have to convert it to dictionary first like with using the `dict` function and modify it outside. \ No newline at end of file diff --git a/docs/Examples/selectorless_stackoverflow.py b/docs/Examples/selectorless_stackoverflow.py deleted file mode 100644 index 8619653..0000000 --- a/docs/Examples/selectorless_stackoverflow.py +++ /dev/null @@ -1,25 +0,0 @@ -""" -I only made this example to show how Scrapling features can be used to scrape a website without writing any selector - so this script doesn't depend on the website structure. -""" - -import requests - -from scrapling import Adaptor - -response = requests.get('https://stackoverflow.com/questions/tagged/web-scraping?sort=MostVotes&filters=NoAcceptedAnswer&edited=true&pagesize=50&page=2') -page = Adaptor(response.text, url=response.url) -# First we will extract the first question title and its author based on the text content -first_question_title = page.find_by_text('Run Selenium Python Script on Remote Server') -first_question_author = page.find_by_text('Ryan') -# because this page changes a lot -if first_question_title and first_question_author: - # If you want you can extract other questions tags like below - first_question = first_question_title.find_ancestor( - lambda ancestor: ancestor.attrib.get('id') and 'question-summary' in ancestor.attrib.get('id') - ) - rest_of_questions = first_question.find_similar() - # But since nothing to rely on to extract other titles/authors from these elements without CSS/XPath selectors due to the website nature - # We will get all the rest of the titles/authors in the page depending on the first title and the first author we got above as a starting point - for i, (title, author) in enumerate(zip(first_question_title.find_similar(), first_question_author.find_similar()), start=1): - print(i, title.text, author.text) diff --git a/docs/Extending Scrapling/writing storage system.md b/docs/Extending Scrapling/writing storage system.md deleted file mode 100644 index dec88f2..0000000 --- a/docs/Extending Scrapling/writing storage system.md +++ /dev/null @@ -1,17 +0,0 @@ -Scrapling by default is using SQLite but in case you want to write your storage system to store elements properties there for the auto-matching, this tutorial got you covered. - -You might want to use FireBase for example and share the database between multiple spiders on different machines, it's a great idea to use an online database like that because this way the spiders will share with each others. - -So first to make your storage class work, it must do the big 3: -1. Inherit from the abstract class `scrapling.storage_adaptors.StorageSystemMixin` and accept a string argument which will be the `url` argument to maintain the library logic. -2. Use the decorator `functools.lru_cache` on top of the class itself to follow the Singleton design pattern as other classes. -3. Implement methods `save` and `retrieve`, as you see from the type hints: - - The method `save` returns nothing and will get two arguments from the library - * The first one is of type `lxml.html.HtmlElement` which is the element itself, ofc. It must be converted to dictionary using the function `scrapling.utils._StorageTools.element_to_dict` so we keep the same format then saved to your database as you wish. - * The second one is string which is the identifier used for retrieval. The combination of this identifier and the `url` argument from initialization must be unique for each row or the auto-match will be messed up. - - The method `retrieve` takes a string which is the identifier, using it with the `url` passed on initialization the element's dictionary is retrieved from the database and returned if it exist otherwise it returns `None` -> If the instructions weren't clear enough for you, you can check my implementation using SQLite3 in [storage_adaptors](https://github.com/D4Vinci/Scrapling/blob/main/scrapling/storage_adaptors.py) file - -If your class satisfy this, the rest is easy. If you are planning to use the library in a threaded application, make sure that your class supports it. The default used class is thread-safe. - -There are some helper functions added to the abstract class if you want to use it. It's easier to see it for yourself in the [code](https://github.com/D4Vinci/Scrapling/blob/main/scrapling/storage_adaptors.py), it's heavily commented :) diff --git a/docs/api-reference/adaptor.md b/docs/api-reference/adaptor.md new file mode 100644 index 0000000..9d63c44 --- /dev/null +++ b/docs/api-reference/adaptor.md @@ -0,0 +1,20 @@ +# Adaptor Class + +The `Adaptor` class is the core parsing engine in Scrapling that provides HTML parsing and element selection capabilities. + +Here's the reference information for the `Adaptor` class, with all its parameters, attributes, and methods. + +You can import the `Adaptor` class directly from `scrapling`: + +```python +from scrapling.parser import Adaptor +``` + +## ::: scrapling.parser.Adaptor + handler: python + :docstring: + +## ::: scrapling.parser.Adaptors + handler: python + :docstring: + diff --git a/docs/api-reference/custom-types.md b/docs/api-reference/custom-types.md new file mode 100644 index 0000000..9a3d58d --- /dev/null +++ b/docs/api-reference/custom-types.md @@ -0,0 +1,21 @@ +# Custom Types API Reference + +Here's the reference information for all custom types of classes Scrapling implemented, with all their parameters, attributes, and methods. + +You can import all of them directly like below: + +```python +from scrapling.core.custom_types import TextHandler, TextHandlers, AttributesHandler +``` + +## ::: scrapling.core.custom_types.TextHandler + handler: python + :docstring: + +## ::: scrapling.core.custom_types.TextHandlers + handler: python + :docstring: + +## ::: scrapling.core.custom_types.AttributesHandler + handler: python + :docstring: diff --git a/docs/api-reference/fetchers.md b/docs/api-reference/fetchers.md new file mode 100644 index 0000000..dff2ade --- /dev/null +++ b/docs/api-reference/fetchers.md @@ -0,0 +1,25 @@ +# Fetchers Classes + +Here's the reference information for all fetcher-type classes' parameters, attributes, and methods. + +You can import all of them directly like below: + +```python +from scrapling.fetchers import Fetcher, AsyncFetcher, StealthyFetcher, PlayWrightFetcher +``` + +## ::: scrapling.fetchers.Fetcher + handler: python + :docstring: + +## ::: scrapling.fetchers.AsyncFetcher + handler: python + :docstring: + +## ::: scrapling.fetchers.PlayWrightFetcher + handler: python + :docstring: + +## ::: scrapling.fetchers.StealthyFetcher + handler: python + :docstring: diff --git a/docs/benchmarks.md b/docs/benchmarks.md new file mode 100644 index 0000000..bf5106e --- /dev/null +++ b/docs/benchmarks.md @@ -0,0 +1,44 @@ +Scrapling isn't just powerful - it's also blazing fast. Scrapling implements many best practices, design patterns, and numerous optimizations to save fractions of seconds. All of that while focusing exclusively on parsing HTML documents. + +Here are benchmarks comparing Scrapling's parsing speed to popular Python libraries in two tests. + +### Text Extraction Speed Test + +This test consists of extracting the text content of 5000 nested div elements. + +Here are the results comparing Scrapling to all well-known parsing libraries: + + +| # | Library | Time (ms) | vs Scrapling | +|---|:-----------------:|:---------:|:------------:| +| 1 | Scrapling | 5.44 | 1.0x | +| 2 | Parsel/Scrapy | 5.53 | 1.017x | +| 3 | Raw Lxml | 6.76 | 1.243x | +| 4 | PyQuery | 21.96 | 4.037x | +| 5 | Selectolax | 67.12 | 12.338x | +| 6 | BS4 with Lxml | 1307.03 | 240.263x | +| 7 | MechanicalSoup | 1322.64 | 243.132x | +| 8 | BS4 with html5lib | 3373.75 | 620.175x | + +As you see, Scrapling is on par with Scrapy and slightly faster than Lxml, which both libraries are built on top of. These are the closest results to Scrapling. PyQuery is also built on top of Lxml, but Scrapling is four times faster. + +### Extraction By Text Speed Test + +Scrapling can find elements based on its text content and find elements similar to these elements. The only known library with these two features, too, is AutoScraper. + +So, we compared this to see how fast Scrapling can be in these two tasks compared to AutoScraper. + +Here are the results: + +| Library | Time (ms) | vs Scrapling | +|-------------|:---------:|:------------:| +| Scrapling | 2.51 | 1.0x | +| AutoScraper | 11.41 | 4.546x | + +Scrapling can find elements with more methods and returns the entire element's `Adaptor` object, not only text like AutoScraper. So, to make this test fair, both libraries will extract an element with text, find similar elements, and then extract the text content for all of them. + +As you see, Scrapling is still 4.5 times faster at the same task. + +If we made Scrapling extract the elements only without stopping to extract each element's text, we would get speed twice as fast as this, but as I said, to make it fair comparison a bit :smile: + +> All benchmarks' results are an average of 100 runs. See our [benchmarks.py](https://github.com/D4Vinci/Scrapling/blob/main/benchmarks.py) for methodology and to run your comparisons. \ No newline at end of file diff --git a/docs/contributing.md b/docs/contributing.md new file mode 100644 index 0000000..5095959 --- /dev/null +++ b/docs/contributing.md @@ -0,0 +1,102 @@ +Thank you for your interest in contributing to Scrapling! + +Everybody is invited and welcome to contribute to Scrapling. + +Smaller changes have a better chance of getting included in a timely manner. Adding unit tests for new features or test cases for bugs you've fixed helps us to ensure that the Pull Request (PR) is acceptable. + +There is a lot to do... + +- If you are not a developer, you can help us improve the documentation. +- If you are a developer, most of the features I'm planning to add in the future are moved to [roadmap file](https://github.com/D4Vinci/Scrapling/blob/main/ROADMAP.md), so consider reading it. + +## Running tests +Scrapling includes a comprehensive test suite that can be executed with pytest, but first, you need to install all libraries and `pytest-plugins` inside `tests/requirements.txt`. Then, running the tests will result in an output like this: + ```bash + $ pytest tests + =============================== test session starts =============================== + platform darwin -- Python 3.12.8, pytest-8.3.3, pluggy-1.5.0 -- /Users//.venv/bin/python3.12 + cachedir: .pytest_cache + rootdir: /Users//scrapling + configfile: pytest.ini + plugins: cov-5.0.0, asyncio-0.25.0, base-url-2.1.0, httpbin-2.1.0, playwright-0.5.2, anyio-4.6.2.post1, xdist-3.6.1, typeguard-4.3.0 + asyncio: mode=Mode.AUTO, asyncio_default_fixture_loop_scope=function + collected 83 items + + ...... + + =============================== 83 passed in 157.52s (0:02:37) ===================== + ``` +Hence, you can add `-n auto` to the command above to run tests in threads to increase speed. + +Bonus: You can also see the test coverage with the pytest plugin below +```bash +pytest --cov=scrapling tests/ +``` + +## Installing the latest unstable version from the dev branch +```bash +pip3 install git+https://github.com/D4Vinci/Scrapling.git@dev +``` + +## Development +Setting the scrapling logging level to `debug` makes it easier to know what's happening in the background. + ```python + >>> import logging + >>> logging.getLogger("scrapling").setLevel(logging.DEBUG) + ``` +### Code Style + +We use: + +1. Type hints for better code clarity +2. Flake8, bandit, isort, and other hooks through `pre-commit`.
Please install the hooks before committing with: + ```bash + pip install pre-commit + pre-commit install + ``` + It will run automatically on the code you push with each commit. +3. Conventional commit messages format. We use the below format for commit messages + + | Prefix | When to use it | + |-------------|--------------------------| + | `feat:` | New feature added | + | `fix:` | Bug fix | + | `docs:` | Documentation change/add | + | `test:` | Tests | + | `refactor:` | Code refactoring | + | `chore:` | Maintenance tasks | + + Example: + ``` + feat: add auto-matching for similar elements + + - Added find_similar() method + - Implemented pattern matching + - Added tests and documentation + ``` + +### Push changes to the library + +Then, the process is straightforward. + + - Read [How to get faster PR reviews](https://github.com/kubernetes/community/blob/master/contributors/guide/pull-requests.md#best-practices-for-faster-reviews) by Kubernetes (but skip step 0 and 1) + - Fork Scrapling [Git repository](https://github.com/D4Vinci/Scrapling.git). + - Make your changes, and don't forget to create a separate virtual environment for this project. + - Ensure all tests are passing. + - Create a Pull Request against the [**dev**](https://github.com/D4Vinci/Scrapling/tree/dev) branch of Scrapling. + +A bonus: if you have more than one version of Python installed, you can use tox to run tests on each version with: +```bash +pip install tox +tox +``` + +> Note: All tests are automatically run with each push on Github on all supported Python versions using tox, so ensure all tests pass, or your PR will not be accepted. + + +## Building Documentation +```bash +pip install mkdocs-material +mkdocs serve # Local preview +mkdocs build # Build the static site +``` \ No newline at end of file diff --git a/docs/development/automatch_storage_system.md b/docs/development/automatch_storage_system.md new file mode 100644 index 0000000..9936d37 --- /dev/null +++ b/docs/development/automatch_storage_system.md @@ -0,0 +1,66 @@ +Scrapling uses SQLite by default, but this tutorial covers writing your storage system to store element properties there for auto-matching. + +You might want to use FireBase, for example, and share the database between multiple spiders on different machines. It's a great idea to use an online database like that because the spiders will share with each other. + +So first, to make your storage class work, it must do the big 3: + +1. Inherit from the abstract class `scrapling.core.storage_adaptors.StorageSystemMixin` and accept a string argument, which will be the `url` argument to maintain the library logic. +2. Use the decorator `functools.lru_cache` on top of the class to follow the Singleton design pattern as other classes. +3. Implement methods `save` and `retrieve`, as you see from the type hints: + - The method `save` returns nothing and will get two arguments from the library + * The first one is of type `lxml.html.HtmlElement`, which is the element itself. It must be converted to a dictionary using the function `element_to_dict` in submodule `scrapling.core.utils._StorageTools` to keep the same format and save it to your database as you wish. + * The second one is a string, the identifier used for retrieval. The combination result of this identifier and the `url` argument from initialization must be unique for each row, or the auto-match will be messed up. + - The method `retrieve` takes a string, which is the identifier; using it with the `url` passed on initialization, the element's dictionary is retrieved from the database and returned if it exists; otherwise, it returns `None`. + +> If the instructions weren't clear enough for you, you can check my implementation using SQLite3 in [storage_adaptors](https://github.com/D4Vinci/Scrapling/blob/main/scrapling/core/storage_adaptors.py) file + +If your class meets these criteria, the rest is easy. If you plan to use the library in a threaded application, ensure your class supports it. The default used class is thread-safe. + +Some helper functions are added to the abstract class if you want to use them. It's easier to see it for yourself in the [code](https://github.com/D4Vinci/Scrapling/blob/main/scrapling/core/storage_adaptors.py); it's heavily commented :) + + +## Real-World Example: Redis Storage + +Here's a more practical example generated by AI using Redis: + +```python +import redis +import orjson +from functools import lru_cache +from scrapling.core.storage_adaptors import StorageSystemMixin +from scrapling.core.utils import _StorageTools + +@lru_cache(None) +class RedisStorage(StorageSystemMixin): + def __init__(self, host='localhost', port=6379, db=0, url=None): + super().__init__(url) + self.redis = redis.Redis( + host=host, + port=port, + db=db, + decode_responses=False + ) + + def save(self, element, identifier: str) -> None: + # Convert element to dictionary + element_dict = _StorageTools.element_to_dict(element) + + # Create key + key = f"scrapling:{self._get_base_url()}:{identifier}" + + # Store as JSON + self.redis.set( + key, + orjson.dumps(element_dict) + ) + + def retrieve(self, identifier: str) -> dict: + # Get data + key = f"scrapling:{self._get_base_url()}:{identifier}" + data = self.redis.get(key) + + # Parse JSON if exists + if data: + return orjson.loads(data) + return None +``` \ No newline at end of file diff --git a/docs/development/scrapling_custom_types.md b/docs/development/scrapling_custom_types.md new file mode 100644 index 0000000..aee8c1f --- /dev/null +++ b/docs/development/scrapling_custom_types.md @@ -0,0 +1,21 @@ +> You can take advantage of the custom-made types for Scrapling and use them outside the library if you want. It's better than copying their code, after all :) + +### All current types can be imported alone like below +```python +>>> from scrapling.core.custom_types import TextHandler, AttributesHandler + +>>> somestring = TextHandler('{}') +>>> somestring.json() +'{}' +>>> somedict_1 = AttributesHandler({'a': 1}) +>>> somedict_2 = AttributesHandler(a=1) +``` + +Note that `TextHandler` is a subclass of Python's `str`, so all normal operations/methods that work with Python strings will work. +If you want to check for the type in your code, it's better to depend on Python's built-in function `issubclass`. + +The class `AttributesHandler` is a subclass of `collections.abc.Mapping`, so it's immutable (read-only), and all operations are inherited from it. The data passed can be accessed later through the `_data` property, but be careful; it's of type `types.MappingProxyType`, so it's immutable (read-only) as well (faster than `collections.abc.Mapping` by fractions of seconds). + +So, to make it simple for you if you are new to Python, the same operations and methods from the Python standard `dict` type will all work with class `AttributesHandler` except the ones that try to modify the actual data. + +If you want to modify the data inside `AttributesHandler,` you have to convert it to a dictionary first, like using the `dict` function, and then modify it outside. \ No newline at end of file diff --git a/docs/donate.md b/docs/donate.md new file mode 100644 index 0000000..15aceaf --- /dev/null +++ b/docs/donate.md @@ -0,0 +1,27 @@ +I've been working on Scrapling and other public projects in my spare time and have invested considerable resources and effort to provide these projects for free to the community. By becoming a sponsor, you'd be directly funding my coffee reserves, helping me continuously update existing projects and potentially create new ones. + +You can sponsor me directly through [Github sponsors program](https://github.com/sponsors/D4Vinci) or [Buy Me A Coffe](https://buymeacoffee.com/d4vinci). If you are a **company** and looking to **advertise** your business through Scrapling or another project, check out the available plans on my [Github Sponsors page](https://github.com/sponsors/D4Vinci). + +Below is the list of our Gold tier sponsors. + +Thank you, stay curious, and hack the planet! ❤️ + +--- + +## Top Sponsors +### Scrapeless + +[Scrapeless Deep SerpApi](https://www.scrapeless.com/en/product/deep-serp-api?utm_source=website&utm_medium=ads&utm_campaign=scraping&utm_term=d4vinci) From $0.10 per 1,000 queries with a 1-2 second response time! + +[![Scrapeless Banner](https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/scrapeless.jpg)](https://www.scrapeless.com/?utm_source=github&utm_medium=ads&utm_campaign=scraping&utm_term=D4Vinci) + +Deep SerpApi is a dedicated search engine designed for large language models (LLMs) and AI agents. It aims to provide real-time, accurate, and unbiased information to help AI applications retrieve and process data efficiently. + +- covering 20+ Google SERP scenarios and mainstream search engines. +- support real-time data updates to ensure real-time and accurate information. +- It can integrate information from all available online channels and search engines. +- Deep SerpApi will simplify the process of integrating dynamic web information into AI solutions, and ultimately achieve an ALL-in-One API for one-click search and extraction of web data. +- **Developer Support Program**: Integrate Scrapeless Deep SerpApi into your AI tools, applications or projects. [We already support Dify, and will soon support frameworks such as Langchain, Langflow, FlowiseAI]. Then share your results on GitHub or social media, and you will get a 1-12 month free developer support opportunity, up to 500 free usage per month. +- 🚀 **Scraping API**: Effortless and highly customizable data extraction with a single API call, providing structured data from any website. +- ⚡ **Scraping Browser**: AI-powered and LLM-driven, it simulates human-like behavior with genuine fingerprints and headless browser support, ensuring seamless, block-free scraping. +- 🌐 **Proxies**: Use high-quality, rotating proxies to scrape top platforms like Amazon, Shopee, and more, with global coverage in 195+ countries. diff --git a/docs/fetching/choosing.md b/docs/fetching/choosing.md new file mode 100644 index 0000000..746286c --- /dev/null +++ b/docs/fetching/choosing.md @@ -0,0 +1,77 @@ +## Introduction +Fetchers are classes that can do requests or fetch pages for you easily in a single-line fashion with many features and then return a [Response](#response-object) object. + +This feature was introduced because the only option before v0.2 was to fetch the page as you wanted, then pass it manually to the `Adaptor` class and start playing with it. + +> Fetchers are not wrappers built on top of other libraries, but they use these libraries as an engine to make requests/fetch pages easily for you while fully utilizing that engine and adding features for you that aren't included in those engines + +## Fetchers Overview + +Scrapling provides three different fetcher classes, each designed for specific use cases. + +The following table compares them and can be quickly used for guidance. + + +| Feature | Fetcher | PlayWrightFetcher | StealthyFetcher | +|--------------------|----------------|--------------------------------------------------------------------------------|--------------------------------------------------------------------------------------| +| Relative speed | ⭐⭐⭐⭐⭐ | ⭐⭐⭐ | ⭐⭐ | +| Stealth | ⭐ | ⭐⭐ | ⭐⭐⭐⭐ | +| Anti-Bot options | ⭐ | ⭐⭐ | ⭐⭐⭐⭐ | +| JavaScript loading | ❌ | ✅ | ✅ | +| Memory Usage | ⭐ | ⭐⭐⭐ | ⭐⭐⭐ | +| Best used for | Basic scraping | - Dynamically loaded websites
- Small automation
- Slight protections | - Dynamically loaded websites
- Small automation
- Complicated protections | +| Browser(s) | ❌ | Chromium and Google Chrome | Modified Firefox | +| Browser API used | ❌ | PlayWright | PlayWright | +| Setup Complexity | Simple | Simple | Simple | + +In the following pages, we will talk about each one in detail. + +## Parser configuration in all fetchers +All fetchers classes share the same import, as you will see in the upcoming pages +```python +>>> from scrapling.fetchers import Fetcher, AsyncFetcher, StealthyFetcher, PlayWrightFetcher +``` +Then you use it right away without initializing like this, and it will use the default parser settings: +```python +>>> page = StealthyFetcher.fetch('https://example.com') +``` +If you want to configure the parser ([Adaptor class](../parsing/main_classes.md#adaptor)) that will be used on the response before returning it for you, then do this first: +```python +>>> from scrapling.fetchers import Fetcher +>>> Fetcher.configure(auto_match=True, encoding="utf8", keep_comments=False, keep_cdata=False) # and the rest +``` +or +```python +>>> from scrapling.fetchers import Fetcher +>>> Fetcher.auto_match=True +>>> Fetcher.encoding="utf8" +>>> Fetcher.keep_comments=False +>>> Fetcher.keep_cdata=False # and the rest +``` +Then, continue your code as usual. + +The available configuration arguments are: `auto_match`, `huge_tree`, `keep_comments`, `keep_cdata`, `storage`, and `storage_args`, which are the same ones you give to the `Adaptor` class. You can display the current configuration anytime by running `.display_config()`. + +> Note: The `auto_match` argument is disabled by default; you must enable it to use that feature. + +### Set parser config per request +As you probably understood, the logic above for setting the parser config will work globally for all requests/fetches done through that class, and it's intended for simplicity. + +If your use case requires a different configuration for each request/fetch, you can pass a dictionary to the request method (`fetch`/`get`/`post`/...) to an argument named `custom_config`. + +## Response Object +The `Response` object is the same as the [Adaptor](../parsing/main_classes.md#adaptor) class, but it has added details about the response like response headers, status, cookies, etc... as shown below: +```python +>>> from scrapling.fetchers import Fetcher +>>> page = Fetcher.get('https://example.com') + +>>> page.status # HTTP status code +>>> page.reason # Status message +>>> page.cookies # Response cookies as a dictionary +>>> page.headers # Response headers +>>> page.request_headers # Request headers +>>> page.history # Response history of redirections, if any +>>> page.body # Raw response body +>>> page.encoding # Response encoding +``` +All fetchers return the `Response` object. \ No newline at end of file diff --git a/docs/fetching/dynamic.md b/docs/fetching/dynamic.md new file mode 100644 index 0000000..08f59ea --- /dev/null +++ b/docs/fetching/dynamic.md @@ -0,0 +1,248 @@ +# Introduction + +Here, we will discuss the `PlayWrightFetcher` class. This class provides flexible browser automation with multiple configuration options and some stealth capabilities. It uses [PlayWright](https://playwright.dev/python/docs/intro) as an engine for fetching websites. + +As we will explain later, to automate the page, you need some knowledge of [PlayWright's Page API](https://playwright.dev/python/docs/api/class-page). + +## Basic Usage +You have one primary way to import this Fetcher, which is the same for all fetchers. + +```python +>>> from scrapling.fetchers import PlayWrightFetcher +``` +Check out how to configure the parsing options [here](choosing.md#parser-configuration-in-all-fetchers) + +Now we will go over most of the arguments one by one with examples if you want to jump to a table of all arguments for quick reference [click here](#full-list-of-arguments) + +> Notes: +> +> 1. Every time you fetch a website with this fetcher, it waits by default for all JavaScript to fully load and execute, so you don't have to (waits for the `domcontentloaded` state). +> 2. Of course, the async version of the `fetch` method is the `async_fetch` method. + + +This fetcher currently provides 4 main run options, but they can be mixed as you want. + +Which are: + +### 1. Vanilla Playwright +```python +PlayWrightFetcher.fetch('https://example.com') +``` +Using it like that will open a Chromium browser and fetch the page. There are no tricks or extra features; it's just a plain PlayWright API. + +### 2. Stealth Mode +```python +PlayWrightFetcher.fetch('https://example.com', stealth=True) +``` +It's the same as the vanilla PlayWright option, but it provides a simple stealth mode suitable for websites with a small-to-medium protection layer(s). + +Some of the things this fetcher's stealth mode does include: + + * Patching the CDP runtime fingerprint. + * Mimics some of the real browsers' properties by injecting several JS files and using custom options. + * Custom flags are used on launch to hide Playwright even more and make it faster. + * Generates real browser headers of the same type and user OS, then append them to the request's headers. + +### 3. Real Chrome +```python +PlayWrightFetcher.fetch('https://example.com', real_chrome=True) +``` +If you have a Google Chrome browser installed, use this option. It's the same as the first option but will use the Google Chrome browser you installed on your device instead of Chromium. + +This will make your requests look more like requests coming from an actual human, so it's less detectable, and you can even use the `stealth=True` mode with it for better results like below: +```python +PlayWrightFetcher.fetch('https://example.com', real_chrome=True, stealth=True) +``` +If you don't have Google Chrome installed and want to use this option, you can use the command below in the terminal to install it for the library instead of installing it manually: +```commandline +playwright install chrome +``` + +### 4. CDP Connection +```python +PlayWrightFetcher.fetch('https://example.com', cdp_url='ws://localhost:9222') +``` +Instead of launching a browser locally (Chromium/Google Chrome), you can connect to a remote browser through the [Chrome DevTools Protocol](https://chromedevtools.github.io/devtools-protocol/). + +This fetcher takes it even a step further. You can use [NSTBrowser](https://app.nstbrowser.io/r/1vO5e5)'s [docker browserless](https://hub.docker.com/r/nstbrowser/browserless) option by passing the CDP URL and enabling `nstbrowser_mode` option like below +```python +PlayWrightFetcher.fetch('https://example.com', cdp_url='ws://localhost:9222', nstbrowser_mode=True) +``` +There's also a `nstbrowser_config` argument to send the config you want to send with the requests to the NSTBrowser. If you leave it empty, Scrapling defaults to an optimized NSTBrowser's docker browserless config. + +## Full list of arguments +Scrapling provides many options with this fetcher, which works in all modes except the [NSTBrowser](https://app.nstbrowser.io/r/1vO5e5) mode. To make it as simple as possible, we will list the options here and give examples of using most of them. + +| Argument | Description | Optional | +|:-------------------:|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:--------:| +| url | Target url | ❌ | +| headless | Pass `True` to run the browser in headless/hidden (**default**) or `False` for headful/visible mode. | ✔️ | +| disable_resources | Drop requests of unnecessary resources for a speed boost. It depends, but it made requests ~25% faster in my tests for some websites.
Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`. _This can help save your proxy usage, but be careful with this option as it makes some websites never finish loading._ | ✔️ | +| useragent | Pass a useragent string to be used. **Otherwise, the fetcher will generate and use a real Useragent of the same browser.** | ✔️ | +| network_idle | Wait for the page until there are no network connections for at least 500 ms. | ✔️ | +| timeout | The timeout (milliseconds) used in all operations and waits through the page. The default is 30000. | ✔️ | +| wait | The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the `Response` object. | ✔️ | +| page_action | Added for automation. Pass a function that takes the `page` object and does the necessary automation, then returns `page` again. | ✔️ | +| wait_selector | Wait for a specific css selector to be in a specific state. | ✔️ | +| wait_selector_state | Scrapling will wait for the given state to be fulfilled for the selector given with `wait_selector`. _Default state is `attached`._ | ✔️ | +| google_search | Enabled by default, Scrapling will set the referer header as if this request came from a Google search for this website's domain name. | ✔️ | +| extra_headers | A dictionary of extra headers to add to the request. The referer set by the `google_search` argument takes priority over the referer set here if used together. | ✔️ | +| proxy | The proxy to be used with requests. It can be a string or a dictionary with the keys 'server', 'username', and 'password' only. | ✔️ | +| hide_canvas | Add random noise to canvas operations to prevent fingerprinting. | ✔️ | +| disable_webgl | Disables WebGL and WebGL 2.0 support entirely. | ✔️ | +| stealth | Enables stealth mode; you should always check the documentation to see what stealth mode does currently. | ✔️ | +| real_chrome | If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch and use an instance of your browser and use it. | ✔️ | +| locale | Set the locale for the browser if wanted. The default value is `en-US`. | ✔️ | +| cdp_url | Instead of launching a new browser instance, connect to this CDP URL to control real browsers/NSTBrowser through CDP. | ✔️ | +| nstbrowser_mode | Enables NSTBrowser mode, **it have to be used with `cdp_url` argument or it will get completely ignored.** | ✔️ | +| nstbrowser_config | The config you want to send with requests to the NSTBrowser. _Scrapling defaults to an optimized NSTBrowser's docker browserless config if you leave this argument empty._ | ✔️ | + + +## Examples +It's easier to understand with examples, so let's look at it. + +### Resource Control + +```python +# Disable unnecessary resources +page = PlayWrightFetcher.fetch( + 'https://example.com', + disable_resources=True # Blocks fonts, images, media, etc... +) +``` + +### Network Control + +```python +# Wait for network idle (Consider fetch to be finished when there are no network connections for at least 500 ms) +page = PlayWrightFetcher.fetch('https://example.com', network_idle=True) + +# Custom timeout (in milliseconds) +page = PlayWrightFetcher.fetch('https://example.com', timeout=30000) # 30 seconds + +# Proxy support +page = PlayWrightFetcher.fetch( + 'https://example.com', + proxy='http://username:password@host:port' # Or it can be a dictionary with the keys 'server', 'username', and 'password' only +) +``` + +### Browser Automation +This is where your knowledge about [PlayWright's Page API](https://playwright.dev/python/docs/api/class-page) comes into play. The function you pass here takes the page object from Playwright's API, does what you want, and then returns it again for the current fetcher to continue working on it. + +This function is executed right after waiting for network_idle (if enabled) and before waiting for the `wait_selector` argument, so it can be used for many things, not just automation. You can alter the page as you want. + +In the example below, I used page [mouse events](https://playwright.dev/python/docs/api/class-mouse) to move the mouse wheel to scroll the page and then move the mouse. +```python +from playwright.sync_api import Page + +def scroll_page(page: Page): + page.mouse.wheel(10, 0) + page.mouse.move(100, 400) + page.mouse.up() + return page + +page = PlayWrightFetcher.fetch( + 'https://example.com', + page_action=scroll_page +) +``` +Of course, if you use the async fetch version, the function must also be async. +```python +from playwright.async_api import Page + +async def scroll_page(page: Page): + await page.mouse.wheel(10, 0) + await page.mouse.move(100, 400) + await page.mouse.up() + return page + +page = await PlayWrightFetcher.async_fetch( + 'https://example.com', + page_action=scroll_page +) +``` + +### Wait Conditions + +```python +# Wait for the selector +page = PlayWrightFetcher.fetch( + 'https://example.com', + wait_selector='h1', + wait_selector_state='visible' +) +``` +This is the last wait the fetcher will do before returning the response (if enabled). You pass a CSS selector to the `wait_selector` argument, and the fetcher will wait for the state you passed in the `wait_selector_state` argument to be fulfilled. If you didn't pass a state, the default would be `attached`, which means it will wait for the element to be present in the DOM. + +After that, the fetcher will check again to see if all JS files are loaded and executed (the `domcontentloaded` state) and wait for them to be. If you have enabled `network_idle` with this, the fetcher will wait for `network_idle` to be fulfilled again, as explained above. + +The states the fetcher can wait for can be either ([source](https://playwright.dev/python/docs/api/class-page#page-wait-for-selector)): + +- `attached`: Wait for an element to be present in DOM. +- `detached`: Wait for an element to not be present in DOM. +- `visible`: wait for an element to have a non-empty bounding box and no `visibility:hidden`. Note that an element without any content or with `display:none` has an empty bounding box and is not considered visible. +- `hidden`: wait for an element to be either detached from DOM, or have an empty bounding box or `visibility:hidden`. This is opposite to the `'visible'` option. + +### Some Stealth Features + +```python +# Full stealth mode +page = PlayWrightFetcher.fetch( + 'https://example.com', + stealth=True, + hide_canvas=True, + disable_webgl=True, + google_search=True +) + +# Custom user agent +page = PlayWrightFetcher.fetch( + 'https://example.com', + useragent='Mozilla/5.0...' +) + +# Set browser locale +page = PlayWrightFetcher.fetch( + 'https://example.com', + locale='en-US' +) +``` +Hence, the `hide_canvas` argument doesn't disable canvas but hides it by adding random noise to canvas operations to prevent fingerprinting. Also, if you didn't set a useragent (preferred), the fetcher will generate a real Useragent of the same browser and use it. + +The `google_search` argument is enabled by default, making the request look like it came from Google. So, a request for `https://example.com` will set the referer to `https://www.google.com/search?q=example`. Also, if used together, it takes priority over the referer set by the `extra_headers` argument. + +### General example +```python +from scrapling.fetchers import PlayWrightFetcher + +def scrape_dynamic_content(): + # Use PlayWright for JavaScript content + page = PlayWrightFetcher.fetch( + 'https://example.com/dynamic', + network_idle=True, + wait_selector='.content' + ) + + # Extract dynamic content + content = page.css('.content') + + return { + 'title': content.css_first('h1::text'), + 'items': [ + item.text for item in content.css('.item') + ] + } +``` + +## When to Use + +Use PlayWrightFetcher when: + +- Need browser automation +- Want multiple browser options +- Using a real Chrome browser +- Need custom browser config +- Want flexible stealth options + +If you want more stealth and control without much config, check out the [StealthyFetcher](stealthy.md). \ No newline at end of file diff --git a/docs/fetching/static.md b/docs/fetching/static.md new file mode 100644 index 0000000..669144c --- /dev/null +++ b/docs/fetching/static.md @@ -0,0 +1,300 @@ +# Introduction + +The `Fetcher` class provides fast and lightweight HTTP requests with some stealth capabilities. This class uses [httpx](https://www.python-httpx.org/) as an engine for making requests. For advanced usages, you will need some knowledge about [httpx](https://www.python-httpx.org/), but it becomes simpler and simpler with user feedback and updates. + +## Basic Usage +You have one primary way to import this Fetcher, which is the same for all fetchers. + +```python +>>> from scrapling.fetchers import Fetcher +``` +Check out how to configure the parsing options [here](choosing.md#parser-configuration-in-all-fetchers) + +### Shared arguments +All methods for making requests here share some arguments, so let's discuss them first. + +- **url**: The URL you want to request, of course :) +- **proxy**: As the name implies, the proxy for this request is used to route all traffic (HTTP and HTTPS). The format accepted here is `http://username:password@localhost:8030`. +- **stealthy_headers**: Generate and use real browser's headers, then create a referer header as if this request came from a Google search page of this URL's domain. Enabled by default, all headers generated can be overwritten by you through the `headers` argument. +- **follow_redirects**: As the name implies, tell the fetcher to follow redirections. Enabled by default +- **timeout**: The timeout to wait for each request to be finished in milliseconds. The default is 30000ms (30 seconds). +- **retries**: The number of retries that [httpx](https://www.python-httpx.org/) will do for failed requests. The default number of retries is 3. + +Other than this, you can pass any arguments that `httpx.` takes, and that's why I said, in the beginning, you need a bit of knowledge about [httpx](https://www.python-httpx.org/), but in the following examples, we will try to cover most cases. + +### HTTP Methods +Examples are the best way to explain this + +> Hence: `OPTIONS` and `HEAD` methods are not supported. +#### GET +```python +>>> from scrapling.fetchers import Fetcher +>>> # Basic GET +>>> page = Fetcher.get('https://example.com') +>>> page = Fetcher.get('https://httpbin.org/get', stealthy_headers=True, follow_redirects=True) +>>> page = Fetcher.get('https://httpbin.org/get', proxy='http://username:password@localhost:8030') +>>> # With parameters +>>> page = Fetcher.get('https://example.com/search', params={'q': 'query'}) +>>> +>>> # With headers +>>> page = Fetcher.get('https://example.com', headers={'User-Agent': 'Custom/1.0'}) +>>> # Basic HTTP authentication +>>> page = Fetcher.get("https://example.com", auth=("my_user", "password123")) +``` +And for asynchronous requests, it's a small adjustment +```python +>>> from scrapling.fetchers import AsyncFetcher +>>> # Basic GET +>>> page = await AsyncFetcher.get('https://example.com') +>>> page = await AsyncFetcher.get('https://httpbin.org/get', stealthy_headers=True, follow_redirects=True) +>>> page = await AsyncFetcher.get('https://httpbin.org/get', proxy='http://username:password@localhost:8030') +>>> # With parameters +>>> page = await AsyncFetcher.get('https://example.com/search', params={'q': 'query'}) +>>> +>>> # With headers +>>> page = await AsyncFetcher.get('https://example.com', headers={'User-Agent': 'Custom/1.0'}) +>>> # Basic HTTP authentication +>>> page = await AsyncFetcher.get("https://example.com", auth=("my_user", "password123")) +``` +Needless to say, the `page` object in all cases is [Response](choosing.md#response-object) object, which is an `Adaptor` as we said, so you will use it directly +```python +>>> page.css('.something.something') + +>>> page = Fetcher.get('https://api.github.com/events') +>>> page.json() +[{'id': '', + 'type': 'PushEvent', + 'actor': {'id': '', + 'login': '', + 'display_login': '', + 'gravatar_id': '', + 'url': 'https://api.github.com/users/', + 'avatar_url': 'https://avatars.githubusercontent.com/u/'}, + 'repo': {'id': '', +... +``` +#### POST +```python +>>> from scrapling.fetchers import Fetcher +>>> # Basic POST +>>> page = Fetcher.post('https://httpbin.org/post', data={'key': 'value'}) +>>> page = Fetcher.post('https://httpbin.org/post', data={'key': 'value'}, stealthy_headers=True, follow_redirects=True) +>>> page = Fetcher.post('https://httpbin.org/post', data={'key': 'value'}, proxy='http://username:password@localhost:8030') +>>> # Another example of form-encoded data +>>> page = Fetcher.post('https://example.com/submit', data={'username': 'user', 'password': 'pass'}) +>>> # JSON data +>>> page = Fetcher.post('https://example.com/api', json={'key': 'value'}) +>>> # Uploading file +>>> r = Fetcher.post("https://httpbin.org/post", files={'upload-file': open('something.xlsx', 'rb')}) +``` +And for asynchronous requests, it's a small adjustment +```python +>>> from scrapling.fetchers import AsyncFetcher +>>> # Basic POST +>>> page = await AsyncFetcher.post('https://httpbin.org/post', data={'key': 'value'}) +>>> page = await AsyncFetcher.post('https://httpbin.org/post', data={'key': 'value'}, stealthy_headers=True, follow_redirects=True) +>>> page = await AsyncFetcher.post('https://httpbin.org/post', data={'key': 'value'}, proxy='http://username:password@localhost:8030') +>>> # Another example of form-encoded data +>>> page = await AsyncFetcher.post('https://example.com/submit', data={'username': 'user', 'password': 'pass'}) +>>> # JSON data +>>> page = await AsyncFetcher.post('https://example.com/api', json={'key': 'value'}) +>>> # Uploading file +>>> r = await AsyncFetcher.post("https://httpbin.org/post", files={'upload-file': open('something.xlsx', 'rb')}) +``` +#### PUT +```python +>>> from scrapling.fetchers import Fetcher +>>> # Basic PUT +>>> page = Fetcher.put('https://example.com/update', data={'status': 'updated'}) +>>> page = Fetcher.put('https://example.com/update', data={'status': 'updated'}, stealthy_headers=True, follow_redirects=True) +>>> page = Fetcher.put('https://example.com/update', data={'status': 'updated'}, proxy='http://username:password@localhost:8030') +>>> # Another example of form-encoded data +>>> page = Fetcher.put("https://httpbin.org/put", data={'key': ['value1', 'value2']}) +``` +And for asynchronous requests, it's a small adjustment +```python +>>> from scrapling.fetchers import AsyncFetcher +>>> # Basic PUT +>>> page = await AsyncFetcher.put('https://example.com/update', data={'status': 'updated'}) +>>> page = await AsyncFetcher.put('https://example.com/update', data={'status': 'updated'}, stealthy_headers=True, follow_redirects=True) +>>> page = await AsyncFetcher.put('https://example.com/update', data={'status': 'updated'}, proxy='http://username:password@localhost:8030') +>>> # Another example of form-encoded data +>>> page = await AsyncFetcher.put("https://httpbin.org/put", data={'key': ['value1', 'value2']}) +``` + +#### DELETE +```python +>>> from scrapling.fetchers import Fetcher +>>> page = Fetcher.delete('https://example.com/resource/123') +>>> page = Fetcher.delete('https://example.com/resource/123', stealthy_headers=True, follow_redirects=True) +>>> page = Fetcher.delete('https://example.com/resource/123', proxy='http://username:password@localhost:8030') +``` +And for asynchronous requests, it's a small adjustment +```python +>>> from scrapling.fetchers import AsyncFetcher +>>> page = await AsyncFetcher.delete('https://example.com/resource/123') +>>> page = await AsyncFetcher.delete('https://example.com/resource/123', stealthy_headers=True, follow_redirects=True) +>>> page = await AsyncFetcher.delete('https://example.com/resource/123', proxy='http://username:password@localhost:8030') +``` + +## Examples +Some well-rounded examples to aid newcomers to Web Scraping + +### Basic HTTP Request + +```python +from scrapling.fetchers import Fetcher + +# Make a request +page = Fetcher.get('https://example.com') + +# Check the status +if page.status == 200: + # Extract title + title = page.css_first('title::text') + print(f"Page title: {title}") + + # Extract all links + links = page.css('a::attr(href)') + print(f"Found {len(links)} links") +``` + +### Product Scraping + +```python +from scrapling.fetchers import Fetcher + +def scrape_products(): + page = Fetcher.get('https://example.com/products') + + # Find all product elements + products = page.css('.product') + + results = [] + for product in products: + results.append({ + 'title': product.css_first('.title::text'), + 'price': product.css_first('.price::text').re_first(r'\d+\.\d{2}'), + 'description': product.css_first('.description::text'), + 'in_stock': product.has_class('in-stock') + }) + + return results +``` + +### Pagination Handling + +```python +from scrapling.fetchers import Fetcher + +def scrape_all_pages(): + base_url = 'https://example.com/products?page={}' + page_num = 1 + all_products = [] + + while True: + # Get current page + page = Fetcher.get(base_url.format(page_num)) + + # Find products + products = page.css('.product') + if not products: + break + + # Process products + for product in products: + all_products.append({ + 'name': product.css_first('.name::text'), + 'price': product.css_first('.price::text') + }) + + # Next page + page_num += 1 + + return all_products +``` + +### Form Submission + +```python +from scrapling.fetchers import Fetcher + +# Submit login form +response = Fetcher.post( + 'https://example.com/login', + data={ + 'username': 'user@example.com', + 'password': 'password123' + } +) + +# Check login success +if response.status == 200: + # Extract user info + user_name = response.css_first('.user-name::text') + print(f"Logged in as: {user_name}") +``` + +### Table Extraction + +```python +from scrapling.fetchers import Fetcher + +def extract_table(): + page = Fetcher.get('https://example.com/data') + + # Find table + table = page.css_first('table') + + # Extract headers + headers = [ + th.text for th in table.css('thead th') + ] + + # Extract rows + rows = [] + for row in table.css('tbody tr'): + cells = [td.text for td in row.css('td')] + rows.append(dict(zip(headers, cells))) + + return rows +``` + +### Navigation Menu + +```python +from scrapling.fetchers import Fetcher + +def extract_menu(): + page = Fetcher.get('https://example.com') + + # Find navigation + nav = page.css_first('nav') + + menu = {} + for item in nav.css('li'): + link = item.css_first('a') + if link: + menu[link.text] = { + 'url': link.attrib['href'], + 'has_submenu': bool(item.css('.submenu')) + } + + return menu +``` + +## When to Use + +Use `Fetcher` when: + +- Need fast HTTP requests +- Want minimal overhead +- Don't need JavaScript +- Want simple configuration +- Need basic stealth features + +Use other fetchers when: + +- Need browser automation. +- Need advanced anti-bot/stealth. +- Need JavaScript support. \ No newline at end of file diff --git a/docs/fetching/stealthy.md b/docs/fetching/stealthy.md new file mode 100644 index 0000000..df3251c --- /dev/null +++ b/docs/fetching/stealthy.md @@ -0,0 +1,218 @@ +# Introduction + +Here, we will discuss the `StealthyFetcher` class. This class is similar to [PlayWrightFetcher](dynamic.md#introduction) in many ways, like browser automation and using [PlayWright](https://playwright.dev/python/docs/intro) as an engine for fetching websites. The main difference is that this class provides advanced anti-bot protection bypass capabilities and a modified Firefox browser called [Camoufox](https://github.com/daijro/camoufox), from which most stealth comes. + +As with [PlayWrightFetcher](dynamic.md#introduction), you will need some knowledge about [PlayWright's Page API](https://playwright.dev/python/docs/api/class-page) to automate the page, as we will explain later. + +## Basic Usage +You have one primary way to import this Fetcher, which is the same for all fetchers. + +```python +>>> from scrapling.fetchers import StealthyFetcher +``` +Check out how to configure the parsing options [here](choosing.md#parser-configuration-in-all-fetchers) + +> Notes: +> +> 1. Every time you fetch a website with this fetcher, it waits by default for all JavaScript to fully load and execute, so you don't have to (waits for the `domcontentloaded` state). +> 2. Of course, the async version of the `fetch` method is the `async_fetch` method. + +## Full list of arguments +Before jumping to [examples](#examples), here's the full list of arguments + + +| Argument | Description | Optional | +|:--------------------:|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:--------:| +| url | Target url | ❌ | +| headless | Pass `True` to run the browser in headless/hidden (**default**), `virtual` to run it in virtual screen mode, or `False` for headful/visible mode. The `virtual` mode requires having `xvfb` installed. | ✔️ | +| block_images | Prevent the loading of images through Firefox preferences. _This can help save your proxy usage, but be careful with this option as it makes some websites never finish loading._ | ✔️ | +| disable_resources | Drop requests of unnecessary resources for a speed boost. It depends, but it made requests ~25% faster in my tests for some websites.
Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`. _This can help save your proxy usage, but be careful with this option as it makes some websites never finish loading._ | ✔️ | +| google_search | Enabled by default, Scrapling will set the referer header as if this request came from a Google search for this website's domain name. | ✔️ | +| extra_headers | A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ | ✔️ | +| block_webrtc | Blocks WebRTC entirely. | ✔️ | +| page_action | Added for automation. A function that takes the `page` object and does the automation you need, then returns `page` again. | ✔️ | +| addons | List of Firefox addons to use. **Must be paths to extracted addons.** | ✔️ | +| humanize | Humanize the cursor movement. The cursor movement takes either True or the MAX duration in seconds. The cursor typically takes up to 1.5 seconds to move across the window. | ✔️ | +| allow_webgl | Enabled by default. Disabling WebGL is not recommended, as many WAFs now check if WebGL is enabled. | ✔️ | +| geoip | Recommended to use with proxies; Automatically use IP's longitude, latitude, timezone, country, locale, & spoof the WebRTC IP address. It will also calculate and spoof the browser's language based on the distribution of language speakers in the target region. | ✔️ | +| os_randomize | If enabled, Scrapling will randomize the OS fingerprints used. The default is matching the fingerprints with the current OS. | ✔️ | +| disable_ads | Disabled by default; this installs the `uBlock Origin` addon on the browser if enabled. | ✔️ | +| network_idle | Wait for the page until there are no network connections for at least 500 ms. | ✔️ | +| timeout | The timeout used in all operations and waits through the page. It's in milliseconds, and the default is 30000. | ✔️ | +| wait | The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the `Response` object. | ✔️ | +| wait_selector | Wait for a specific css selector to be in a specific state. | ✔️ | +| wait_selector_state | Scrapling will wait for the given state to be fulfilled for the selector given with `wait_selector`. _Default state is `attached`._ | ✔️ | +| proxy | The proxy to be used with requests. It can be a string or a dictionary with the keys 'server', 'username', and 'password' only. | ✔️ | +| additional_arguments | Arguments passed to Camoufox as additional settings that take higher priority than Scrapling's. | ✔️ | + + +## Examples +It's easier to understand with examples, so now we will go over most of the arguments individually with examples. + +### Browser Modes + +```python +# Headless/hidden mode (default) +page = StealthyFetcher.fetch('https://example.com', headless=True) + +# Virtual display mode (requires having `xvfb` installed) +page = StealthyFetcher.fetch('https://example.com', headless='virtual') + +# Visible browser mode +page = StealthyFetcher.fetch('https://example.com', headless=False) +``` + +### Resource Control + +```python +# Block images +page = StealthyFetcher.fetch('https://example.com', block_images=True) + +# Disable unnecessary resources +page = StealthyFetcher.fetch('https://example.com', disable_resources=True) # Blocks fonts, images, media, etc. +``` + +### Additional stealth options + +```python +page = StealthyFetcher.fetch( + 'https://example.com', + block_webrtc=True, # Block WebRTC + allow_webgl=False, # Disable WebGL + humanize=True, # Make the mouse move as how a human would move it + geoip=True, # Use IP's longitude, latitude, timezone, country, and locale, then spoof the WebRTC IP address... + os_randomize=True, # Randomize the OS fingerprints used. The default is matching the fingerprints with the current OS. + disable_ads=True, # Block ads with uBlock Origin addon (enabled by default) + google_search=True +) + +# Custom user agent +page = StealthyFetcher.fetch( + 'https://example.com', + useragent='Mozilla/5.0...' +) + +# Custom humanization duration +page = StealthyFetcher.fetch( + 'https://example.com', + humanize=1.5 # Max 1.5 seconds for cursor movement +) +``` + +The `google_search` argument is enabled by default. It makes the request as if it came from Google, so for a request for `https://example.com`, it will set the referer to `https://www.google.com/search?q=example`. Also, if used together, it takes priority over the referer set by the `extra_headers` argument. + +### Network Control + +```python +# Wait for network idle (Consider fetch to be finished when there are no network connections for at least 500 ms) +page = StealthyFetcher.fetch('https://example.com', network_idle=True) + +# Custom timeout (in milliseconds) +page = StealthyFetcher.fetch('https://example.com', timeout=30000) # 30 seconds + +# Proxy support +page = StealthyFetcher.fetch( + 'https://example.com', + proxy='http://username:password@host:port' # Or it can be a dictionary with the keys 'server', 'username', and 'password' only +) +``` + +### Browser Automation +This is where your knowledge about [PlayWright's Page API](https://playwright.dev/python/docs/api/class-page) comes into play. The function you pass here takes the page object from Playwright's API, does what you want, and then returns it again for the current fetcher to continue working on it. + +This function is executed right after waiting for `network_idle` (if enabled) and before waiting for the `wait_selector` argument, so it can be used for many things, not just automation. You can alter the page as you want. + +In the example below, I used page [mouse events](https://playwright.dev/python/docs/api/class-mouse) to move the mouse wheel to scroll the page and then move the mouse. +```python +from playwright.sync_api import Page + +def scroll_page(page: Page): + page.mouse.wheel(10, 0) + page.mouse.move(100, 400) + page.mouse.up() + return page + +page = StealthyFetcher.fetch( + 'https://example.com', + page_action=scroll_page +) +``` +Of course, if you use the async fetch version, the function must also be async. +```python +from playwright.async_api import Page + +async def scroll_page(page: Page): + await page.mouse.wheel(10, 0) + await page.mouse.move(100, 400) + await page.mouse.up() + return page + +page = await StealthyFetcher.async_fetch( + 'https://example.com', + page_action=scroll_page +) +``` + +### Wait Conditions +```python +# Wait for the selector +page = StealthyFetcher.fetch( + 'https://example.com', + wait_selector='h1', + wait_selector_state='visible' +) +``` +This is the last wait the fetcher will do before returning the response (if enabled). You pass a CSS selector to the `wait_selector` argument, and the fetcher will wait for the state you passed in the `wait_selector_state` argument to be fulfilled. If you didn't pass a state, the default would be `attached`, which means it will wait for the element to be present in the DOM. + +After that, the fetcher will check again to see if all JS files are loaded and executed (the `domcontentloaded` state) and wait for them to be. If you have enabled `network_idle` with this, the fetcher will wait for `network_idle` to be fulfilled again, as explained above. + +The states the fetcher can wait for can be either ([source](https://playwright.dev/python/docs/api/class-page#page-wait-for-selector)): + +- `attached`: wait for the element to be present in DOM. +- `detached`: wait for the element to not be present in DOM. +- `visible`: wait for the element to have a non-empty bounding box and no `visibility:hidden`. Note that an element without any content or with `display:none` has an empty bounding box and is not considered visible. +- `hidden`: Wait for the element to be detached from DOM, have an empty bounding box, or have `visibility:hidden`. This is opposite to the `'visible'` option. + +### Firefox Addons + +```python +# Custom Firefox addons +page = StealthyFetcher.fetch( + 'https://example.com', + addons=['/path/to/addon1', '/path/to/addon2'] +) +``` +The paths here must be paths of extracted addons, which will be installed automatically upon browser launch. + +### Real-world example (Amazon) +This is for educational purposes only; this example was generated by AI, which shows too how easy it is to work with Scrapling through AI +```python +def scrape_amazon_product(url): + # Use StealthyFetcher to bypass protection + page = StealthyFetcher.fetch(url) + + # Extract product details + return { + 'title': page.css_first('#productTitle::text').clean(), + 'price': page.css_first('.a-price .a-offscreen::text'), + 'rating': page.css_first('[data-feature-name="averageCustomerReviews"] .a-popover-trigger .a-color-base::text'), + 'reviews_count': page.css('#acrCustomerReviewText::text').re_first(r'[\d,]+'), + 'features': [ + li.clean() for li in page.css('#feature-bullets li span::text') + ], + 'availability': page.css_first('#availability').get_all_text(strip=True), + 'images': [ + img.attrib['src'] for img in page.css('#altImages img') + ] + } +``` + +## When to Use + +Use StealthyFetcher when: + +- Bypassing anti-bot protection +- Need a reliable browser fingerprint +- Full JavaScript support needed +- Want automatic stealth features +- Need browser automation \ No newline at end of file diff --git a/docs/index.md b/docs/index.md index 5987678..3316d07 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,2 +1,107 @@ -# This section is still under work but any help is highly appreciated -## I will try to make full detailed documentation with Sphinx ASAP. \ No newline at end of file +# Scrapling + +Scrapling is an Undetectable, high-performance, intelligent Web scraping library for Python 3 to make Web Scraping easy! + +Scrapling isn't only about making undetectable requests or fetching pages under the radar! + +It has its own parser that adapts to website changes and provides many element selection/querying options other than traditional selectors, powerful DOM traversal API, and many other features while significantly outperforming popular parsing alternatives. + +Scrapling is built from the ground up by Web scraping experts for beginners and experts. The goal is to provide powerful features while maintaining simplicity and minimal boilerplate code. + +```python +>> from scrapling.fetchers import Fetcher, AsyncFetcher, StealthyFetcher, PlayWrightFetcher +>> StealthyFetcher.auto_match = True +# Fetch websites' source under the radar! +>> page = StealthyFetcher.fetch('https://example.com', headless=True, network_idle=True) +>> print(page.status) +200 +>> products = page.css('.product', auto_save=True) # Scrape data that survives website design changes! +>> # Later, if the website structure changes, pass `auto_match=True` +>> products = page.css('.product', auto_match=True) # and Scrapling still finds them! +``` +## Key Features +### Fetch websites as you prefer with async support +- **HTTP Requests**: Fast and stealthy HTTP requests with the `Fetcher` class. +- **Dynamic Loading & Automation**: Fetch dynamic websites with the `PlayWrightFetcher` class through your real browser, Scrapling's stealth mode, Playwright's Chromium browser, or [NSTbrowser](https://app.nstbrowser.io/r/1vO5e5)'s browserless! +- **Anti-bot Protections Bypass**: Easily bypass protections with the `StealthyFetcher` and `PlayWrightFetcher` classes. + +### Easy Scraping +- **Smart Element Tracking**: Relocate elements after website changes using an intelligent similarity system and integrated storage. +- **Flexible Selection**: CSS selectors, XPath selectors, filters-based search, text search, regex search, and more. +- **Find Similar Elements**: Automatically locate elements similar to the element you found! +- **Smart Content Scraping**: Extract data from multiple websites without specific selectors using Scrapling powerful features. + +### High Performance +- **Lightning Fast**: Built from the ground up with performance in mind, outperforming most popular Python scraping libraries. +- **Memory Efficient**: Optimized data structures for minimal memory footprint. +- **Fast JSON serialization**: 10x faster than standard library. + +### Developer Friendly +- **Powerful Navigation API**: Easy DOM traversal in all directions. +- **Rich Text Processing**: All strings have built-in regex, cleaning methods, and more. All elements' attributes are optimized dictionaries that use less memory than standard dictionaries with added methods. +- **Auto Selectors Generation**: Generate robust short and full CSS/XPath selectors for any element. +- **Familiar API**: Similar to Scrapy/BeautifulSoup and the same CSS pseudo-elements used in Scrapy. +- **Type hints**: Complete type/doc-strings coverage for future-proofing and best autocompletion support. + +## Star History +Scrapling’s GitHub stars have grown steadily since its release (see chart below). + + + + + +## Installation +Scrapling is a breeze to get started with!
Starting from version 0.2.9, we require at least Python 3.9 to work. + +Run this command to install it with Python's pip. +```bash +pip3 install scrapling +``` +You are ready if you plan to use the parser only (the `Adaptor` class). + +But if you are going to make requests or fetch pages with Scrapling, then run this command to install browsers' dependencies needed to use the Fetchers +```bash +scrapling install +``` +If you have any installation issues, please open an [issue](https://github.com/D4Vinci/Scrapling/issues/new/choose). + +## How the documentation is organized +Scrapling has a lot of documentation, so we try to follow a guideline called the [Diátaxis documentation framework](https://diataxis.fr/). + +## Support + +If you like Scrapling and want to support its development: + +- ⭐ Star the [GitHub repository](https://github.com/D4Vinci/Scrapling) +- 💝 Consider [sponsoring the project or buying me a coffe](donate.md) :wink: +- 🐛 Report bugs and suggest features through [GitHub Issues](https://github.com/D4Vinci/Scrapling/issues) + +## License + +This project is licensed under BSD-3 License. See the [LICENSE](https://github.com/D4Vinci/Scrapling/blob/main/LICENSE) file for details. \ No newline at end of file diff --git a/docs/overview.md b/docs/overview.md new file mode 100644 index 0000000..99224e1 --- /dev/null +++ b/docs/overview.md @@ -0,0 +1,328 @@ +We will start by quickly reviewing the parsing capabilities. Then, we will fetch websites with custom browsers, make requests, and parse the response. + +Here's an HTML document generated by ChatGPT we will be using as an example throughout this page: +```html + + + Complex Web Page + + + +
+ +
+
+
+

Products

+
+
+

Product 1

+

This is product 1

+ $10.99 + +
+ +
+

Product 2

+

This is product 2

+ $20.99 + +
+ +
+

Product 3

+

This is product 3

+ $15.99 + +
+
+
+ +
+

Customer Reviews

+
+
+

Great product!

+ John Doe +
+
+

Good value for money.

+ Jane Smith +
+
+
+
+ + + +``` +Starting with loading raw HTML above like this +```python +from scrapling.parser import Adaptor +page = Adaptor(html_doc) +page # Complex Web Page</tit...'> +``` +Get all text content on the page recursively +```python +page.get_all_text(ignore_tags=('script', 'style')) +# 'Complex Web Page\nHome\nAbout\nContact\nProducts\nProduct 1\nThis is product 1\n$10.99\nIn stock: 5\nProduct 2\nThis is product 2\n$20.99\nIn stock: 3\nProduct 3\nThis is product 3\n$15.99\nOut of stock\nCustomer Reviews\nGreat product!\nJohn Doe\nGood value for money.\nJane Smith' +``` + +## Finding elements +If there's an element you want to find on the page, you will! Your creativity level is the only limitation! + +Finding the first HTML `section` element +```python +section_element = page.find('section') +# <data='<section id="products" schema='{"jsonabl...' parent='<main><section id="products" schema='{"j...'> +``` +Find all `section` elements +```python +section_elements = page.find_all('section') +# [<data='<section id="products" schema='{"jsonabl...' parent='<main><section id="products" schema='{"j...'>, <data='<section id="reviews"><h2>Customer Revie...' parent='<main><section id="products" schema='{"j...'>] +``` +Find all `section` elements whose `id` attribute value is `products` +```python +section_elements = page.find_all('section', {'id':"products"}) +# Same as +section_elements = page.find_all('section', id="products") +# [<data='<section id="products" schema='{"jsonabl...' parent='<main><section id="products" schema='{"j...'>] +``` +Find all `section` elements that its `id` attribute value contains `product` +```python +section_elements = page.find_all('section', {'id*':"product"}) +``` +Find all `h3` elements whose text content matches this regex `Product \d` +```python +page.find_all('h3', re.compile(r'Product \d')) +# [<data='<h3>Product 1</h3>' parent='<article class="product" data-id="1"><h3...'>, <data='<h3>Product 2</h3>' parent='<article class="product" data-id="2"><h3...'>, <data='<h3>Product 3</h3>' parent='<article class="product" data-id="3"><h3...'>] +``` +Find all `h3` and `h2` elements whose text content matches regex `Product` only +```python +page.find_all(['h3', 'h2'], re.compile(r'Product')) +# [<data='<h3>Product 1</h3>' parent='<article class="product" data-id="1"><h3...'>, <data='<h3>Product 2</h3>' parent='<article class="product" data-id="2"><h3...'>, <data='<h3>Product 3</h3>' parent='<article class="product" data-id="3"><h3...'>, <data='<h2>Products</h2>' parent='<section id="products" schema='{"jsonabl...'>] +``` +Find all elements that its text content matches exactly `Products` (Whitespaces are not taken into consideration) +```python +page.find_by_text('Products', first_match=False) +# [<data='<h2>Products</h2>' parent='<section id="products" schema='{"jsonabl...'>] +``` +Or find all elements whose text content matches regex `Product \d` +```python +page.find_by_regex(r'Product \d', first_match=False) +# [<data='<h3>Product 1</h3>' parent='<article class="product" data-id="1"><h3...'>, <data='<h3>Product 2</h3>' parent='<article class="product" data-id="2"><h3...'>, <data='<h3>Product 3</h3>' parent='<article class="product" data-id="3"><h3...'>] +``` +Find all elements that are similar to the element you want +```python +target_element = page.find_by_regex(r'Product \d', first_match=True) +# <data='<h3>Product 1</h3>' parent='<article class="product" data-id="1"><h3...'> +target_element.find_similar() +# [<data='<h3>Product 2</h3>' parent='<article class="product" data-id="2"><h3...'>, <data='<h3>Product 3</h3>' parent='<article class="product" data-id="3"><h3...'>] +``` +Find the first element that matches a CSS selector +```python +page.css_first('.product-list [data-id="1"]') +# <data='<article class="product" data-id="1"><h3...' parent='<div class="product-list"> <article clas...'> +``` +Find all elements that match a CSS selector +```python +page.css('.product-list article') +# [<data='<article class="product" data-id="1"><h3...' parent='<div class="product-list"> <article clas...'>, <data='<article class="product" data-id="2"><h3...' parent='<div class="product-list"> <article clas...'>, <data='<article class="product" data-id="3"><h3...' parent='<div class="product-list"> <article clas...'>] +``` +Find the first element that matches an XPath selector +```python +page.xpath_first("//*[@id='products']/div/article") +# <data='<article class="product" data-id="1"><h3...' parent='<div class="product-list"> <article clas...'> +``` +Find all elements that match an XPath selector +```python +page.xpath("//*[@id='products']/div/article") +# [<data='<article class="product" data-id="1"><h3...' parent='<div class="product-list"> <article clas...'>, <data='<article class="product" data-id="2"><h3...' parent='<div class="product-list"> <article clas...'>, <data='<article class="product" data-id="3"><h3...' parent='<div class="product-list"> <article clas...'>] +``` + +With this, we just scratched the surface of these functions; more advanced options with these selection methods are shown later. +## Accessing elements' data +It's as simple as +```python +>>> section_element.tag +'section' +>>> print(section_element.attrib) +{'id': 'products', 'schema': '{"jsonable": "data"}'} +>>> section_element.attrib['schema'].json() # If an attribute value can be converted to json, then use `.json()` to convert it +{'jsonable': 'data'} +>>> section_element.text # Direct text content +'' +>>> section_element.get_all_text() # All text content recursively +'Products\nProduct 1\nThis is product 1\n$10.99\nIn stock: 5\nProduct 2\nThis is product 2\n$20.99\nIn stock: 3\nProduct 3\nThis is product 3\n$15.99\nOut of stock' +>>> section_element.html_content # The HTML content of the element +'<section id="products" schema=\'{"jsonable": "data"}\'><h2>Products</h2>\n <div class="product-list">\n <article class="product" data-id="1"><h3>Product 1</h3>\n <p class="description">This is product 1</p>\n <span class="price">$10.99</span>\n <div class="hidden stock">In stock: 5</div>\n </article><article class="product" data-id="2"><h3>Product 2</h3>\n <p class="description">This is product 2</p>\n <span class="price">$20.99</span>\n <div class="hidden stock">In stock: 3</div>\n </article><article class="product" data-id="3"><h3>Product 3</h3>\n <p class="description">This is product 3</p>\n <span class="price">$15.99</span>\n <div class="hidden stock">Out of stock</div>\n </article></div>\n </section>' +>>> print(section_element.prettify()) # The prettified version +''' +<section id="products" schema='{"jsonable": "data"}'><h2>Products</h2> + <div class="product-list"> + <article class="product" data-id="1"><h3>Product 1</h3> + <p class="description">This is product 1</p> + <span class="price">$10.99</span> + <div class="hidden stock">In stock: 5</div> + </article><article class="product" data-id="2"><h3>Product 2</h3> + <p class="description">This is product 2</p> + <span class="price">$20.99</span> + <div class="hidden stock">In stock: 3</div> + </article><article class="product" data-id="3"><h3>Product 3</h3> + <p class="description">This is product 3</p> + <span class="price">$15.99</span> + <div class="hidden stock">Out of stock</div> + </article> + </div> +</section> +''' +>>> section_element.path # All the ancestors in the DOM tree of this element +[<data='<main><section id="products" schema='{"j...' parent='<body> <header><nav><ul><li> <a href="#h...'>, + <data='<body> <header><nav><ul><li> <a href="#h...' parent='<html><head><title>Complex Web Page</tit...'>, + <data='<html><head><title>Complex Web Page</tit...'>] +>>> section_element.generate_css_selector +'#products' +>>> section_element.generate_full_css_selector +'body > main > #products > #products' +>>> section_element.generate_xpath_selector +"//*[@id='products']" +>>> section_element.generate_full_xpath_selector +"//body/main/*[@id='products']" +``` + +## Navigation +Using the elements we found above + +```python +>>> section_element.parent +<data='<main><section id="products" schema='{"j...' parent='<body> <header><nav><ul><li> <a href="#h...'> +>>> section_element.parent.tag +'main' +>>> section_element.parent.parent.tag +'body' +>>> section_element.children +[<data='<h2>Products</h2>' parent='<section id="products" schema='{"jsonabl...'>, + <data='<div class="product-list"> <article clas...' parent='<section id="products" schema='{"jsonabl...'>] +>>> section_element.siblings +[<data='<section id="reviews"><h2>Customer Revie...' parent='<main><section id="products" schema='{"j...'>] +>>> section_element.next # gets the next element, the same logic applies to `quote.previous` +<data='<section id="reviews"><h2>Customer Revie...' parent='<main><section id="products" schema='{"j...'> +>>> section_element.children.css('h2::text') +['Products'] +>>> page.css_first('[data-id="1"]').has_class('product') +True +``` +If your case needs more than the element's parent, you can iterate over the whole ancestors' tree of any element like the one below +```python +for ancestor in quote.iterancestors(): + # do something with it... +``` +You can search for a specific ancestor of an element that satisfies a function; all you need to do is to pass a function that takes an `Adaptor` object as an argument and return `True` if the condition satisfies or `False` otherwise like below: +```python +>>> section_element.find_ancestor(lambda ancestor: ancestor.css('nav')) +<data='<body> <header><nav><ul><li> <a href="#h...' parent='<html><head><title>Complex Web Page</tit...'> +``` + +## Fetching websites +Instead of passing the raw HTML to Scrapling, you can get a website's response directly through HTTP requests or by fetching it from browsers. + +A fetcher is made for every use case. + +### HTTP Requests +For simple HTTP requests, there's a `Fetcher` class that can be imported as below: +```python +from scrapling.fetchers import Fetcher +``` +But that's class, so you will need to create an instance of the Fetcher first like this: +```python +from scrapling.fetchers import Fetcher +fetcher = Fetcher() +page = fetcher.get('https://httpbin.org/get') +``` +This is intended, and you will find it with all fetchers because there are settings you can pass to `Fetcher()` initialization, but more on this later. + +If you are going to use the default settings anyway, you can do this instead for a cleaner approach: +```python +from scrapling.fetchers import Fetcher +page = Fetcher.get('https://httpbin.org/get') +``` +With that out of the way, here's how to do all HTTP methods: +```python +>>> from scrapling.fetchers import Fetcher +>>> page = Fetcher.get('https://httpbin.org/get', stealthy_headers=True, follow_redirects=True) +>>> page = Fetcher.post('https://httpbin.org/post', data={'key': 'value'}, proxy='http://username:password@localhost:8030') +>>> page = Fetcher.put('https://httpbin.org/put', data={'key': 'value'}) +>>> page = Fetcher.delete('https://httpbin.org/delete') +``` +For Async requests, you will just replace the import like below: +```python +>>> from scrapling.fetchers import AsyncFetcher +>>> page = await AsyncFetcher.get('https://httpbin.org/get', stealthy_headers=True, follow_redirects=True) +>>> page = await AsyncFetcher.post('https://httpbin.org/post', data={'key': 'value'}, proxy='http://username:password@localhost:8030') +>>> page = await AsyncFetcher.put('https://httpbin.org/put', data={'key': 'value'}) +>>> page = await AsyncFetcher.delete('https://httpbin.org/delete') +``` + +> Note: You have the `stealthy_headers` argument, which, when enabled, makes requests to generate real browser headers and use them, including a referer header, as if this request came from Google's search of this URL's domain. It's enabled by default. + +This is just the tip of this fetcher; check the full page from [here](fetching/static.md) + +### Dynamic loading +We have you covered if you deal with dynamic websites like most today! + +The `PlayWrightFetcher` class provides many options to fetch/load websites' pages through browsers. +```python +>>> from scrapling.fetchers import PlayWrightFetcher +>>> page = PlayWrightFetcher.fetch('https://www.google.com/search?q=%22Scrapling%22', disable_resources=True) # Vanilla Playwright option +>>> page.css_first("#search a::attr(href)") +'https://github.com/D4Vinci/Scrapling' +>>> # The async version of fetch +>>> page = await PlayWrightFetcher.async_fetch('https://www.google.com/search?q=%22Scrapling%22', disable_resources=True) +>>> page.css_first("#search a::attr(href)") +'https://github.com/D4Vinci/Scrapling' +``` +It's named like that because it's built on top of [Playwright](https://playwright.dev/python/), and it currently provides 4 main run options that can be mixed as you want: + +- Vanilla Playwright without any modifications other than the ones you chose. +- Stealthy Playwright with custom stealth mode explicitly written for it. It's not top-tier stealth mode but bypasses many online tests like [Sannysoft's](https://bot.sannysoft.com/). Check out the `StealthyFetcher` class below for more advanced stealth mode. +- Real browsers by passing the `real_chrome` argument or the CDP URL of your browser to be controlled by the Fetcher, and most of the options can be enabled on it. +- [NSTBrowser](https://app.nstbrowser.io/r/1vO5e5)'s [docker browserless](https://hub.docker.com/r/nstbrowser/browserless) option by passing the CDP URL and enabling `nstbrowser_mode` option. + +> Note: All requests done by this fetcher are waited by default for all javascript to be fully loaded and executed. In detail, it waits for the `load` and `domcontentloaded` load states to be reached; you can make it wait for the `networkidle` load state by passing 'network_idle=True', as you will see later. + +Again, this is just the tip of this fetcher. Check out the full page from [here](fetching/dynamic.md) for all details and the complete list of arguments. + +### Dynamic anti-protection loading +We also have you covered if you deal with dynamic websites with annoying anti-protections! + +The `StealthyFetcher` class uses a modified Firefox browser called [Camoufox](https://github.com/daijro/camoufox), bypassing most anti-bot protections by default. Scrapling adds extra layers of flavors and configurations to further increase performance and undetectability. +```python +>>> page = StealthyFetcher().fetch('https://www.browserscan.net/bot-detection') # Running headless by default +>>> page.status == 200 +True +>>> page = StealthyFetcher().fetch('https://www.browserscan.net/bot-detection', humanize=True, os_randomize=True) # and the rest of arguments... +>>> # The async version of fetch +>>> page = await StealthyFetcher().async_fetch('https://www.browserscan.net/bot-detection') +>>> page.status == 200 +True +``` +> Note: All requests done by this fetcher are waited by default for all javascript to be fully loaded and executed. In detail, it waits for the `load` and `domcontentloaded` load states to be reached; you can make it wait for the `networkidle` load state by passing 'network_idle=True', as you will see later. + +Again, this is just the tip of this fetcher. Check out the full page from [here](fetching/dynamic.md) for all details and the complete list of arguments. + +--- + +That's Scrapling at a glance. If you want to learn more about it, continue to the next section. \ No newline at end of file diff --git a/docs/parsing/automatch.md b/docs/parsing/automatch.md new file mode 100644 index 0000000..ab959a9 --- /dev/null +++ b/docs/parsing/automatch.md @@ -0,0 +1,220 @@ +## Introduction +Auto-matching is one of Scrapling's most powerful features. It allows your scraper to survive website changes by intelligently tracking and relocating elements. + +Let's say you are scraping a page with a structure like this: +```html +<div class="container"> + <section class="products"> + <article class="product" id="p1"> + <h3>Product 1</h3> + <p class="description">Description 1</p> + </article> + <article class="product" id="p2"> + <h3>Product 2</h3> + <p class="description">Description 2</p> + </article> + </section> +</div> +``` +And you want to scrape the first product, the one with the `p1` ID. You will probably write a selector like this +```python +page.css('#p1') +``` +When website owners implement structural changes like +```html +<div class="new-container"> + <div class="product-wrapper"> + <section class="products"> + <article class="product new-class" data-id="p1"> + <div class="product-info"> + <h3>Product 1</h3> + <p class="new-description">Description 1</p> + </div> + </article> + <article class="product new-class" data-id="p2"> + <div class="product-info"> + <h3>Product 2</h3> + <p class="new-description">Description 2</p> + </div> + </article> + </section> + </div> +</div> +``` +The selector will no longer function, and your code needs maintenance. That's where Scrapling's auto-matching feature comes into play. + +With Scrapling, you can enable the `automatch` feature the first time you select an element, and the next time you select that element and it doesn't exist, Scrapling will remember its properties and search on the website for the element with the highest percentage of similarity to that element and without AI :) + +```python +from scrapling import Adaptor, Fetcher +# Before the change +page = Adaptor(page_source, auto_match=True, url='example.com') +# or +Fetcher.auto_match = True +page = Fetcher.get('https://example.com') +# then +element = page.css('#p1' auto_save=True) +if not element: # One day website changes? + element = page.css('#p1', auto_match=True) # Scrapling still finds it! +# the rest of your code... +``` +Below, I will show you one usage example for this feature. Then, we will dive deep into how to use it and provide details about this feature. + +## Real-World Scenario +Let's use a real website as an example and use one of the fetchers to fetch its source. To do this, we need to find a website that will soon change its design/structure, take a copy of its source, and then wait for the website to make the change. Of course, that's nearly impossible to know unless I know the website's owner, but that will make it a staged test, haha. + +To solve this issue, I will use [The Web Archive](https://archive.org/)'s [Wayback Machine](https://web.archive.org/). Here is a copy of [StackOverFlow's website in 2010](https://web.archive.org/web/20100102003420/http://stackoverflow.com/); pretty old, eh?</br>Let's test if the automatch feature can extract the same button in the old design from 2010 and the current design using the same selector :) + +If I want to extract the Questions button from the old design, I can use a selector like this: `#hmenus > div:nth-child(1) > ul > li:nth-child(1) > a` This selector is too specific because it was generated by Google Chrome. + + +Now, let's test the same selector in both versions +```python +>> from scrapling import Fetcher +>> selector = '#hmenus > div:nth-child(1) > ul > li:nth-child(1) > a' +>> old_url = "https://web.archive.org/web/20100102003420/http://stackoverflow.com/" +>> new_url = "https://stackoverflow.com/" +>> Fetcher.configure(auto_match = True, automatch_domain='stackoverflow.com') +>> +>> page = Fetcher.get(old_url, timeout=30) +>> element1 = page.css_first(selector, auto_save=True) +>> +>> # Same selector but used in the updated website +>> page = Fetcher.get(new_url) +>> element2 = page.css_first(selector, auto_match=True) +>> +>> if element1.text == element2.text: +... print('Scrapling found the same element in the old and new designs!') +'Scrapling found the same element in the old and new designs!' +``` +Note that I used a new argument called `automatch_domain`; this is because, for Scrapling, these are two different domains(`archive.org` and `stackoverflow.com`), so scrapling will isolate their `auto_match` data. To tell Scrapling they are the same website, we need to pass the custom domain we want to use while saving auto-match data for them both so Scrapling doesn't isolate them. + +The code will be the same in a real-world scenario, except it will use the same URL for both requests, so you won't need to use the `automatch_domain` argument. This is the closest example I can give to real-world cases, so I hope it didn't confuse you :) + +Hence, in the two examples above, I used both the `Adaptor` class and the `Fetcher` class to show you that the logic for automatch is the same. + +## How the automatch feature works +Auto-matching works in two phases: + +1. **Save Phase**: Store unique properties of elements +2. **Match Phase**: Find elements with similar properties later + +Let's say you have an element you got through selection or any method and want the library to find it the next time you scrape this website, even if it had structural/design changes. + +As little technical details as possible, the general logic goes as the following: + + 1. You tell Scrapling to save that element's unique properties in one of the ways we will show below. + 2. Scrapling uses its configured database (SQLite by default) and saves each element's unique properties. + 3. Now, because everything about the element can be changed or removed from the website's owner(s), nothing from the element can be used as a unique identifier for the database. To solve this issue, I made the storage system rely on two things: + 1. The domain of the current website. If you are using the `Adaptor` class, you should pass it while initializing the class, or if you are using one of the fetchers, the domain will be taken from the URL automatically. + 2. An `identifier` to query that element's properties from the database. You don't always have to set the identifier yourself, as you will see later when we discuss this. + + Together, they will be used to retrieve the element's unique properties from the database later. + + 4. Later, when the website structural changes, you tell Scrapling to automatch the element. Scrapling retrieves the element's unique properties and matches all elements on the page against the unique properties we already have for this element. A score is calculated for their similarity to the element we want. In that comparison, everything is taken into consideration, as you will see later + 5. The element(s) with the highest similarity score to the wanted element are returned. + +### The unique properties +You might wonder, if all aspects of an element can be removed or changed, what unique properties we are talking about. + +For Scrapling, the unique elements we are relying on are: + +- Element tag name, text, attributes (names and values), siblings (tag names only), and path (tag names only). +- Element's parent tag name, attributes (names and values), and text. + +But you need to understand that the comparison between elements is not exact; it's more about finding how similar these values are. So everything is considered, even the values' order, like the order in which the element class names were written before and the order in which the same element class names are written now. + +## How to use automatch feature +The automatch feature can be used on any element you have, and it's added as arguments to CSS/XPath Selection methods, as you saw above, but we will get back to that later. + +First, you must enable the automatch feature by passing `auto_match=True` to the [Adaptor](main_classes.md#adaptor) class when you initialize it or enable it in the fetcher you are using of the available fetchers, as we will show. + +Examples: +```python +>>> from scrapling import Adaptor, Fetcher +>>> page = Adaptor(html_doc, auto_match=True) +# OR +>>> Fetcher.auto_match = True +>>> page = Fetcher.fetch('https://example.com') +``` +If you are using the [Adaptor](main_classes.md#adaptor) class, you need to pass the url of the website you are using with the argument `url` so Scrapling can separate the properties saved for each element by domain. + +If you didn't pass a URL, the word `default` will be used in place of the URL field while saving the element's unique properties. So, this will only be an issue if you used the same identifier later for a different website and didn't pass the URL parameter while initializing it. The save process will overwrite the previous data, and auto-matching only uses the latest saved properties. + +Besides those arguments, we have `storage` and `storage_args`. Both are for the class to be used to connect to the database; by default, it's set to the SQLite class that the library is using. Those arguments shouldn't matter unless you want to write your own storage system, which we will cover on a [separate page in the development section](../development/automatch_storage_system.md). + +Now, after enabling the automatch feature globally, you have two main ways to use it. + +### The CSS/XPath Selection way +As you have seen in the example above, first, you have to use the `auto_save` argument while selecting an element that exists on the page like below +```python +element = page.css('#p1' auto_save=True) +``` +and when the element doesn't exist, you can use the same selector and the `auto_match` argument, and the library will find it for you +```python +element = page.css('#p1', auto_match=True) +``` +Pretty simple, eh? + +Well, a lot happened under the hood here. Remember the identifier part we mentioned before that you need to set so you can retrieve the element you want? Here, with the `css`/`css_first`/`xpath`/`xpath_first` methods, the identifier is set automatically as the selector you passed here to make things easier :) + +Also, that's why here, for all these methods, you can pass the `identifier` argument to set it yourself, and there are cases for this, or you can use it to save the properties with the `auto_save` argument. + +### The manual way +You manually save and retrieve an element, then relocate it, which all happens within the automatch feature, as shown below. This allows you to automatch any element you have by any way or any selection method! + +First, let's say you got an element like this by text: +```python +>>> element = page.find_by_text('Tipping the Velvet', first_match=True) +``` +You can save its unique properties with the `save` method like below, but you must set the identifier yourself. For this example, I chose `my_special_element` as an identifier, but it's best to use a meaningful identifier in your code for the same reason you use meaningful variable names :) +```python +>>> page.save(element, 'my_special_element') +``` +Now, later, when you want to retrieve it and relocate it inside the page with auto-matching, it would be like this +```python +>>> element_dict = page.retrieve('my_special_element') +>>> page.relocate(element_dict, adaptor_type=True) +[<data='<a href="catalogue/tipping-the-velvet_99...' parent='<h3><a href="catalogue/tipping-the-velve...'>] +>>> page.relocate(element_dict, adaptor_type=True).css('::text') +['Tipping the Velvet'] +``` +Hence, the `retrieve` and relocate` methods are used. + +if you want to keep it as `lxml.etree` object, leave the `adaptor_type` argument +```python +>>> page.relocate(element_dict) +[<Element a at 0x105a2a7b0>] +``` + +## Troubleshooting + +### No Matches Found +```python +# 1. Check if data was saved +element_data = page.retrieve('identifier') +if not element_data: + print("No data saved for this identifier") + +# 2. Try with different identifier +products = page.css('.product', auto_match=True, identifier='old_selector') + +# 3. Save again with new identifier +products = page.css('.new-product', auto_save=True, identifier='new_identifier') +``` + +### Wrong Elements Matched +```python +# Use more specific selectors +products = page.css('.product-list .product', auto_save=True) + +# Or save with more context +product = page.find_by_text('Product Name').parent +page.save(product, 'specific_product') +``` + +## Known Issues +In the auto-matching save process, the unique properties of the first element from the selection results are the only ones that get saved. So if the selector you are using selects different elements on the page in different locations, auto-matching will return the first element to you only when you relocate it later. This doesn't include combined CSS selectors (Using commas to combine more than one selector, for example), as these selectors get separated, and each selector gets executed alone. + +## Final thoughts +Explaining this feature in detail without complications turned out to be challenging, but still, if there's something left unclear, you can head out to the [discussions section](https://github.com/D4Vinci/Scrapling/discussions), and I will reply to you ASAP or reach out to me privately and have a chat :) \ No newline at end of file diff --git a/docs/parsing/main_classes.md b/docs/parsing/main_classes.md new file mode 100644 index 0000000..3ffa21d --- /dev/null +++ b/docs/parsing/main_classes.md @@ -0,0 +1,539 @@ +## Introduction +After exploring the various ways to select elements with Scrapling and related features, Let's take a step back and examine the [Adaptor](#adaptor) class generally and other objects to better understand the parsing engine. + +The [Adaptor](#adaptor) class is the core parsing engine in Scrapling that provides HTML parsing and element selection capabilities. You can always import it with any of the following imports +```python +from scrapling import Adaptor +from scrapling.parser import Adaptor +``` +then use it directly as you already learned in the [overview](../overview.md) page +```python +adaptor = Adaptor( + text='<html>...</html>', + url='https://example.com' +) + +# Then select elements as you like +elements = adaptor.css('.product') +``` +In Scrapling, the main object you deal with after passing an HTML source or fetching a website is, of course, an [Adaptor](#adaptor) object. Any operation you do, like selection, navigation, etc., will return either an [Adaptor](#adaptor) object or an [Adaptors](#adaptors) object, given that the result is element/elements from the page, not text or similar. + +In other words, the main page is a [Adaptor](#adaptor) object, and the elements within are [Adaptor](#adaptor) objects, and so on. Any text, such as the text content inside elements or the text inside element attributes, is a [TextHandler](#texthandler) object, and the attributes of each element are stored as [AttributesHandler](#attributeshandler). We will return to both objects later, so let's focus on the [Adaptor](#adaptor) object. + +## Adaptor +### Arguments explained +The most important ones are `text` and `body`. Both are used to pass the HTML code you want to parse, but the first one accepts `str`, and the latter accepts `bytes` like how you used to do with `parsel` :) + +Otherwise, you have the arguments `url`, `auto_match`, `storage`, and `storage_args`. All these arguments are settings used with the `auto_match` feature, and they don't make a difference if you are not going to use that feature, so just ignore them for now, and we will explain them in the [automatch](automatch.md) feature page. + +Then you have the arguments for adjustments for parsing or adjusting/manipulating the HTML while the library parsing it: + +- **encoding**: This is the encoding that will be used while parsing the HTML. The default is `UTF-8`. +- **keep_comments**: This tells the library whether to keep HTML comments while parsing the page. It's disabled by default, as it can mess up your scraping in many ways. +- **keep_cdata**: Same logic as the HTML comments. [cdata](https://stackoverflow.com/questions/7092236/what-is-cdata-in-html) is removed by default for cleaner HTML. This also means when you check for the raw html content, you will find it doesn't have the cdata. + +I have intended to ignore the arguments `huge_tree` and `root` to avoid making this page more complicated than needed. +You may notice that I'm doing that a lot, and that's because it's something you don't need to know to use the library. The development section will cover these missing parts if you are that interested. + +After that, for the main page and elements within, most properties don't get initialized until you use it like the text content of a page/element, and this is one of the reasons for Scrapling speed :) + +### Properties +You have already seen much of this on the [overview](../overview.md) page, but don't worry if you didn't. We will review it more thoroughly using more advanced methods/usages. For clarity, the properties for traversal are separated below in the [traversal](#traversal) section. + +Let's say we are parsing this HTML page for simplicity: +```html +<html> + <head> + <title>Some page + + +
+
+

Product 1

+

This is product 1

+ $10.99 + +
+ +
+

Product 2

+

This is product 2

+ $20.99 + +
+ +
+

Product 3

+

This is product 3

+ $15.99 + +
+
+ + + + +``` +Load the page directly as shown before: +```python +from scrapling import Adaptor +page = Adaptor(html_doc) +``` +Get all text content on the page recursively +```python +>>> page.get_all_text() +'Some page\n\n \n\n \nProduct 1\nThis is product 1\n$10.99\nIn stock: 5\nProduct 2\nThis is product 2\n$20.99\nIn stock: 3\nProduct 3\nThis is product 3\n$15.99\nOut of stock' +``` +Get the first article as explained before; we will use it as an example +```python +article = page.find('article') +``` +With the same logic, get all text content on the element recursively +```python +>>> article.get_all_text() +'Product 1\nThis is product 1\n$10.99\nIn stock: 5' +``` +But if you try to get the direct text content, it will be empty; notice the logic difference +```python +>>> article.text +'' +``` +The `get_all_text` method has the following optional arguments: + +1. **separator**: All strings collected will be concatenated using this separator. The default is '\n' +2. **strip**: If enabled, strings will be stripped before concatenation. Disabled by default. +3. **ignore_tags**: A tuple of all tag names you want to ignore in the final results. The default is `('script', 'style',)`. +4. **valid_values**: If enabled, the method will only collect elements with real values, so all elements with empty text content or only whitespaces will be ignored. It's enabled by default + +By the way, the text returned here is not a standard string but a [TextHandler](#texthandler); we will get to this in detail later, so if the text content can be serialized to JSON, then use `.json()` on it +```python +>>> script = page.find('script') +>>> script.json() +{'lastUpdated': '2024-09-22T10:30:00Z', 'totalProducts': 3} +``` +Let's continue to get the element tag +```python +>>> article.tag +'article' +``` +If you used it on the page directly, you will find you are operating on the root `html` element +```python +>>> page.tag +'html' +``` +Now, I think I hammered the (`page`/`element`) idea, so I won't return to it again. + +Getting the attributes of the element +```python +>>> print(article.attrib) +{'class': 'product', 'data-id': '1'} +``` +Get the HTML content of the element +```python +>>> article.html_content +'

Product 1

\n

This is product 1

\n $10.99\n \n
' +``` +It's the same if you used the `.body` property +```python +>>> article.body +'

Product 1

\n

This is product 1

\n $10.99\n \n
' +``` +Get the prettified version of the HTML content of the element +```python +>>> print(article.prettify()) +

Product 1

+

This is product 1

+ $10.99 + +
+``` +To get all the ancestors in the DOM tree of this element +```python +>>> article.path +[
, +
, + Some page] +``` +Generate a CSS shortened selector if possible, or generate the full selector +```python +>>> article.generate_css_selector +'body > div > article' +>>> article.generate_full_css_selector +'body > div > article' +``` +Same case with XPath +```python +>>> article.generate_xpath_selector +"//body/div/article" +>>> article.generate_full_xpath_selector +"//body/div/article" +``` + +### Traversal +Using the elements we found above, we will go over the properties/methods for moving in the page in detail. + +If you are unfamiliar with the DOM tree or the tree data structure in general, the following traversal part can be confusing. I recommend you look up these concepts online for a better understanding. + +If you are too lazy to search about it, here's a quick explanation to give you a good idea.
+Simply put, the `html` element is the root of the website's tree, as every page starts with an `html` element.
+This element will be directly above elements like `head` and `body`. These are considered "children" of the `html` element, and the `html` element is considered their "parent." The element `body` is a "sibling" of the element `head` and vice versa. + +Accessing the parent of an element +```python +>>> article.parent +
+>>> article.parent.tag +'div' +``` +You can chain it as you want, which applies to all similar properties/methods we will review. +```python +>>> article.parent.parent.tag +'body' +``` +Get the children of an element +```python +>>> article.children +[Product 1' parent='
, + This is product 1...' parent='
, + $10.99' parent='
, +