diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index cd477fe..b8eac74 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -5,10 +5,8 @@ ## Proposed change @@ -34,12 +32,12 @@ ### Additional information -- This PR fixes or closes issue: fixes # -- This PR is related to issue: +- This PR fixes or closes an issue: fixes # +- This PR is related to an issue: # - Link to documentation pull request: ** ### Checklist: diff --git a/.github/workflows/code-quality.yml b/.github/workflows/code-quality.yml new file mode 100644 index 0000000..8d15d0c --- /dev/null +++ b/.github/workflows/code-quality.yml @@ -0,0 +1,150 @@ +name: Code Quality + +on: + push: + branches: + - main + - dev + paths-ignore: + - '*.md' + - '**/*.md' + - 'docs/**' + - 'images/**' + - '.github/**' + - '!.github/workflows/code-quality.yml' # Always run when this workflow changes + pull_request: + branches: + - main + - dev + paths-ignore: + - '*.md' + - '**/*.md' + - 'docs/**' + - 'images/**' + workflow_dispatch: # Allow manual triggering + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + code-quality: + name: Code Quality Checks + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: write # For PR annotations + + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 # Full history for better analysis + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.10' + cache: 'pip' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install bandit[toml] ruff vermin + + - name: Run Bandit (Security Linter) + id: bandit + continue-on-error: true + run: | + echo "::group::Bandit - Security Linter" + bandit -r -c .bandit.yml scrapling/ -f json -o bandit-report.json + bandit -r -c .bandit.yml scrapling/ + echo "::endgroup::" + + - name: Run Ruff Linter + id: ruff-lint + continue-on-error: true + run: | + echo "::group::Ruff - Linter" + ruff check scrapling/ --output-format=github + echo "::endgroup::" + + - name: Run Ruff Formatter Check + id: ruff-format + continue-on-error: true + run: | + echo "::group::Ruff - Formatter Check" + ruff format --check scrapling/ --diff + echo "::endgroup::" + + - name: Run Vermin (Python Version Compatibility) + id: vermin + continue-on-error: true + run: | + echo "::group::Vermin - Python 3.10+ Compatibility Check" + vermin -t=3.10- --violations --eval-annotations --no-tips scrapling/ + echo "::endgroup::" + + - name: Check results and create summary + if: always() + run: | + echo "# Code Quality Check Results" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + + # Initialize status + all_passed=true + + # Check Bandit + if [ "${{ steps.bandit.outcome }}" == "success" ]; then + echo "✅ **Bandit (Security)**: Passed" >> $GITHUB_STEP_SUMMARY + else + echo "❌ **Bandit (Security)**: Failed" >> $GITHUB_STEP_SUMMARY + all_passed=false + fi + + # Check Ruff Linter + if [ "${{ steps.ruff-lint.outcome }}" == "success" ]; then + echo "✅ **Ruff Linter**: Passed" >> $GITHUB_STEP_SUMMARY + else + echo "❌ **Ruff Linter**: Failed" >> $GITHUB_STEP_SUMMARY + all_passed=false + fi + + # Check Ruff Formatter + if [ "${{ steps.ruff-format.outcome }}" == "success" ]; then + echo "✅ **Ruff Formatter**: Passed" >> $GITHUB_STEP_SUMMARY + else + echo "❌ **Ruff Formatter**: Failed" >> $GITHUB_STEP_SUMMARY + all_passed=false + fi + + # Check Vermin + if [ "${{ steps.vermin.outcome }}" == "success" ]; then + echo "✅ **Vermin (Python 3.10+)**: Passed" >> $GITHUB_STEP_SUMMARY + else + echo "❌ **Vermin (Python 3.10+)**: Failed" >> $GITHUB_STEP_SUMMARY + all_passed=false + fi + + echo "" >> $GITHUB_STEP_SUMMARY + + if [ "$all_passed" == "true" ]; then + echo "### 🎉 All checks passed!" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "Your code meets all quality standards." >> $GITHUB_STEP_SUMMARY + else + echo "### ⚠️ Some checks failed" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "Please review the errors above and fix them." >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "**Tip**: Run \`pre-commit run --all-files\` locally to catch these issues before pushing." >> $GITHUB_STEP_SUMMARY + exit 1 + fi + + - name: Upload Bandit report + if: always() && steps.bandit.outcome != 'skipped' + uses: actions/upload-artifact@v4 + with: + name: bandit-security-report + path: bandit-report.json + retention-days: 30 diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 4a62cb1..f1a7343 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -60,6 +60,13 @@ jobs: python3 -m pip install --upgrade pip python3 -m pip install playwright>=1.55.0 patchright>=1.55.0 camoufox + - name: Get Playwright version + id: playwright-version + run: | + PLAYWRIGHT_VERSION=$(python3 -c "import importlib.metadata; print(importlib.metadata.version('playwright'))") + echo "version=$PLAYWRIGHT_VERSION" >> $GITHUB_OUTPUT + echo "Playwright version: $PLAYWRIGHT_VERSION" + - name: Retrieve Playwright browsers from cache if any id: playwright-cache uses: actions/cache@v4 @@ -68,16 +75,24 @@ jobs: ~/.cache/ms-playwright ~/Library/Caches/ms-playwright ~/.ms-playwright - key: ${{ runner.os }}-playwright-${{ hashFiles('pyproject.toml') }} + key: ${{ runner.os }}-playwright-${{ steps.playwright-version.outputs.version }}-v1 restore-keys: | + ${{ runner.os }}-playwright-${{ steps.playwright-version.outputs.version }}- ${{ runner.os }}-playwright- - name: Install Playwright browsers - if: steps.playwright-cache.outputs.cache-hit != 'true' run: | + echo "Cache hit: ${{ steps.playwright-cache.outputs.cache-hit }}" python3 -m playwright install chromium python3 -m playwright install-deps chromium firefox + - name: Get Camoufox version + id: camoufox-version + run: | + CAMOUFOX_VERSION=$(python3 -c "import importlib.metadata; print(importlib.metadata.version('camoufox'))") + echo "version=$CAMOUFOX_VERSION" >> $GITHUB_OUTPUT + echo "Camoufox version: $CAMOUFOX_VERSION" + - name: Retrieve Camoufox browser from cache if any id: camoufox-cache uses: actions/cache@v4 @@ -85,13 +100,14 @@ jobs: path: | ~/.cache/camoufox ~/Library/Caches/camoufox - key: ${{ runner.os }}-camoufox-${{ hashFiles('pyproject.toml') }} + key: ${{ runner.os }}-camoufox-${{ steps.camoufox-version.outputs.version }}-v1 restore-keys: | + ${{ runner.os }}-camoufox-${{ steps.camoufox-version.outputs.version }}- ${{ runner.os }}-camoufox- - name: Install Camoufox browser - if: steps.camoufox-cache.outputs.cache-hit != 'true' run: | + echo "Cache hit: ${{ steps.camoufox-cache.outputs.cache-hit }}" python3 -m camoufox fetch --browserforge # Cache tox environments diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1a20012..205e355 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,12 +1,12 @@ repos: - repo: https://github.com/PyCQA/bandit - rev: 1.8.6 + rev: 1.9.0 hooks: - id: bandit args: [-r, -c, .bandit.yml] - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. - rev: v0.13.3 + rev: v0.14.5 hooks: # Run the linter. - id: ruff @@ -14,7 +14,7 @@ repos: # Run the formatter. - id: ruff-format - repo: https://github.com/netromdk/vermin - rev: v1.6.0 + rev: v1.7.0 hooks: - id: vermin args: ['-t=3.10-', '--violations', '--eval-annotations', '--no-tips'] diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0b1fe2b..740033b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,39 +1,106 @@ # Contributing to Scrapling -Everybody is invited and welcome to contribute to Scrapling. Smaller changes have a better chance to get included in a timely manner. Adding unit tests for new features or test cases for bugs you've fixed help us to ensure that the Pull Request (PR) is fine. -There is a lot to do... -- If you are not a developer perhaps you would like to help with the [documentation](https://github.com/D4Vinci/Scrapling/tree/docs)? -- If you are a developer, most of the features I'm planning to add in the future are moved to [roadmap file](https://github.com/D4Vinci/Scrapling/blob/main/ROADMAP.md) so consider reading it. +Thank you for your interest in contributing to Scrapling! -Scrapling includes a comprehensive test suite which can be executed with pytest: -```bash -$ pytest -=============================== test session starts =============================== -platform darwin -- Python 3.12.7, pytest-8.3.3, pluggy-1.5.0 -rootdir: //Scrapling -configfile: pytest.ini -plugins: cov-5.0.0, anyio-4.6.0 -collected 16 items +Everybody is invited and welcome to contribute to Scrapling. -tests/test_parser_functions.py ................ [100%] +Minor changes have a better chance of being included promptly. Adding unit tests for new features or test cases for bugs you've fixed helps us ensure that the Pull Request (PR) is acceptable. -=============================== 16 passed in 0.22s ================================ -``` -Also, consider setting the scrapling logging level to `debug` so it's easier to know what's happening in the background. +There are many ways to contribute to Scrapling. Here are some of them: + +- Report bugs and request features using the [GitHub issues](https://github.com/D4Vinci/Scrapling/issues). Please follow the issue template to help us resolve your issue quickly. +- Blog about Scrapling. Tell the world how you’re using Scrapling. This will help newcomers with more examples and increase the Scrapling project's visibility. +- Join the [Discord community](https://discord.gg/EMgGbDceNQ) and share your ideas on how to improve Scrapling. We’re always open to suggestions. +- If you are not a developer, perhaps you would like to help with translating the [documentation](https://github.com/D4Vinci/Scrapling/tree/docs)? + + +## Finding work + +If you have decided to make a contribution to Scrapling, but you do not know what to contribute, here are some ways to find pending work: + +- Check out the [contribution](https://github.com/D4Vinci/Scrapling/contribute) GitHub page, which lists open issues tagged as good first issue. These issues provide a good starting point. +- There are also the [help wanted](https://github.com/D4Vinci/Scrapling/issues?q=is%3Aissue%20label%3A%22help%20wanted%22%20state%3Aopen) issues, but know that some may require familiarity with the Scrapling code base first. You can also target any other issue, provided it is not tagged as `invalid`, `wontfix`, or similar tags. +- If you enjoy writing automated tests, you can work on increasing our test coverage. Currently, the test coverage is around 90–92%. +- Join the [Discord community](https://discord.gg/EMgGbDceNQ) and ask questions in the `#help` channel. + +## Coding style +Please follow these coding conventions as we do when writing code for Scrapling: +- We use [pre-commit](https://pre-commit.com/) to automatically address simple code issues before every commit, so please install it and run `pre-commit install` to set it up. This will install hooks to run [ruff](https://docs.astral.sh/ruff/), [bandit](https://github.com/PyCQA/bandit), and [vermin](https://github.com/netromdk/vermin) on every commit. We are currently using a workflow to automatically run these tools on every PR, so if your code doesn't pass these checks, the PR will be rejected. +- We use type hints for better code clarity and [pyright](https://github.com/microsoft/pyright) for static type checking, which depends on the type hints, of course. +- We use the conventional commit messages format as [here](https://gist.github.com/qoomon/5dfcdf8eec66a051ecd85625518cfd13#types), so for example, we use the following prefixes for commit messages: + + | Prefix | When to use it | + |-------------|--------------------------| + | `feat:` | New feature added | + | `fix:` | Bug fix | + | `docs:` | Documentation change/add | + | `test:` | Tests | + | `refactor:` | Code refactoring | + | `chore:` | Maintenance tasks | + + Then include the details of the change in the body/description of the commit message. + + Example: + ``` + feat: add `adaptive` for similar elements + + - Added find_similar() method + - Implemented pattern matching + - Added tests and documentation + ``` + +> Please don’t put your name in the code you contribute; git provides enough metadata to identify the author of the code. + +## Development +Setting the scrapling logging level to `debug` makes it easier to know what's happening in the background. ```python ->>> import logging ->>> logging.getLogger("scrapling").setLevel(logging.DEBUG) +import logging +logging.getLogger("scrapling").setLevel(logging.DEBUG) ``` - -### The process is straight-forward. - - - Read [How to get faster PR reviews](https://github.com/kubernetes/community/blob/master/contributors/guide/pull-requests.md#best-practices-for-faster-reviews) by Kubernetes (but skip step 0 and 1) - - Fork Scrapling [git repository](https://github.com/D4Vinci/Scrapling). - - Make your changes. - - Ensure tests work. - - Create a Pull Request against the [**dev**](https://github.com/D4Vinci/Scrapling/tree/dev) branch of Scrapling. - -### Installing the latest changes from the dev branch +Bonus: You can install the beta of the upcoming update from the dev branch as follows ```commandline pip3 install git+https://github.com/D4Vinci/Scrapling.git@dev ``` + +## Building Documentation +Documentation is built using [MkDocs](https://www.mkdocs.org/). You can build it locally using the following commands: +```bash +pip install mkdocs-material +mkdocs serve # Local preview +mkdocs build # Build the static site +``` + +## Tests +Scrapling includes a comprehensive test suite that can be executed with pytest. However, first, you need to install all libraries and `pytest-plugins` listed in `tests/requirements.txt`. Then, running the tests will result in an output like this: + ```bash + $ pytest tests -n auto + =============================== test session starts =============================== + platform darwin -- Python 3.13.8, pytest-8.4.2, pluggy-1.6.0 -- /Users//.venv/bin/python3.13 + cachedir: .pytest_cache + rootdir: /Users//scrapling + configfile: pytest.ini + plugins: asyncio-1.2.0, anyio-4.11.0, xdist-3.8.0, httpbin-2.1.0, cov-7.0.0 + asyncio: mode=Mode.AUTO, debug=False, asyncio_default_fixture_loop_scope=function, asyncio_default_test_loop_scope=function + 10 workers [271 items] + scheduling tests via LoadScheduling + + ...... + + =============================== 271 passed in 52.68s ============================== + ``` +Hence, we used `-n auto` in the command above to run tests in threads to increase speed. + +Bonus: You can also see the test coverage with the `pytest` plugin below +```bash +pytest --cov=scrapling tests/ +``` + +## Making a Pull Request +To ensure that your PR gets accepted, please make sure that your PR is based on the latest changes from the dev branch and that it satisfies the following requirements: + +- The PR should be made against the [**dev**](https://github.com/D4Vinci/Scrapling/tree/dev) branch of Scrapling. Any PR made against the main branch will be rejected. +- The code should be passing all available tests. We are using tox with GitHub's CI to run the current tests on all supported Python versions with every commit. +- The code should be passing all code quality checks we mentioned above. We are using GitHub's CI to enforce the code style checks performed by pre-commit. If you were using the pre-commit hooks we discussed above, you should not see any issues when committing your changes. +- Make your changes, keep the code clean with an explanation of any part that might be vague, and remember to create a separate virtual environment for this project. +- If you are adding a new feature, please add tests for it. +- If you are fixing a bug, please add code with the PR that reproduces the bug. \ No newline at end of file diff --git a/README.md b/docs/README.md similarity index 94% rename from README.md rename to docs/README.md index 5915c8a..81cd47a 100644 --- a/README.md +++ b/docs/README.md @@ -1,6 +1,9 @@ +Automated translations: [العربيه](README_AR.md) | [Español](README_ES.md) | [Deutsch](README_DE.md) | [简体中文](README_CN.md) | [日本語](README_JP.md) | [Русский](README_RU.md) + +


- + main poster
Easy, effortless Web Scraping as it should be!

@@ -49,7 +52,7 @@ Scrapling isn't just another Web Scraping library. It's the first **adaptive** scraping library that learns from website changes and evolves with them. While other libraries break when websites update their structure, Scrapling automatically relocates your elements and keeps your scrapers running. -Built for the modern Web, Scrapling features its own rapid parsing engine and fetchers to handle all Web Scraping challenges you face or will face. Built by Web Scrapers for Web Scrapers and regular users, there's something for everyone. +Built for the modern Web, Scrapling features **its own rapid parsing engine** and fetchers to handle all Web Scraping challenges you face or will face. Built by Web Scrapers for Web Scrapers and regular users, there's something for everyone. ```python >> from scrapling.fetchers import Fetcher, AsyncFetcher, StealthyFetcher, DynamicFetcher @@ -147,6 +150,9 @@ page = DynamicFetcher.fetch('https://quotes.toscrape.com/') data = page.css('.quote .text::text') ``` +> [!NOTE] +> There's a wonderful guide to get you started quickly with Scraping [here](https://substack.thewebscraping.club/p/scrapling-hands-on-guide) written by The Web Scraping Club. In case you find it easier to get you started than the [documentation website](https://scrapling.readthedocs.io/en/latest/). + ### Advanced Parsing & Navigation ```python from scrapling.fetchers import Fetcher @@ -271,7 +277,7 @@ Starting with v0.3.2, this installation only includes the parser engine and its ### Optional Dependencies -1. If you are going to use any of the extra features below, the fetchers, or their classes, then you need to install fetchers' dependencies, and then install their browser dependencies with +1. If you are going to use any of the extra features below, the fetchers, or their classes, then you need to install fetchers' dependencies and then install their browser dependencies with ```bash pip install "scrapling[fetchers]" @@ -293,7 +299,7 @@ Starting with v0.3.2, this installation only includes the parser engine and its ```bash pip install "scrapling[all]" ``` - Don't forget that you need to install the browser dependencies with `scrapling install` after any of these extras (if you didn't already) + Remember that you need to install the browser dependencies with `scrapling install` after any of these extras (if you didn't already) ### Docker You can also install a Docker image with all extras and browsers with the following command from DockerHub: diff --git a/docs/README_AR.md b/docs/README_AR.md new file mode 100644 index 0000000..e3e585f --- /dev/null +++ b/docs/README_AR.md @@ -0,0 +1,331 @@ +

+
+ main poster +
+ استخراج بيانات الويب بسهولة ويسر كما يجب أن يكون! +

+

+ + Tests + + PyPI version + + PyPI Downloads +
+ + Discord + + + X (formerly Twitter) Follow + +
+ + Supported Python versions +

+ +

+ + طرق الاختيار + + · + + اختيار الجالب + + · + + واجهة سطر الأوامر + + · + + وضع MCP + + · + + الانتقال من Beautifulsoup + +

+ +**توقف عن محاربة أنظمة مكافحة الروبوتات. توقف عن إعادة كتابة المحددات بعد كل تحديث للموقع.** + +Scrapling ليست مجرد مكتبة أخرى لاستخراج بيانات الويب. إنها أول مكتبة استخراج **تكيفية** تتعلم من تغييرات المواقع وتتطور معها. بينما تتعطل المكتبات الأخرى عندما تحدث المواقع بنيتها، يعيد Scrapling تحديد موقع عناصرك تلقائياً ويحافظ على عمل أدوات الاستخراج الخاصة بك. + +مبني للويب الحديث، يتميز Scrapling **بمحرك تحليل سريع خاص به** وجوالب للتعامل مع جميع تحديات استخراج بيانات الويب التي تواجهها أو ستواجهها. مبني بواسطة مستخرجي الويب لمستخرجي الويب والمستخدمين العاديين، هناك شيء للجميع. + +```python +>> from scrapling.fetchers import Fetcher, AsyncFetcher, StealthyFetcher, DynamicFetcher +>> StealthyFetcher.adaptive = True +# احصل على كود المصدر للمواقع بشكل خفي! +>> page = StealthyFetcher.fetch('https://example.com', headless=True, network_idle=True) +>> print(page.status) +200 +>> products = page.css('.product', auto_save=True) # استخرج البيانات التي تنجو من تغييرات تصميم الموقع! +>> # لاحقاً، إذا تغيرت بنية الموقع، مرر `adaptive=True` +>> products = page.css('.product', adaptive=True) # و Scrapling لا يزال يجدها! +``` + +# الرعاة + + + + + + + + + + + + + +هل تريد عرض إعلانك هنا؟ انقر [هنا](https://github.com/sponsors/D4Vinci) واختر المستوى الذي يناسبك! + +--- + +## الميزات الرئيسية + +### جلب متقدم للمواقع مع دعم الجلسات +- **طلبات HTTP**: طلبات HTTP سريعة وخفية مع فئة `Fetcher`. يمكنها تقليد بصمة TLS للمتصفح والرؤوس واستخدام HTTP3. +- **التحميل الديناميكي**: جلب المواقع الديناميكية مع أتمتة كاملة للمتصفح من خلال فئة `DynamicFetcher` التي تدعم Chromium من Playwright، وChrome الحقيقي، ووضع التخفي المخصص. +- **تجاوز مكافحة الروبوتات**: قدرات تخفي متقدمة مع `StealthyFetcher` باستخدام نسخة معدلة من Firefox وانتحال البصمات. يمكنه تجاوز جميع أنواع Turnstile وInterstitial من Cloudflare بسهولة بالأتمتة. +- **إدارة الجلسات**: دعم الجلسات المستمرة مع فئات `FetcherSession` و`StealthySession` و`DynamicSession` لإدارة ملفات تعريف الارتباط والحالة عبر الطلبات. +- **دعم Async**: دعم async كامل عبر جميع الجوالب وفئات الجلسات async المخصصة. + +### الاستخراج التكيفي والتكامل مع الذكاء الاصطناعي +- 🔄 **تتبع العناصر الذكي**: إعادة تحديد موقع العناصر بعد تغييرات الموقع باستخدام خوارزميات التشابه الذكية. +- 🎯 **الاختيار المرن الذكي**: محددات CSS، محددات XPath، البحث القائم على الفلاتر، البحث النصي، البحث بالتعبيرات العادية والمزيد. +- 🔍 **البحث عن عناصر مشابهة**: تحديد العناصر المشابهة للعناصر الموجودة تلقائياً. +- 🤖 **خادم MCP للاستخدام مع الذكاء الاصطناعي**: خادم MCP مدمج لاستخراج بيانات الويب بمساعدة الذكاء الاصطناعي واستخراج البيانات. يتميز خادم MCP بقدرات مخصصة قوية تستخدم Scrapling لاستخراج المحتوى المستهدف قبل تمريره إلى الذكاء الاصطناعي (Claude/Cursor/إلخ)، وبالتالي تسريع العمليات وتقليل التكاليف عن طريق تقليل استخدام الرموز. ([فيديو توضيحي](https://www.youtube.com/watch?v=qyFk3ZNwOxE)) + +### بنية عالية الأداء ومختبرة في المعارك +- 🚀 **سريع كالبرق**: أداء محسّن يتفوق على معظم مكتبات استخراج Python. +- 🔋 **فعال في استخدام الذاكرة**: هياكل بيانات محسّنة وتحميل كسول لأقل استخدام للذاكرة. +- ⚡ **تسلسل JSON سريع**: أسرع 10 مرات من المكتبة القياسية. +- 🏗️ **مُختبر في المعارك**: لا يمتلك Scrapling فقط تغطية اختبار بنسبة 92٪ وتغطية كاملة لتلميحات الأنواع، ولكن تم استخدامه يومياً من قبل مئات مستخرجي الويب خلال العام الماضي. + +### تجربة صديقة للمطورين/مستخرجي الويب +- 🎯 **غلاف استخراج ويب تفاعلي**: غلاف IPython مدمج اختياري مع تكامل Scrapling، واختصارات، وأدوات جديدة لتسريع تطوير سكريبتات استخراج الويب، مثل تحويل طلبات curl إلى طلبات Scrapling وعرض نتائج الطلبات في متصفحك. +- 🚀 **استخدمه مباشرة من الطرفية**: اختيارياً، يمكنك استخدام Scrapling لاستخراج عنوان URL دون كتابة سطر واحد من الكود! +- 🛠️ **واجهة برمجة تطبيقات التنقل الغنية**: اجتياز DOM متقدم مع طرق التنقل بين الوالدين والأشقاء والأطفال. +- 🧬 **معالجة نصوص محسّنة**: تعبيرات عادية مدمجة وطرق تنظيف وعمليات سلسلة محسّنة. +- 📝 **إنشاء محدد تلقائي**: إنشاء محددات CSS/XPath قوية لأي عنصر. +- 🔌 **واجهة برمجة تطبيقات مألوفة**: مشابه لـ Scrapy/BeautifulSoup مع نفس العناصر الزائفة المستخدمة في Scrapy/Parsel. +- 📘 **تغطية كاملة للأنواع**: تلميحات نوع كاملة لدعم IDE ممتاز وإكمال الكود. +- 🔋 **صورة Docker جاهزة**: مع كل إصدار، يتم بناء ودفع صورة Docker تحتوي على جميع المتصفحات تلقائياً. + +## البدء + +### الاستخدام الأساسي +```python +from scrapling.fetchers import Fetcher, StealthyFetcher, DynamicFetcher +from scrapling.fetchers import FetcherSession, StealthySession, DynamicSession + +# طلبات HTTP مع دعم الجلسات +with FetcherSession(impersonate='chrome') as session: # استخدم أحدث إصدار من بصمة TLS لـ Chrome + page = session.get('https://quotes.toscrape.com/', stealthy_headers=True) + quotes = page.css('.quote .text::text') + +# أو استخدم طلبات لمرة واحدة +page = Fetcher.get('https://quotes.toscrape.com/') +quotes = page.css('.quote .text::text') + +# وضع التخفي المتقدم (احتفظ بالمتصفح مفتوحاً حتى تنتهي) +with StealthySession(headless=True, solve_cloudflare=True) as session: + page = session.fetch('https://nopecha.com/demo/cloudflare', google_search=False) + data = page.css('#padded_content a') + +# أو استخدم نمط الطلب لمرة واحدة، يفتح المتصفح لهذا الطلب، ثم يغلقه بعد الانتهاء +page = StealthyFetcher.fetch('https://nopecha.com/demo/cloudflare') +data = page.css('#padded_content a') + +# أتمتة المتصفح الكاملة (احتفظ بالمتصفح مفتوحاً حتى تنتهي) +with DynamicSession(headless=True) as session: + page = session.fetch('https://quotes.toscrape.com/', network_idle=True) + quotes = page.css('.quote .text::text') + +# أو استخدم نمط الطلب لمرة واحدة +page = DynamicFetcher.fetch('https://quotes.toscrape.com/', network_idle=True) +quotes = page.css('.quote .text::text') +``` + +### اختيار العناصر +```python +# محددات CSS +page.css('a::text') # استخراج النص +page.css('a::attr(href)') # استخراج السمات +page.css('a', recursive=False) # العناصر المباشرة فقط +page.css('a', auto_save=True) # حفظ مواضع العناصر تلقائياً + +# XPath +page.xpath('//a/text()') + +# بحث مرن +page.find_by_text('Python', first_match=True) # البحث بالنص +page.find_by_regex(r'\d{4}') # البحث بنمط التعبير العادي +page.find('div', {'class': 'container'}) # البحث بالسمات + +# التنقل +element.parent # الحصول على العنصر الوالد +element.next_sibling # الحصول على الشقيق التالي +element.children # الحصول على الأطفال + +# عناصر مشابهة +similar = page.get_similar(element) # البحث عن عناصر مشابهة + +# الاستخراج التكيفي +saved_elements = page.css('.product', auto_save=True) +# لاحقاً، عندما يتغير الموقع: +page.css('.product', adaptive=True) # البحث عن العناصر باستخدام المواضع المحفوظة +``` + +### استخدام الجلسة +```python +from scrapling.fetchers import FetcherSession, AsyncFetcherSession + +# جلسة متزامنة +with FetcherSession() as session: + # يتم الاحتفاظ بملفات تعريف الارتباط تلقائياً + page1 = session.get('https://quotes.toscrape.com/login') + page2 = session.post('https://quotes.toscrape.com/login', data={'username': 'admin', 'password': 'admin'}) + + # تبديل بصمة المتصفح إذا لزم الأمر + page2 = session.get('https://quotes.toscrape.com/', impersonate='firefox135') + +# استخدام جلسة async +async with AsyncStealthySession(max_pages=2) as session: + tasks = [] + urls = ['https://example.com/page1', 'https://example.com/page2'] + + for url in urls: + task = session.fetch(url) + tasks.append(task) + + print(session.get_pool_stats()) # اختياري - حالة مجموعة علامات تبويب المتصفح (مشغول/حر/خطأ) + results = await asyncio.gather(*tasks) + print(session.get_pool_stats()) +``` + +## واجهة سطر الأوامر والغلاف التفاعلي + +يتضمن Scrapling v0.3 واجهة سطر أوامر قوية: + +[![asciicast](https://asciinema.org/a/736339.svg)](https://asciinema.org/a/736339) + +```bash +# تشغيل غلاف استخراج الويب التفاعلي +scrapling shell + +# استخراج الصفحات إلى ملف مباشرة دون برمجة (يستخرج المحتوى داخل وسم `body` افتراضياً) +# إذا انتهى ملف الإخراج بـ `.txt`، فسيتم استخراج محتوى النص للهدف. +# إذا انتهى بـ `.md`، فسيكون تمثيل markdown لمحتوى HTML، و`.html` سيكون محتوى HTML مباشرة. +scrapling extract get 'https://example.com' content.md +scrapling extract get 'https://example.com' content.txt --css-selector '#fromSkipToProducts' --impersonate 'chrome' # جميع العناصر المطابقة لمحدد CSS '#fromSkipToProducts' +scrapling extract fetch 'https://example.com' content.md --css-selector '#fromSkipToProducts' --no-headless +scrapling extract stealthy-fetch 'https://nopecha.com/demo/cloudflare' captchas.html --css-selector '#padded_content a' --solve-cloudflare +``` + +> [!NOTE] +> هناك العديد من الميزات الإضافية، لكننا نريد إبقاء هذه الصفحة موجزة، مثل خادم MCP وغلاف استخراج الويب التفاعلي. تحقق من الوثائق الكاملة [هنا](https://scrapling.readthedocs.io/en/latest/) + +## معايير الأداء + +Scrapling ليس قوياً فقط - إنه أيضاً سريع بشكل مذهل، والتحديثات منذ الإصدار 0.3 قدمت تحسينات أداء استثنائية عبر جميع العمليات. + +### اختبار سرعة استخراج النص (5000 عنصر متداخل) + +| # | المكتبة | الوقت (ms) | vs Scrapling | +|---|:-----------------:|:---------:|:------------:| +| 1 | Scrapling | 1.92 | 1.0x | +| 2 | Parsel/Scrapy | 1.99 | 1.036x | +| 3 | Raw Lxml | 2.33 | 1.214x | +| 4 | PyQuery | 20.61 | ~11x | +| 5 | Selectolax | 80.65 | ~42x | +| 6 | BS4 with Lxml | 1283.21 | ~698x | +| 7 | MechanicalSoup | 1304.57 | ~679x | +| 8 | BS4 with html5lib | 3331.96 | ~1735x | + +### أداء تشابه العناصر والبحث النصي + +قدرات العثور على العناصر التكيفية لـ Scrapling تتفوق بشكل كبير على البدائل: + +| المكتبة | الوقت (ms) | vs Scrapling | +|-------------|:---------:|:------------:| +| Scrapling | 1.87 | 1.0x | +| AutoScraper | 10.24 | 5.476x | + + +> تمثل جميع المعايير متوسطات أكثر من 100 تشغيل. انظر [benchmarks.py](https://github.com/D4Vinci/Scrapling/blob/main/benchmarks.py) للمنهجية. + +## التثبيت + +يتطلب Scrapling Python 3.10 أو أعلى: + +```bash +pip install scrapling +``` + +بدءاً من v0.3.2، يتضمن هذا التثبيت فقط محرك المحلل وتبعياته، بدون أي جوالب أو تبعيات سطر أوامر. + +### التبعيات الاختيارية + +1. إذا كنت ستستخدم أياً من الميزات الإضافية أدناه، أو الجوالب، أو فئاتها، فأنت بحاجة إلى تثبيت تبعيات الجوالب ثم تثبيت تبعيات المتصفح الخاصة بها بـ + ```bash + pip install "scrapling[fetchers]" + + scrapling install + ``` + + يقوم هذا بتنزيل جميع المتصفحات مع تبعيات النظام وتبعيات معالجة البصمات الخاصة بها. + +2. ميزات إضافية: + - تثبيت ميزة خادم MCP: + ```bash + pip install "scrapling[ai]" + ``` + - تثبيت ميزات الغلاف (غلاف استخراج الويب وأمر `extract`): + ```bash + pip install "scrapling[shell]" + ``` + - تثبيت كل شيء: + ```bash + pip install "scrapling[all]" + ``` + تذكر أنك تحتاج إلى تثبيت تبعيات المتصفح مع `scrapling install` بعد أي من هذه الإضافات (إذا لم تكن قد فعلت ذلك بالفعل) + +### Docker +يمكنك أيضاً تثبيت صورة Docker مع جميع الإضافات والمتصفحات باستخدام الأمر التالي من DockerHub: +```bash +docker pull pyd4vinci/scrapling +``` +أو تنزيلها من سجل GitHub: +```bash +docker pull ghcr.io/d4vinci/scrapling:latest +``` +يتم بناء هذه الصورة ودفعها تلقائياً من خلال إجراءات GitHub على الفرع الرئيسي للمستودع. + +## المساهمة + +نرحب بالمساهمات! يرجى قراءة [إرشادات المساهمة](https://github.com/D4Vinci/Scrapling/blob/main/CONTRIBUTING.md) قبل البدء. + +## إخلاء المسؤولية + +> [!CAUTION] +> يتم توفير هذه المكتبة للأغراض التعليمية والبحثية فقط. باستخدام هذه المكتبة، فإنك توافق على الامتثال لقوانين استخراج البيانات والخصوصية المحلية والدولية. المؤلفون والمساهمون غير مسؤولين عن أي إساءة استخدام لهذا البرنامج. احترم دائماً شروط خدمة المواقع وملفات robots.txt. + +## الترخيص + +هذا العمل مرخص بموجب ترخيص BSD-3-Clause. + +## الشكر والتقدير + +يتضمن هذا المشروع كوداً معدلاً من: +- Parsel (ترخيص BSD) - يستخدم للوحدة الفرعية [translator](https://github.com/D4Vinci/Scrapling/blob/main/scrapling/core/translator.py) + +## الشكر والمراجع + +- العمل الرائع لـ [Daijro](https://github.com/daijro) على [BrowserForge](https://github.com/daijro/browserforge) و[Camoufox](https://github.com/daijro/camoufox) +- العمل الرائع لـ [Vinyzu](https://github.com/Vinyzu) على [Botright](https://github.com/Vinyzu/Botright) و[PatchRight](https://github.com/Kaliiiiiiiiii-Vinyzu/patchright) +- [brotector](https://github.com/kaliiiiiiiiii/brotector) لتقنيات تجاوز اكتشاف المتصفح +- [fakebrowser](https://github.com/kkoooqq/fakebrowser) و[BotBrowser](https://github.com/botswin/BotBrowser) لأبحاث البصمات + +--- +
مصمم ومصنوع بـ ❤️ بواسطة كريم شعير.

\ No newline at end of file diff --git a/docs/README_CN.md b/docs/README_CN.md new file mode 100644 index 0000000..87d52ce --- /dev/null +++ b/docs/README_CN.md @@ -0,0 +1,331 @@ +

+
+ main poster +
+ 简单、轻松的网页抓取，本该如此！ +

+

+ + Tests + + PyPI version + + PyPI Downloads +
+ + Discord + + + X (formerly Twitter) Follow + +
+ + Supported Python versions +

+ +

+ + 选择方法 + + · + + 选择获取器 + + · + + 命令行界面 + + · + + MCP模式 + + · + + 从Beautifulsoup迁移 + +

+ +**停止与反机器人系统斗争。停止在每次网站更新后重写选择器。** + +Scrapling不仅仅是另一个网页抓取库。它是第一个**自适应**抓取库，能够从网站变化中学习并与之共同进化。当其他库在网站更新结构时失效，Scrapling会自动重新定位您的元素并保持抓取器运行。 + +为现代网络而构建，Scrapling具有**自己的快速解析引擎**和获取器来处理您面临或将要面临的所有网页抓取挑战。由网页抓取者为网页抓取者和普通用户构建，适合每个人。 + +```python +>> from scrapling.fetchers import Fetcher, AsyncFetcher, StealthyFetcher, DynamicFetcher +>> StealthyFetcher.adaptive = True +# 隐秘地获取网站源代码！ +>> page = StealthyFetcher.fetch('https://example.com', headless=True, network_idle=True) +>> print(page.status) +200 +>> products = page.css('.product', auto_save=True) # 抓取在网站设计变更后仍能存活的数据！ +>> # 之后，如果网站结构改变，传递 `adaptive=True` +>> products = page.css('.product', adaptive=True) # Scrapling仍然能找到它们！ +``` + +# 赞助商 + + + + + + + + + + + + + +想在这里展示您的广告吗？点击[这里](https://github.com/sponsors/D4Vinci)并选择适合您的级别！ + +--- + +## 主要特性 + +### 支持会话的高级网站获取 +- **HTTP请求**：使用`Fetcher`类进行快速和隐秘的HTTP请求。可以模拟浏览器的TLS指纹、标头并使用HTTP3。 +- **动态加载**：通过`DynamicFetcher`类使用完整的浏览器自动化获取动态网站，支持Playwright的Chromium、真实Chrome和自定义隐秘模式。 +- **反机器人绕过**：使用`StealthyFetcher`的高级隐秘功能，使用修改版Firefox和指纹伪装。可以轻松自动绕过所有类型的Cloudflare的Turnstile和Interstitial。 +- **会话管理**：使用`FetcherSession`、`StealthySession`和`DynamicSession`类持久化会话支持，用于跨请求的cookie和状态管理。 +- **异步支持**：所有获取器和专用异步会话类的完整异步支持。 + +### 自适应抓取和AI集成 +- 🔄 **智能元素跟踪**：使用智能相似性算法在网站更改后重新定位元素。 +- 🎯 **智能灵活选择**：CSS选择器、XPath选择器、基于过滤器的搜索、文本搜索、正则表达式搜索等。 +- 🔍 **查找相似元素**：自动定位与找到的元素相似的元素。 +- 🤖 **与AI一起使用的MCP服务器**：内置MCP服务器用于AI辅助网页抓取和数据提取。MCP服务器具有自定义的强大功能，利用Scrapling在将内容传递给AI（Claude/Cursor等）之前提取目标内容，从而加快操作并通过最小化令牌使用来降低成本。（[演示视频](https://www.youtube.com/watch?v=qyFk3ZNwOxE)） + +### 高性能和经过实战测试的架构 +- 🚀 **闪电般快速**：优化性能超越大多数Python抓取库。 +- 🔋 **内存高效**：优化的数据结构和延迟加载，最小内存占用。 +- ⚡ **快速JSON序列化**：比标准库快10倍。 +- 🏗️ **经过实战测试**：Scrapling不仅拥有92%的测试覆盖率和完整的类型提示覆盖率，而且在过去一年中每天被数百名网页抓取者使用。 + +### 对开发者/网页抓取者友好的体验 +- 🎯 **交互式网页抓取Shell**：可选的内置IPython shell，具有Scrapling集成、快捷方式和新工具，可加快网页抓取脚本开发，例如将curl请求转换为Scrapling请求并在浏览器中查看请求结果。 +- 🚀 **直接从终端使用**：可选地，您可以使用Scrapling抓取URL而无需编写任何代码！ +- 🛠️ **丰富的导航API**：使用父级、兄弟级和子级导航方法进行高级DOM遍历。 +- 🧬 **增强的文本处理**：内置正则表达式、清理方法和优化的字符串操作。 +- 📝 **自动选择器生成**：为任何元素生成强大的CSS/XPath选择器。 +- 🔌 **熟悉的API**：类似于Scrapy/BeautifulSoup，使用与Scrapy/Parsel相同的伪元素。 +- 📘 **完整的类型覆盖**：完整的类型提示，出色的IDE支持和代码补全。 +- 🔋 **现成的Docker镜像**：每次发布时，包含所有浏览器的Docker镜像会自动构建和推送。 + +## 入门 + +### 基本用法 +```python +from scrapling.fetchers import Fetcher, StealthyFetcher, DynamicFetcher +from scrapling.fetchers import FetcherSession, StealthySession, DynamicSession + +# 支持会话的HTTP请求 +with FetcherSession(impersonate='chrome') as session: # 使用Chrome的最新版本TLS指纹 + page = session.get('https://quotes.toscrape.com/', stealthy_headers=True) + quotes = page.css('.quote .text::text') + +# 或使用一次性请求 +page = Fetcher.get('https://quotes.toscrape.com/') +quotes = page.css('.quote .text::text') + +# 高级隐秘模式（保持浏览器打开直到完成） +with StealthySession(headless=True, solve_cloudflare=True) as session: + page = session.fetch('https://nopecha.com/demo/cloudflare', google_search=False) + data = page.css('#padded_content a') + +# 或使用一次性请求样式，为此请求打开浏览器，完成后关闭 +page = StealthyFetcher.fetch('https://nopecha.com/demo/cloudflare') +data = page.css('#padded_content a') + +# 完整的浏览器自动化（保持浏览器打开直到完成） +with DynamicSession(headless=True) as session: + page = session.fetch('https://quotes.toscrape.com/', network_idle=True) + quotes = page.css('.quote .text::text') + +# 或使用一次性请求样式 +page = DynamicFetcher.fetch('https://quotes.toscrape.com/', network_idle=True) +quotes = page.css('.quote .text::text') +``` + +### 元素选择 +```python +# CSS选择器 +page.css('a::text') # 提取文本 +page.css('a::attr(href)') # 提取属性 +page.css('a', recursive=False) # 仅直接元素 +page.css('a', auto_save=True) # 自动保存元素位置 + +# XPath +page.xpath('//a/text()') + +# 灵活搜索 +page.find_by_text('Python', first_match=True) # 按文本查找 +page.find_by_regex(r'\d{4}') # 按正则表达式模式查找 +page.find('div', {'class': 'container'}) # 按属性查找 + +# 导航 +element.parent # 获取父元素 +element.next_sibling # 获取下一个兄弟元素 +element.children # 获取子元素 + +# 相似元素 +similar = page.get_similar(element) # 查找相似元素 + +# 自适应抓取 +saved_elements = page.css('.product', auto_save=True) +# 之后，当网站更改时： +page.css('.product', adaptive=True) # 使用保存的位置查找元素 +``` + +### 会话使用 +```python +from scrapling.fetchers import FetcherSession, AsyncFetcherSession + +# 同步会话 +with FetcherSession() as session: + # Cookie自动保持 + page1 = session.get('https://quotes.toscrape.com/login') + page2 = session.post('https://quotes.toscrape.com/login', data={'username': 'admin', 'password': 'admin'}) + + # 如需要，切换浏览器指纹 + page2 = session.get('https://quotes.toscrape.com/', impersonate='firefox135') + +# 异步会话使用 +async with AsyncStealthySession(max_pages=2) as session: + tasks = [] + urls = ['https://example.com/page1', 'https://example.com/page2'] + + for url in urls: + task = session.fetch(url) + tasks.append(task) + + print(session.get_pool_stats()) # 可选 - 浏览器标签池的状态（忙/空闲/错误） + results = await asyncio.gather(*tasks) + print(session.get_pool_stats()) +``` + +## CLI和交互式Shell + +Scrapling v0.3包含强大的命令行界面： + +[![asciicast](https://asciinema.org/a/736339.svg)](https://asciinema.org/a/736339) + +```bash +# 启动交互式网页抓取shell +scrapling shell + +# 直接将页面提取到文件而无需编程（默认提取`body`标签内的内容） +# 如果输出文件以`.txt`结尾，则将提取目标的文本内容。 +# 如果以`.md`结尾，它将是HTML内容的markdown表示，`.html`将直接是HTML内容。 +scrapling extract get 'https://example.com' content.md +scrapling extract get 'https://example.com' content.txt --css-selector '#fromSkipToProducts' --impersonate 'chrome' # 所有匹配CSS选择器'#fromSkipToProducts'的元素 +scrapling extract fetch 'https://example.com' content.md --css-selector '#fromSkipToProducts' --no-headless +scrapling extract stealthy-fetch 'https://nopecha.com/demo/cloudflare' captchas.html --css-selector '#padded_content a' --solve-cloudflare +``` + +> [!NOTE] +> 还有许多其他功能，但我们希望保持此页面简洁，例如MCP服务器和交互式网页抓取Shell。查看完整文档[这里](https://scrapling.readthedocs.io/en/latest/) + +## 性能基准 + +Scrapling不仅功能强大——它还速度极快，自0.3版本以来的更新在所有操作中都提供了卓越的性能改进。 + +### 文本提取速度测试（5000个嵌套元素） + +| # | 库 | 时间(ms) | vs Scrapling | +|---|:--------------:|:--------:|:------------:| +| 1 | Scrapling | 1.92 | 1.0x | +| 2 | Parsel/Scrapy | 1.99 | 1.036x | +| 3 | Raw Lxml | 2.33 | 1.214x | +| 4 | PyQuery | 20.61 | ~11x | +| 5 | Selectolax | 80.65 | ~42x | +| 6 | BS4 with Lxml | 1283.21 | ~698x | +| 7 | MechanicalSoup | 1304.57 | ~679x | +| 8 |BS4 with html5lib| 3331.96 | ~1735x | + +### 元素相似性和文本搜索性能 + +Scrapling的自适应元素查找功能明显优于替代方案： + +| 库 | 时间(ms) | vs Scrapling | +|-------------|:--------:|:------------:| +| Scrapling | 1.87 | 1.0x | +| AutoScraper | 10.24 | 5.476x | + + +> 所有基准测试代表100+次运行的平均值。请参阅[benchmarks.py](https://github.com/D4Vinci/Scrapling/blob/main/benchmarks.py)了解方法。 + +## 安装 + +Scrapling需要Python 3.10或更高版本： + +```bash +pip install scrapling +``` + +从v0.3.2开始，此安装仅包括解析器引擎及其依赖项，没有任何获取器或命令行依赖项。 + +### 可选依赖项 + +1. 如果您要使用以下任何额外功能、获取器或它们的类，那么您需要安装获取器的依赖项，然后使用以下命令安装它们的浏览器依赖项 + ```bash + pip install "scrapling[fetchers]" + + scrapling install + ``` + + 这会下载所有浏览器及其系统依赖项和指纹操作依赖项。 + +2. 额外功能： + - 安装MCP服务器功能： + ```bash + pip install "scrapling[ai]" + ``` + - 安装shell功能（网页抓取shell和`extract`命令）： + ```bash + pip install "scrapling[shell]" + ``` + - 安装所有内容： + ```bash + pip install "scrapling[all]" + ``` + 请记住，在安装任何这些额外功能后（如果您还没有安装），您需要使用`scrapling install`安装浏览器依赖项 + +### Docker +您还可以使用以下命令从DockerHub安装包含所有额外功能和浏览器的Docker镜像： +```bash +docker pull pyd4vinci/scrapling +``` +或从GitHub注册表下载： +```bash +docker pull ghcr.io/d4vinci/scrapling:latest +``` +此镜像通过仓库主分支上的GitHub actions自动构建和推送。 + +## 贡献 + +我们欢迎贡献！在开始之前，请阅读我们的[贡献指南](https://github.com/D4Vinci/Scrapling/blob/main/CONTRIBUTING.md)。 + +## 免责声明 + +> [!CAUTION] +> 此库仅用于教育和研究目的。使用此库即表示您同意遵守本地和国际数据抓取和隐私法律。作者和贡献者对本软件的任何滥用不承担责任。始终尊重网站的服务条款和robots.txt文件。 + +## 许可证 + +本作品根据BSD-3-Clause许可证授权。 + +## 致谢 + +此项目包含改编自以下内容的代码： +- Parsel（BSD许可证）——用于[translator](https://github.com/D4Vinci/Scrapling/blob/main/scrapling/core/translator.py)子模块 + +## 感谢和参考 + +- [Daijro](https://github.com/daijro)在[BrowserForge](https://github.com/daijro/browserforge)和[Camoufox](https://github.com/daijro/camoufox)上的出色工作 +- [Vinyzu](https://github.com/Vinyzu)在[Botright](https://github.com/Vinyzu/Botright)和[PatchRight](https://github.com/Kaliiiiiiiiii-Vinyzu/patchright)上的出色工作 +- [brotector](https://github.com/kaliiiiiiiiii/brotector)提供的浏览器检测绕过技术 +- [fakebrowser](https://github.com/kkoooqq/fakebrowser)和[BotBrowser](https://github.com/botswin/BotBrowser)提供的指纹识别研究 + +--- +
由Karim Shoair用❤️设计和制作。

\ No newline at end of file diff --git a/docs/README_DE.md b/docs/README_DE.md new file mode 100644 index 0000000..486b8b1 --- /dev/null +++ b/docs/README_DE.md @@ -0,0 +1,331 @@ +

+
+ main poster +
+ Einfaches, müheloses Web Scraping, wie es sein sollte! +

+

+ + Tests + + PyPI version + + PyPI Downloads +
+ + Discord + + + X (formerly Twitter) Follow + +
+ + Supported Python versions +

+ +

+ + Auswahlmethoden + + · + + Fetcher wählen + + · + + CLI + + · + + MCP-Modus + + · + + Migration von Beautifulsoup + +

+ +**Hören Sie auf, gegen Anti-Bot-Systeme zu kämpfen. Hören Sie auf, Selektoren nach jedem Website-Update neu zu schreiben.** + +Scrapling ist nicht nur eine weitere Web-Scraping-Bibliothek. Es ist die erste **adaptive** Scraping-Bibliothek, die von Website-Änderungen lernt und sich mit ihnen weiterentwickelt. Während andere Bibliotheken brechen, wenn Websites ihre Struktur aktualisieren, lokalisiert Scrapling Ihre Elemente automatisch neu und hält Ihre Scraper am Laufen. + +Für das moderne Web entwickelt, bietet Scrapling **seine eigene schnelle Parsing-Engine** und Fetcher, um alle Web-Scraping-Herausforderungen zu bewältigen, denen Sie begegnen oder begegnen werden. Von Web Scrapern für Web Scraper und normale Benutzer entwickelt, ist für jeden etwas dabei. + +```python +>> from scrapling.fetchers import Fetcher, AsyncFetcher, StealthyFetcher, DynamicFetcher +>> StealthyFetcher.adaptive = True +# Holen Sie sich Website-Quellcode unter dem Radar! +>> page = StealthyFetcher.fetch('https://example.com', headless=True, network_idle=True) +>> print(page.status) +200 +>> products = page.css('.product', auto_save=True) # Scrapen Sie Daten, die Website-Designänderungen überleben! +>> # Später, wenn sich die Website-Struktur ändert, übergeben Sie `adaptive=True` +>> products = page.css('.product', adaptive=True) # und Scrapling findet sie trotzdem! +``` + +# Sponsoren + + + + + + + + + + + + + +Möchten Sie Ihre Anzeige hier zeigen? Klicken Sie [hier](https://github.com/sponsors/D4Vinci) und wählen Sie die Stufe, die zu Ihnen passt! + +--- + +## Hauptmerkmale + +### Erweiterte Website-Abruf mit Sitzungsunterstützung +- **HTTP-Anfragen**: Schnelle und heimliche HTTP-Anfragen mit der `Fetcher`-Klasse. Kann Browser-TLS-Fingerabdrücke, Header imitieren und HTTP3 verwenden. +- **Dynamisches Laden**: Abrufen dynamischer Websites mit vollständiger Browser-Automatisierung über die `DynamicFetcher`-Klasse, die Playwrights Chromium, echtes Chrome und benutzerdefinierten Stealth-Modus unterstützt. +- **Anti-Bot-Umgehung**: Erweiterte Stealth-Fähigkeiten mit `StealthyFetcher` unter Verwendung einer modifizierten Firefox-Version und Fingerabdruck-Spoofing. Kann alle Arten von Cloudflares Turnstile und Interstitial einfach mit Automatisierung umgehen. +- **Sitzungsverwaltung**: Persistente Sitzungsunterstützung mit den Klassen `FetcherSession`, `StealthySession` und `DynamicSession` für Cookie- und Zustandsverwaltung über Anfragen hinweg. +- **Async-Unterstützung**: Vollständige Async-Unterstützung über alle Fetcher und dedizierte Async-Sitzungsklassen hinweg. + +### Adaptives Scraping & KI-Integration +- 🔄 **Intelligente Element-Verfolgung**: Elemente nach Website-Änderungen mit intelligenten Ähnlichkeitsalgorithmen neu lokalisieren. +- 🎯 **Intelligente flexible Auswahl**: CSS-Selektoren, XPath-Selektoren, filterbasierte Suche, Textsuche, Regex-Suche und mehr. +- 🔍 **Ähnliche Elemente finden**: Elemente, die gefundenen Elementen ähnlich sind, automatisch lokalisieren. +- 🤖 **MCP-Server für die Verwendung mit KI**: Integrierter MCP-Server für KI-unterstütztes Web Scraping und Datenextraktion. Der MCP-Server verfügt über benutzerdefinierte, leistungsstarke Funktionen, die Scrapling nutzen, um gezielten Inhalt zu extrahieren, bevor er an die KI (Claude/Cursor/etc.) übergeben wird, wodurch Vorgänge beschleunigt und Kosten durch Minimierung der Token-Nutzung gesenkt werden. ([Demo-Video](https://www.youtube.com/watch?v=qyFk3ZNwOxE)) + +### Hochleistungs- und praxiserprobte Architektur +- 🚀 **Blitzschnell**: Optimierte Leistung, die die meisten Python-Scraping-Bibliotheken übertrifft. +- 🔋 **Speichereffizient**: Optimierte Datenstrukturen und Lazy Loading für einen minimalen Speicher-Footprint. +- ⚡ **Schnelle JSON-Serialisierung**: 10x schneller als die Standardbibliothek. +- 🏗️ **Praxiserprobt**: Scrapling hat nicht nur eine Testabdeckung von 92% und eine vollständige Type-Hints-Abdeckung, sondern wird seit dem letzten Jahr täglich von Hunderten von Web Scrapern verwendet. + +### Entwickler/Web-Scraper-freundliche Erfahrung +- 🎯 **Interaktive Web-Scraping-Shell**: Optionale integrierte IPython-Shell mit Scrapling-Integration, Shortcuts und neuen Tools zur Beschleunigung der Web-Scraping-Skriptentwicklung, wie das Konvertieren von Curl-Anfragen in Scrapling-Anfragen und das Anzeigen von Anfrageergebnissen in Ihrem Browser. +- 🚀 **Direkt vom Terminal aus verwenden**: Optional können Sie Scrapling verwenden, um eine URL zu scrapen, ohne eine einzige Codezeile zu schreiben! +- 🛠️ **Umfangreiche Navigations-API**: Erweiterte DOM-Traversierung mit Eltern-, Geschwister- und Kind-Navigationsmethoden. +- 🧬 **Verbesserte Textverarbeitung**: Integrierte Regex, Bereinigungsmethoden und optimierte String-Operationen. +- 📝 **Automatische Selektorgenerierung**: Robuste CSS/XPath-Selektoren für jedes Element generieren. +- 🔌 **Vertraute API**: Ähnlich wie Scrapy/BeautifulSoup mit denselben Pseudo-Elementen, die in Scrapy/Parsel verwendet werden. +- 📘 **Vollständige Typabdeckung**: Vollständige Type Hints für hervorragende IDE-Unterstützung und Code-Vervollständigung. +- 🔋 **Fertiges Docker-Image**: Mit jeder Veröffentlichung wird automatisch ein Docker-Image erstellt und gepusht, das alle Browser enthält. + +## Erste Schritte + +### Grundlegende Verwendung +```python +from scrapling.fetchers import Fetcher, StealthyFetcher, DynamicFetcher +from scrapling.fetchers import FetcherSession, StealthySession, DynamicSession + +# HTTP-Anfragen mit Sitzungsunterstützung +with FetcherSession(impersonate='chrome') as session: # Verwenden Sie die neueste Version von Chromes TLS-Fingerabdruck + page = session.get('https://quotes.toscrape.com/', stealthy_headers=True) + quotes = page.css('.quote .text::text') + +# Oder verwenden Sie einmalige Anfragen +page = Fetcher.get('https://quotes.toscrape.com/') +quotes = page.css('.quote .text::text') + +# Erweiterter Stealth-Modus (Browser offen halten, bis Sie fertig sind) +with StealthySession(headless=True, solve_cloudflare=True) as session: + page = session.fetch('https://nopecha.com/demo/cloudflare', google_search=False) + data = page.css('#padded_content a') + +# Oder verwenden Sie den einmaligen Anfragenstil, öffnet den Browser für diese Anfrage und schließt ihn dann nach Abschluss +page = StealthyFetcher.fetch('https://nopecha.com/demo/cloudflare') +data = page.css('#padded_content a') + +# Vollständige Browser-Automatisierung (Browser offen halten, bis Sie fertig sind) +with DynamicSession(headless=True) as session: + page = session.fetch('https://quotes.toscrape.com/', network_idle=True) + quotes = page.css('.quote .text::text') + +# Oder verwenden Sie den einmaligen Anfragenstil +page = DynamicFetcher.fetch('https://quotes.toscrape.com/', network_idle=True) +quotes = page.css('.quote .text::text') +``` + +### Elementauswahl +```python +# CSS-Selektoren +page.css('a::text') # Text extrahieren +page.css('a::attr(href)') # Attribute extrahieren +page.css('a', recursive=False) # Nur direkte Elemente +page.css('a', auto_save=True) # Elementpositionen automatisch speichern + +# XPath +page.xpath('//a/text()') + +# Flexible Suche +page.find_by_text('Python', first_match=True) # Nach Text suchen +page.find_by_regex(r'\d{4}') # Nach Regex-Muster suchen +page.find('div', {'class': 'container'}) # Nach Attributen suchen + +# Navigation +element.parent # Elternelement abrufen +element.next_sibling # Nächstes Geschwister abrufen +element.children # Kindelemente abrufen + +# Ähnliche Elemente +similar = page.get_similar(element) # Ähnliche Elemente finden + +# Adaptives Scraping +saved_elements = page.css('.product', auto_save=True) +# Später, wenn sich die Website ändert: +page.css('.product', adaptive=True) # Elemente mithilfe gespeicherter Positionen finden +``` + +### Sitzungsverwendung +```python +from scrapling.fetchers import FetcherSession, AsyncFetcherSession + +# Synchrone Sitzung +with FetcherSession() as session: + # Cookies werden automatisch beibehalten + page1 = session.get('https://quotes.toscrape.com/login') + page2 = session.post('https://quotes.toscrape.com/login', data={'username': 'admin', 'password': 'admin'}) + + # Bei Bedarf Browser-Fingerabdruck wechseln + page2 = session.get('https://quotes.toscrape.com/', impersonate='firefox135') + +# Async-Sitzungsverwendung +async with AsyncStealthySession(max_pages=2) as session: + tasks = [] + urls = ['https://example.com/page1', 'https://example.com/page2'] + + for url in urls: + task = session.fetch(url) + tasks.append(task) + + print(session.get_pool_stats()) # Optional - Der Status des Browser-Tab-Pools (beschäftigt/frei/Fehler) + results = await asyncio.gather(*tasks) + print(session.get_pool_stats()) +``` + +## CLI & Interaktive Shell + +Scrapling v0.3 enthält eine leistungsstarke Befehlszeilenschnittstelle: + +[![asciicast](https://asciinema.org/a/736339.svg)](https://asciinema.org/a/736339) + +```bash +# Interaktive Web-Scraping-Shell starten +scrapling shell + +# Seiten direkt ohne Programmierung in eine Datei extrahieren (Extrahiert standardmäßig den Inhalt im `body`-Tag) +# Wenn die Ausgabedatei mit `.txt` endet, wird der Textinhalt des Ziels extrahiert. +# Wenn sie mit `.md` endet, ist es eine Markdown-Darstellung des HTML-Inhalts, und `.html` ist direkt der HTML-Inhalt. +scrapling extract get 'https://example.com' content.md +scrapling extract get 'https://example.com' content.txt --css-selector '#fromSkipToProducts' --impersonate 'chrome' # Alle Elemente, die dem CSS-Selektor '#fromSkipToProducts' entsprechen +scrapling extract fetch 'https://example.com' content.md --css-selector '#fromSkipToProducts' --no-headless +scrapling extract stealthy-fetch 'https://nopecha.com/demo/cloudflare' captchas.html --css-selector '#padded_content a' --solve-cloudflare +``` + +> [!NOTE] +> Es gibt viele zusätzliche Funktionen, aber wir möchten diese Seite prägnant halten, wie den MCP-Server und die interaktive Web-Scraping-Shell. Schauen Sie sich die vollständige Dokumentation [hier](https://scrapling.readthedocs.io/en/latest/) an + +## Leistungsbenchmarks + +Scrapling ist nicht nur leistungsstark – es ist auch blitzschnell, und die Updates seit Version 0.3 haben außergewöhnliche Leistungsverbesserungen bei allen Operationen gebracht. + +### Textextraktions-Geschwindigkeitstest (5000 verschachtelte Elemente) + +| # | Bibliothek | Zeit (ms) | vs Scrapling | +|---|:--------------------:|:---------:|:------------:| +| 1 | Scrapling | 1.92 | 1.0x | +| 2 | Parsel/Scrapy | 1.99 | 1.036x | +| 3 | Raw Lxml | 2.33 | 1.214x | +| 4 | PyQuery | 20.61 | ~11x | +| 5 | Selectolax | 80.65 | ~42x | +| 6 | BS4 mit Lxml | 1283.21 | ~698x | +| 7 | MechanicalSoup | 1304.57 | ~679x | +| 8 | BS4 mit html5lib | 3331.96 | ~1735x | + +### Element-Ähnlichkeit & Textsuche-Leistung + +Scraplings adaptive Element-Finding-Fähigkeiten übertreffen Alternativen deutlich: + +| Bibliothek | Zeit (ms) | vs Scrapling | +|-------------|:---------:|:------------:| +| Scrapling | 1.87 | 1.0x | +| AutoScraper | 10.24 | 5.476x | + + +> Alle Benchmarks stellen Durchschnittswerte von über 100 Durchläufen dar. Siehe [benchmarks.py](https://github.com/D4Vinci/Scrapling/blob/main/benchmarks.py) für die Methodik. + +## Installation + +Scrapling erfordert Python 3.10 oder höher: + +```bash +pip install scrapling +``` + +Ab v0.3.2 enthält diese Installation nur die Parser-Engine und ihre Abhängigkeiten, ohne Fetcher oder Kommandozeilenabhängigkeiten. + +### Optionale Abhängigkeiten + +1. Wenn Sie eine der folgenden zusätzlichen Funktionen, die Fetcher oder ihre Klassen verwenden möchten, müssen Sie die Abhängigkeiten der Fetcher installieren und dann ihre Browser-Abhängigkeiten mit + ```bash + pip install "scrapling[fetchers]" + + scrapling install + ``` + + Dies lädt alle Browser mit ihren Systemabhängigkeiten und Fingerabdruck-Manipulationsabhängigkeiten herunter. + +2. Zusätzliche Funktionen: + - MCP-Server-Funktion installieren: + ```bash + pip install "scrapling[ai]" + ``` + - Shell-Funktionen installieren (Web-Scraping-Shell und der `extract`-Befehl): + ```bash + pip install "scrapling[shell]" + ``` + - Alles installieren: + ```bash + pip install "scrapling[all]" + ``` + Denken Sie daran, dass Sie nach einem dieser Extras (falls noch nicht geschehen) die Browser-Abhängigkeiten mit `scrapling install` installieren müssen + +### Docker +Sie können auch ein Docker-Image mit allen Extras und Browsern mit dem folgenden Befehl von DockerHub installieren: +```bash +docker pull pyd4vinci/scrapling +``` +Oder laden Sie es aus der GitHub-Registry herunter: +```bash +docker pull ghcr.io/d4vinci/scrapling:latest +``` +Dieses Image wird automatisch über GitHub Actions im Hauptzweig des Repositorys erstellt und gepusht. + +## Beitragen + +Wir freuen uns über Beiträge! Bitte lesen Sie unsere [Beitragsrichtlinien](https://github.com/D4Vinci/Scrapling/blob/main/CONTRIBUTING.md), bevor Sie beginnen. + +## Haftungsausschluss + +> [!CAUTION] +> Diese Bibliothek wird nur zu Bildungs- und Forschungszwecken bereitgestellt. Durch die Nutzung dieser Bibliothek erklären Sie sich damit einverstanden, lokale und internationale Gesetze zum Daten-Scraping und Datenschutz einzuhalten. Die Autoren und Mitwirkenden sind nicht verantwortlich für Missbrauch dieser Software. Respektieren Sie immer die Nutzungsbedingungen von Websites und robots.txt-Dateien. + +## Lizenz + +Diese Arbeit ist unter der BSD-3-Clause-Lizenz lizenziert. + +## Danksagungen + +Dieses Projekt enthält angepassten Code von: +- Parsel (BSD-Lizenz) – Verwendet für [translator](https://github.com/D4Vinci/Scrapling/blob/main/scrapling/core/translator.py)-Submodul + +## Dank und Referenzen + +- [Daijros](https://github.com/daijro) brillante Arbeit an [BrowserForge](https://github.com/daijro/browserforge) und [Camoufox](https://github.com/daijro/camoufox) +- [Vinyzus](https://github.com/Vinyzu) brillante Arbeit an [Botright](https://github.com/Vinyzu/Botright) und [PatchRight](https://github.com/Kaliiiiiiiiii-Vinyzu/patchright) +- [brotector](https://github.com/kaliiiiiiiiii/brotector) für Browser-Erkennungs-Umgehungstechniken +- [fakebrowser](https://github.com/kkoooqq/fakebrowser) und [BotBrowser](https://github.com/botswin/BotBrowser) für Fingerprinting-Forschung + +--- +
Entworfen und hergestellt mit ❤️ von Karim Shoair.

\ No newline at end of file diff --git a/docs/README_ES.md b/docs/README_ES.md new file mode 100644 index 0000000..4669594 --- /dev/null +++ b/docs/README_ES.md @@ -0,0 +1,331 @@ +

+
+ main poster +
+ ¡Web Scraping fácil y sin esfuerzo como debería ser! +

+

+ + Tests + + PyPI version + + PyPI Downloads +
+ + Discord + + + X (formerly Twitter) Follow + +
+ + Supported Python versions +

+ +

+ + Métodos de selección + + · + + Elegir un fetcher + + · + + CLI + + · + + Modo MCP + + · + + Migrar desde Beautifulsoup + +

+ +**Deja de luchar contra sistemas anti-bot. Deja de reescribir selectores después de cada actualización del sitio web.** + +Scrapling no es solo otra biblioteca de Web Scraping. Es la primera biblioteca de scraping **adaptativa** que aprende de los cambios de los sitios web y evoluciona con ellos. Mientras que otras bibliotecas se rompen cuando los sitios web actualizan su estructura, Scrapling relocaliza automáticamente tus elementos y mantiene tus scrapers funcionando. + +Construido para la Web moderna, Scrapling presenta **su propio motor de análisis rápido** y fetchers para manejar todos los desafíos de Web Scraping que enfrentas o enfrentarás. Construido por Web Scrapers para Web Scrapers y usuarios regulares, hay algo para todos. + +```python +>> from scrapling.fetchers import Fetcher, AsyncFetcher, StealthyFetcher, DynamicFetcher +>> StealthyFetcher.adaptive = True +# ¡Obtén el código fuente de sitios web bajo el radar! +>> page = StealthyFetcher.fetch('https://example.com', headless=True, network_idle=True) +>> print(page.status) +200 +>> products = page.css('.product', auto_save=True) # ¡Extrae datos que sobreviven a cambios de diseño del sitio web! +>> # Más tarde, si la estructura del sitio web cambia, pasa `adaptive=True` +>> products = page.css('.product', adaptive=True) # ¡y Scrapling aún los encuentra! +``` + +# Patrocinadores + + + + + + + + + + + + + +¿Quieres mostrar tu anuncio aquí? ¡Haz clic [aquí](https://github.com/sponsors/D4Vinci) y elige el nivel que te convenga! + +--- + +## Características Principales + +### Obtención Avanzada de Sitios Web con Soporte de Sesión +- **Solicitudes HTTP**: Solicitudes HTTP rápidas y sigilosas con la clase `Fetcher`. Puede imitar la huella TLS de los navegadores, encabezados y usar HTTP3. +- **Carga Dinámica**: Obtén sitios web dinámicos con automatización completa del navegador a través de la clase `DynamicFetcher` compatible con Chromium de Playwright, Chrome real y modo sigiloso personalizado. +- **Evasión Anti-bot**: Capacidades de sigilo avanzadas con `StealthyFetcher` usando una versión modificada de Firefox y falsificación de huellas digitales. Puede evadir todos los tipos de Turnstile e Interstitial de Cloudflare con automatización fácilmente. +- **Gestión de Sesión**: Soporte de sesión persistente con las clases `FetcherSession`, `StealthySession` y `DynamicSession` para la gestión de cookies y estado entre solicitudes. +- **Soporte Async**: Soporte async completo en todos los fetchers y clases de sesión async dedicadas. + +### Scraping Adaptativo e Integración con IA +- 🔄 **Seguimiento Inteligente de Elementos**: Relocaliza elementos después de cambios en el sitio web usando algoritmos inteligentes de similitud. +- 🎯 **Selección Flexible Inteligente**: Selectores CSS, selectores XPath, búsqueda basada en filtros, búsqueda de texto, búsqueda regex y más. +- 🔍 **Encontrar Elementos Similares**: Localiza automáticamente elementos similares a los elementos encontrados. +- 🤖 **Servidor MCP para usar con IA**: Servidor MCP integrado para Web Scraping asistido por IA y extracción de datos. El servidor MCP presenta capacidades personalizadas y poderosas que utilizan Scrapling para extraer contenido específico antes de pasarlo a la IA (Claude/Cursor/etc), acelerando así las operaciones y reduciendo costos al minimizar el uso de tokens. ([video demo](https://www.youtube.com/watch?v=qyFk3ZNwOxE)) + +### Arquitectura de Alto Rendimiento y Probada en Batalla +- 🚀 **Ultrarrápido**: Rendimiento optimizado que supera a la mayoría de las bibliotecas de scraping de Python. +- 🔋 **Eficiente en Memoria**: Estructuras de datos optimizadas y carga diferida para una huella de memoria mínima. +- ⚡ **Serialización JSON Rápida**: 10 veces más rápido que la biblioteca estándar. +- 🏗️ **Probado en batalla**: Scrapling no solo tiene una cobertura de prueba del 92% y cobertura completa de type hints, sino que ha sido utilizado diariamente por cientos de Web Scrapers durante el último año. + +### Experiencia Amigable para Desarrolladores/Web Scrapers +- 🎯 **Shell Interactivo de Web Scraping**: Shell IPython integrado opcional con integración de Scrapling, atajos y nuevas herramientas para acelerar el desarrollo de scripts de Web Scraping, como convertir solicitudes curl a solicitudes Scrapling y ver resultados de solicitudes en tu navegador. +- 🚀 **Úsalo directamente desde la Terminal**: Opcionalmente, ¡puedes usar Scrapling para hacer scraping de una URL sin escribir ni una sola línea de código! +- 🛠️ **API de Navegación Rica**: Recorrido avanzado del DOM con métodos de navegación de padres, hermanos e hijos. +- 🧬 **Procesamiento de Texto Mejorado**: Métodos integrados de regex, limpieza y operaciones de cadena optimizadas. +- 📝 **Generación Automática de Selectores**: Genera selectores CSS/XPath robustos para cualquier elemento. +- 🔌 **API Familiar**: Similar a Scrapy/BeautifulSoup con los mismos pseudo-elementos usados en Scrapy/Parsel. +- 📘 **Cobertura Completa de Tipos**: Type hints completos para excelente soporte de IDE y autocompletado de código. +- 🔋 **Imagen Docker Lista**: Con cada lanzamiento, se construye y publica automáticamente una imagen Docker que contiene todos los navegadores. + +## Empezando + +### Uso Básico +```python +from scrapling.fetchers import Fetcher, StealthyFetcher, DynamicFetcher +from scrapling.fetchers import FetcherSession, StealthySession, DynamicSession + +# Solicitudes HTTP con soporte de sesión +with FetcherSession(impersonate='chrome') as session: # Usa la última versión de la huella TLS de Chrome + page = session.get('https://quotes.toscrape.com/', stealthy_headers=True) + quotes = page.css('.quote .text::text') + +# O usa solicitudes de una sola vez +page = Fetcher.get('https://quotes.toscrape.com/') +quotes = page.css('.quote .text::text') + +# Modo sigiloso avanzado (Mantén el navegador abierto hasta que termines) +with StealthySession(headless=True, solve_cloudflare=True) as session: + page = session.fetch('https://nopecha.com/demo/cloudflare', google_search=False) + data = page.css('#padded_content a') + +# O usa el estilo de solicitud de una sola vez, abre el navegador para esta solicitud, luego lo cierra después de terminar +page = StealthyFetcher.fetch('https://nopecha.com/demo/cloudflare') +data = page.css('#padded_content a') + +# Automatización completa del navegador (Mantén el navegador abierto hasta que termines) +with DynamicSession(headless=True) as session: + page = session.fetch('https://quotes.toscrape.com/', network_idle=True) + quotes = page.css('.quote .text::text') + +# O usa el estilo de solicitud de una sola vez +page = DynamicFetcher.fetch('https://quotes.toscrape.com/', network_idle=True) +quotes = page.css('.quote .text::text') +``` + +### Selección de Elementos +```python +# CSS selectors +page.css('a::text') # Extracta texto +page.css('a::attr(href)') # Extracta atributos +page.css('a', recursive=False) # Solo elementos directos +page.css('a', auto_save=True) # Guarda posiciones de los elementos automáticamente + +# XPath +page.xpath('//a/text()') + +# Búsqueda flexible +page.find_by_text('Python', first_match=True) # Encuentra por texto +page.find_by_regex(r'\d{4}') # Encuentra por patrón regex +page.find('div', {'class': 'container'}) # Encuentra por atributos + +# Navegación +element.parent # Obtener elemento padre +element.next_sibling # Obtener siguiente hermano +element.children # Obtener hijos + +# Elementos similares +similar = page.get_similar(element) # Encuentra elementos similares + +# Scraping adaptativo +saved_elements = page.css('.product', auto_save=True) +# Más tarde, cuando el sitio web cambia: +page.css('.product', adaptive=True) # Encuentra elementos usando posiciones guardadas +``` + +### Uso de Sesión +```python +from scrapling.fetchers import FetcherSession, AsyncFetcherSession + +# Sesión sincrónica +with FetcherSession() as session: + # Las cookies se mantienen automáticamente + page1 = session.get('https://quotes.toscrape.com/login') + page2 = session.post('https://quotes.toscrape.com/login', data={'username': 'admin', 'password': 'admin'}) + + # Cambiar fingerprint del navegador si es necesario + page2 = session.get('https://quotes.toscrape.com/', impersonate='firefox135') + +# Uso de sesión async +async with AsyncStealthySession(max_pages=2) as session: + tasks = [] + urls = ['https://example.com/page1', 'https://example.com/page2'] + + for url in urls: + task = session.fetch(url) + tasks.append(task) + + print(session.get_pool_stats()) # Opcional - El estado del pool de pestañas del navegador (ocupado/libre/error) + results = await asyncio.gather(*tasks) + print(session.get_pool_stats()) +``` + +## CLI y Shell Interactivo + +Scrapling v0.3 incluye una poderosa interfaz de línea de comandos: + +[![asciicast](https://asciinema.org/a/736339.svg)](https://asciinema.org/a/736339) + +```bash +# Lanzar shell interactivo de Web Scraping +scrapling shell + +# Extraer páginas a un archivo directamente sin programar (Extrae el contenido dentro de la etiqueta `body` por defecto) +# Si el archivo de salida termina con `.txt`, entonces se extraerá el contenido de texto del objetivo. +# Si termina con `.md`, será una representación markdown del contenido HTML, y `.html` será el contenido HTML directamente. +scrapling extract get 'https://example.com' content.md +scrapling extract get 'https://example.com' content.txt --css-selector '#fromSkipToProducts' --impersonate 'chrome' # Todos los elementos que coinciden con el selector CSS '#fromSkipToProducts' +scrapling extract fetch 'https://example.com' content.md --css-selector '#fromSkipToProducts' --no-headless +scrapling extract stealthy-fetch 'https://nopecha.com/demo/cloudflare' captchas.html --css-selector '#padded_content a' --solve-cloudflare +``` + +> [!NOTE] +> Hay muchas características adicionales, pero queremos mantener esta página concisa, como el servidor MCP y el Shell Interactivo de Web Scraping. Consulta la documentación completa [aquí](https://scrapling.readthedocs.io/en/latest/) + +## Benchmarks de Rendimiento + +Scrapling no solo es poderoso, también es increíblemente rápido, y las actualizaciones desde la versión 0.3 han brindado mejoras de rendimiento excepcionales en todas las operaciones. + +### Prueba de Velocidad de Extracción de Texto (5000 elementos anidados) + +| # | Biblioteca | Tiempo (ms) | vs Scrapling | +|---|:--------------------:|:-----------:|:------------:| +| 1 | Scrapling | 1.92 | 1.0x | +| 2 | Parsel/Scrapy | 1.99 | 1.036x | +| 3 | Raw Lxml | 2.33 | 1.214x | +| 4 | PyQuery | 20.61 | ~11x | +| 5 | Selectolax | 80.65 | ~42x | +| 6 | BS4 con Lxml | 1283.21 | ~698x | +| 7 | MechanicalSoup | 1304.57 | ~679x | +| 8 | BS4 con html5lib | 3331.96 | ~1735x | + +### Rendimiento de Similitud de Elementos y Búsqueda de Texto + +Las capacidades de búsqueda adaptativa de elementos de Scrapling superan significativamente a las alternativas: + +| Biblioteca | Tiempo (ms) | vs Scrapling | +|--------------|:-----------:|:------------:| +| Scrapling | 1.87 | 1.0x | +| AutoScraper | 10.24 | 5.476x | + + +> Todos los benchmarks representan promedios de más de 100 ejecuciones. Ver [benchmarks.py](https://github.com/D4Vinci/Scrapling/blob/main/benchmarks.py) para la metodología. + +## Instalación + +Scrapling requiere Python 3.10 o superior: + +```bash +pip install scrapling +``` + +A partir de v0.3.2, esta instalación solo incluye el motor de análisis y sus dependencias, sin ningún fetcher o dependencias de línea de comandos. + +### Dependencias Opcionales + +1. Si vas a usar alguna de las características adicionales a continuación, los fetchers, o sus clases, entonces necesitas instalar las dependencias de los fetchers y luego instalar sus dependencias del navegador con + ```bash + pip install "scrapling[fetchers]" + + scrapling install + ``` + + Esto descarga todos los navegadores con sus dependencias del sistema y dependencias de manipulación de huellas digitales. + +2. Características adicionales: + - Instalar la característica del servidor MCP: + ```bash + pip install "scrapling[ai]" + ``` + - Instalar características del shell (shell de Web Scraping y el comando `extract`): + ```bash + pip install "scrapling[shell]" + ``` + - Instalar todo: + ```bash + pip install "scrapling[all]" + ``` + Recuerda que necesitas instalar las dependencias del navegador con `scrapling install` después de cualquiera de estos extras (si no lo hiciste ya) + +### Docker +También puedes instalar una imagen Docker con todos los extras y navegadores con el siguiente comando desde DockerHub: +```bash +docker pull pyd4vinci/scrapling +``` +O descárgala desde el registro de GitHub: +```bash +docker pull ghcr.io/d4vinci/scrapling:latest +``` +Esta imagen se construye y publica automáticamente a través de GitHub actions en la rama principal del repositorio. + +## Contribuir + +¡Damos la bienvenida a las contribuciones! Por favor lee nuestras [pautas de contribución](https://github.com/D4Vinci/Scrapling/blob/main/CONTRIBUTING.md) antes de comenzar. + +## Descargo de Responsabilidad + +> [!CAUTION] +> Esta biblioteca se proporciona solo con fines educativos y de investigación. Al usar esta biblioteca, aceptas cumplir con las leyes locales e internacionales de scraping de datos y privacidad. Los autores y contribuyentes no son responsables de ningún mal uso de este software. Respeta siempre los términos de servicio de los sitios web y los archivos robots.txt. + +## Licencia + +Este trabajo está licenciado bajo la Licencia BSD-3-Clause. + +## Agradecimientos + +Este proyecto incluye código adaptado de: +- Parsel (Licencia BSD)—Usado para el submódulo [translator](https://github.com/D4Vinci/Scrapling/blob/main/scrapling/core/translator.py) + +## Agradecimientos y Referencias + +- El brillante trabajo de [Daijro](https://github.com/daijro) en [BrowserForge](https://github.com/daijro/browserforge) y [Camoufox](https://github.com/daijro/camoufox) +- El brillante trabajo de [Vinyzu](https://github.com/Vinyzu) en [Botright](https://github.com/Vinyzu/Botright) y [PatchRight](https://github.com/Kaliiiiiiiiii-Vinyzu/patchright) +- [brotector](https://github.com/kaliiiiiiiiii/brotector) por técnicas de evasión de detección de navegador +- [fakebrowser](https://github.com/kkoooqq/fakebrowser) y [BotBrowser](https://github.com/botswin/BotBrowser) por investigación de huellas digitales + +--- +
Diseñado y elaborado con ❤️ por Karim Shoair.

\ No newline at end of file diff --git a/docs/README_JP.md b/docs/README_JP.md new file mode 100644 index 0000000..0a7b4ec --- /dev/null +++ b/docs/README_JP.md @@ -0,0 +1,331 @@ +

+
+ main poster +
+ 簡単で効率的なウェブスクレイピング、あるべき姿！ +

+

+ + Tests + + PyPI version + + PyPI Downloads +
+ + Discord + + + X (formerly Twitter) Follow + +
+ + Supported Python versions +

+ +

+ + 選択メソッド + + · + + フェッチャーの選択 + + · + + CLI + + · + + MCPモード + + · + + Beautifulsoupからの移行 + +

+ +**アンチボットシステムとの戦いをやめましょう。ウェブサイトが更新されるたびにセレクタを書き直すのをやめましょう。** + +Scraplingは単なるウェブスクレイピングライブラリではありません。ウェブサイトの変更から学習し、それとともに進化する最初の**適応型**スクレイピングライブラリです。他のライブラリがウェブサイトの構造が更新されると壊れる一方で、Scraplingは自動的に要素を再配置し、スクレイパーを稼働し続けます。 + +モダンウェブ向けに構築されたScraplingは、**独自の高速パースエンジン**とフェッチャーを備えており、あなたが直面する、または直面するであろうすべてのウェブスクレイピングの課題に対応します。ウェブスクレイパーによってウェブスクレイパーと一般ユーザーのために構築され、誰にでも何かがあります。 + +```python +>> from scrapling.fetchers import Fetcher, AsyncFetcher, StealthyFetcher, DynamicFetcher +>> StealthyFetcher.adaptive = True +# レーダーの下でウェブサイトのソースを取得！ +>> page = StealthyFetcher.fetch('https://example.com', headless=True, network_idle=True) +>> print(page.status) +200 +>> products = page.css('.product', auto_save=True) # ウェブサイトのデザイン変更に耐えるデータをスクレイプ！ +>> # 後でウェブサイトの構造が変わったら、`adaptive=True`を渡す +>> products = page.css('.product', adaptive=True) # そしてScraplingはまだそれらを見つけます！ +``` + +# スポンサー + + + + + + + + + + + + + +ここに広告を表示したいですか？[こちら](https://github.com/sponsors/D4Vinci)をクリックして、あなたに合ったティアを選択してください！ + +--- + +## 主な機能 + +### セッションサポート付き高度なウェブサイト取得 +- **HTTPリクエスト**：`Fetcher`クラスで高速でステルスなHTTPリクエスト。ブラウザのTLSフィンガープリント、ヘッダーを模倣し、HTTP3を使用できます。 +- **動的読み込み**：Playwright's Chromium、実際のChrome、カスタムステルスモードをサポートする`DynamicFetcher`クラスを通じた完全なブラウザ自動化で動的ウェブサイトを取得。 +- **アンチボット回避**：修正されたFirefoxとフィンガープリント偽装を使用した`StealthyFetcher`による高度なステルス機能。自動化でCloudflareのTurnstileとInterstitialのすべてのタイプを簡単に回避できます。 +- **セッション管理**：リクエスト間でCookieと状態を管理するための`FetcherSession`、`StealthySession`、`DynamicSession`クラスによる永続的なセッションサポート。 +- **非同期サポート**：すべてのフェッチャーと専用非同期セッションクラス全体での完全な非同期サポート。 + +### 適応型スクレイピングとAI統合 +- 🔄 **スマート要素追跡**：インテリジェントな類似性アルゴリズムを使用してウェブサイトの変更後に要素を再配置。 +- 🎯 **スマート柔軟選択**：CSSセレクタ、XPathセレクタ、フィルタベース検索、テキスト検索、正規表現検索など。 +- 🔍 **類似要素を見つける**：見つかった要素に類似した要素を自動的に特定。 +- 🤖 **AIと使用するMCPサーバー**：AI支援ウェブスクレイピングとデータ抽出のための組み込みMCPサーバー。MCPサーバーは、AI（Claude/Cursorなど）に渡す前にScraplingを利用してターゲットコンテンツを抽出するカスタムで強力な機能を備えており、操作を高速化し、トークン使用量を最小限に抑えることでコストを削減します。（[デモビデオ](https://www.youtube.com/watch?v=qyFk3ZNwOxE)） + +### 高性能で実戦テスト済みのアーキテクチャ +- 🚀 **高速**：ほとんどのPythonスクレイピングライブラリを上回る最適化されたパフォーマンス。 +- 🔋 **メモリ効率**：最小のメモリフットプリントのための最適化されたデータ構造と遅延読み込み。 +- ⚡ **高速JSONシリアル化**：標準ライブラリの10倍の速度。 +- 🏗️ **実戦テスト済み**：Scraplingは92%のテストカバレッジと完全な型ヒントカバレッジを備えているだけでなく、過去1年間に数百人のウェブスクレイパーによって毎日使用されてきました。 + +### 開発者/ウェブスクレイパーにやさしい体験 +- 🎯 **インタラクティブウェブスクレイピングシェル**：Scraping統合、ショートカット、curlリクエストをScraplingリクエストに変換したり、ブラウザでリクエスト結果を表示したりするなどの新しいツールを備えたオプションの組み込みIPythonシェルで、ウェブスクレイピングスクリプトの開発を加速します。 +- 🚀 **ターミナルから直接使用**：オプションで、コードを一行も書かずにScraplingを使用してURLをスクレイプできます！ +- 🛠️ **豊富なナビゲーションAPI**：親、兄弟、子のナビゲーションメソッドによる高度なDOMトラバーサル。 +- 🧬 **強化されたテキスト処理**：組み込みの正規表現、クリーニングメソッド、最適化された文字列操作。 +- 📝 **自動セレクタ生成**：任意の要素に対して堅牢なCSS/XPathセレクタを生成。 +- 🔌 **馴染みのあるAPI**：Scrapy/Parselで使用されている同じ疑似要素を持つScrapy/BeautifulSoupに似ています。 +- 📘 **完全な型カバレッジ**：優れたIDEサポートとコード補完のための完全な型ヒント。 +- 🔋 **すぐに使えるDockerイメージ**：各リリースで、すべてのブラウザを含むDockerイメージが自動的にビルドおよびプッシュされます。 + +## はじめに + +### 基本的な使い方 +```python +from scrapling.fetchers import Fetcher, StealthyFetcher, DynamicFetcher +from scrapling.fetchers import FetcherSession, StealthySession, DynamicSession + +# セッションサポート付きHTTPリクエスト +with FetcherSession(impersonate='chrome') as session: # ChromeのTLSフィンガープリントの最新バージョンを使用 + page = session.get('https://quotes.toscrape.com/', stealthy_headers=True) + quotes = page.css('.quote .text::text') + +# または一回限りのリクエストを使用 +page = Fetcher.get('https://quotes.toscrape.com/') +quotes = page.css('.quote .text::text') + +# 高度なステルスモード（完了するまでブラウザを開いたままにする） +with StealthySession(headless=True, solve_cloudflare=True) as session: + page = session.fetch('https://nopecha.com/demo/cloudflare', google_search=False) + data = page.css('#padded_content a') + +# または一回限りのリクエストスタイルを使用、このリクエストのためにブラウザを開き、完了後に閉じる +page = StealthyFetcher.fetch('https://nopecha.com/demo/cloudflare') +data = page.css('#padded_content a') + +# 完全なブラウザ自動化（完了するまでブラウザを開いたままにする） +with DynamicSession(headless=True) as session: + page = session.fetch('https://quotes.toscrape.com/', network_idle=True) + quotes = page.css('.quote .text::text') + +# または一回限りのリクエストスタイルを使用 +page = DynamicFetcher.fetch('https://quotes.toscrape.com/', network_idle=True) +quotes = page.css('.quote .text::text') +``` + +### 要素の選択 +```python +# CSSセレクタ +page.css('a::text') # テキストを抽出 +page.css('a::attr(href)') # 属性を抽出 +page.css('a', recursive=False) # 直接の要素のみ +page.css('a', auto_save=True) # 要素の位置を自動保存 + +# XPath +page.xpath('//a/text()') + +# 柔軟な検索 +page.find_by_text('Python', first_match=True) # テキストで検索 +page.find_by_regex(r'\d{4}') # 正規表現パターンで検索 +page.find('div', {'class': 'container'}) # 属性で検索 + +# ナビゲーション +element.parent # 親要素を取得 +element.next_sibling # 次の兄弟を取得 +element.children # 子要素を取得 + +# 類似要素 +similar = page.get_similar(element) # 類似要素を見つける + +# 適応型スクレイピング +saved_elements = page.css('.product', auto_save=True) +# 後でウェブサイトが変更されたとき： +page.css('.product', adaptive=True) # 保存された位置を使用して要素を見つける +``` + +### セッションの使用 +```python +from scrapling.fetchers import FetcherSession, AsyncFetcherSession + +# 同期セッション +with FetcherSession() as session: + # Cookieは自動的に維持されます + page1 = session.get('https://quotes.toscrape.com/login') + page2 = session.post('https://quotes.toscrape.com/login', data={'username': 'admin', 'password': 'admin'}) + + # 必要に応じてブラウザのフィンガープリントを切り替え + page2 = session.get('https://quotes.toscrape.com/', impersonate='firefox135') + +# 非同期セッションの使用 +async with AsyncStealthySession(max_pages=2) as session: + tasks = [] + urls = ['https://example.com/page1', 'https://example.com/page2'] + + for url in urls: + task = session.fetch(url) + tasks.append(task) + + print(session.get_pool_stats()) # オプション - ブラウザタブプールのステータス（ビジー/フリー/エラー） + results = await asyncio.gather(*tasks) + print(session.get_pool_stats()) +``` + +## CLIとインタラクティブシェル + +Scrapling v0.3には強力なコマンドラインインターフェースが含まれています： + +[![asciicast](https://asciinema.org/a/736339.svg)](https://asciinema.org/a/736339) + +```bash +# インタラクティブウェブスクレイピングシェルを起動 +scrapling shell + +# プログラミングせずに直接ページをファイルに抽出（デフォルトで`body`タグ内のコンテンツを抽出） +# 出力ファイルが`.txt`で終わる場合、ターゲットのテキストコンテンツが抽出されます。 +# `.md`で終わる場合、HTMLコンテンツのMarkdown表現になり、`.html`は直接HTMLコンテンツになります。 +scrapling extract get 'https://example.com' content.md +scrapling extract get 'https://example.com' content.txt --css-selector '#fromSkipToProducts' --impersonate 'chrome' # CSSセレクタ'#fromSkipToProducts'に一致するすべての要素 +scrapling extract fetch 'https://example.com' content.md --css-selector '#fromSkipToProducts' --no-headless +scrapling extract stealthy-fetch 'https://nopecha.com/demo/cloudflare' captchas.html --css-selector '#padded_content a' --solve-cloudflare +``` + +> [!NOTE] +> MCPサーバーやインタラクティブウェブスクレイピングシェルなど、他にも多くの追加機能がありますが、このページは簡潔に保ちたいと思います。完全なドキュメントは[こちら](https://scrapling.readthedocs.io/en/latest/)をご覧ください + +## パフォーマンスベンチマーク + +Scraplingは強力であるだけでなく、驚くほど高速で、バージョン0.3以降のアップデートはすべての操作で優れたパフォーマンス向上を実現しています。 + +### テキスト抽出速度テスト（5000個のネストされた要素） + +| # | ライブラリ | 時間(ms) | vs Scrapling | +|---|:-------------------:|:--------:|:------------:| +| 1 | Scrapling | 1.92 | 1.0x | +| 2 | Parsel/Scrapy | 1.99 | 1.036x | +| 3 | Raw Lxml | 2.33 | 1.214x | +| 4 | PyQuery | 20.61 | ~11x | +| 5 | Selectolax | 80.65 | ~42x | +| 6 | BS4 with Lxml | 1283.21 | ~698x | +| 7 | MechanicalSoup | 1304.57 | ~679x | +| 8 | BS4 with html5lib | 3331.96 | ~1735x | + +### 要素類似性とテキスト検索のパフォーマンス + +Scraplingの適応型要素検索機能は代替手段を大幅に上回ります： + +| ライブラリ | 時間(ms) | vs Scrapling | +|-------------|:--------:|:------------:| +| Scrapling | 1.87 | 1.0x | +| AutoScraper | 10.24 | 5.476x | + + +> すべてのベンチマークは100回以上の実行の平均を表します。方法論については[benchmarks.py](https://github.com/D4Vinci/Scrapling/blob/main/benchmarks.py)を参照してください。 + +## インストール + +ScraplingにはPython 3.10以上が必要です： + +```bash +pip install scrapling +``` + +v0.3.2以降、このインストールにはパーサーエンジンとその依存関係のみが含まれており、フェッチャーやコマンドライン依存関係は含まれていません。 + +### オプションの依存関係 + +1. 以下の追加機能、フェッチャー、またはそれらのクラスのいずれかを使用する場合は、フェッチャーの依存関係をインストールしてから、次のコマンドでブラウザの依存関係をインストールする必要があります + ```bash + pip install "scrapling[fetchers]" + + scrapling install + ``` + + これにより、すべてのブラウザとそのシステム依存関係およびフィンガープリント操作依存関係がダウンロードされます。 + +2. 追加機能： + - MCPサーバー機能をインストール： + ```bash + pip install "scrapling[ai]" + ``` + - シェル機能（ウェブスクレイピングシェルと`extract`コマンド）をインストール： + ```bash + pip install "scrapling[shell]" + ``` + - すべてをインストール： + ```bash + pip install "scrapling[all]" + ``` + これらの追加機能のいずれかの後（まだインストールしていない場合）、`scrapling install`でブラウザの依存関係をインストールする必要があることを忘れないでください + +### Docker +DockerHubから次のコマンドですべての追加機能とブラウザを含むDockerイメージをインストールすることもできます： +```bash +docker pull pyd4vinci/scrapling +``` +またはGitHubレジストリからダウンロード： +```bash +docker pull ghcr.io/d4vinci/scrapling:latest +``` +このイメージは、リポジトリのメインブランチでGitHub actionsを通じて自動的にビルドおよびプッシュされます。 + +## 貢献 + +貢献を歓迎します！始める前に[貢献ガイドライン](https://github.com/D4Vinci/Scrapling/blob/main/CONTRIBUTING.md)をお読みください。 + +## 免責事項 + +> [!CAUTION] +> このライブラリは教育および研究目的のみで提供されています。このライブラリを使用することにより、地域および国際的なデータスクレイピングおよびプライバシー法に準拠することに同意したものとみなされます。著者および貢献者は、このソフトウェアの誤用について責任を負いません。常にウェブサイトの利用規約とrobots.txtファイルを尊重してください。 + +## ライセンス + +この作品はBSD-3-Clauseライセンスの下でライセンスされています。 + +## 謝辞 + +このプロジェクトには次から適応されたコードが含まれています： +- Parsel（BSDライセンス）— [translator](https://github.com/D4Vinci/Scrapling/blob/main/scrapling/core/translator.py)サブモジュールに使用 + +## 感謝と参考文献 + +- [Daijro](https://github.com/daijro)の[BrowserForge](https://github.com/daijro/browserforge)と[Camoufox](https://github.com/daijro/camoufox)における素晴らしい仕事 +- [Vinyzu](https://github.com/Vinyzu)の[Botright](https://github.com/Vinyzu/Botright)と[PatchRight](https://github.com/Kaliiiiiiiiii-Vinyzu/patchright)における素晴らしい仕事 +- ブラウザ検出回避技術を提供する[brotector](https://github.com/kaliiiiiiiiii/brotector) +- フィンガープリント研究を提供する[fakebrowser](https://github.com/kkoooqq/fakebrowser)と[BotBrowser](https://github.com/botswin/BotBrowser) + +--- +
Karim Shoairによって❤️でデザインおよび作成されました。

\ No newline at end of file diff --git a/docs/README_RU.md b/docs/README_RU.md new file mode 100644 index 0000000..807ff38 --- /dev/null +++ b/docs/README_RU.md @@ -0,0 +1,331 @@ +

+
+ main poster +
+ Простой, легкий веб-скрапинг, каким он и должен быть! +

+

+ + Tests + + PyPI version + + PyPI Downloads +
+ + Discord + + + X (formerly Twitter) Follow + +
+ + Supported Python versions +

+ +

+ + Методы выбора + + · + + Выбор фетчера + + · + + CLI + + · + + Режим MCP + + · + + Миграция с Beautifulsoup + +

+ +**Прекратите бороться с анти-ботовыми системами. Прекратите переписывать селекторы после каждого обновления сайта.** + +Scrapling - это не просто очередная библиотека для веб-скрапинга. Это первая **адаптивная** библиотека для скрапинга, которая учится на изменениях сайтов и развивается вместе с ними. В то время как другие библиотеки ломаются, когда сайты обновляют свою структуру, Scrapling автоматически перемещает ваши элементы и поддерживает работу ваших скраперов. + +Созданный для современного веба, Scrapling имеет **собственный быстрый движок парсинга** и фетчеры для решения всех задач веб-скрапинга, с которыми вы сталкиваетесь или столкнетесь. Созданный веб-скраперами для веб-скраперов и обычных пользователей, здесь есть что-то для каждого. + +```python +>> from scrapling.fetchers import Fetcher, AsyncFetcher, StealthyFetcher, DynamicFetcher +>> StealthyFetcher.adaptive = True +# Получайте исходный код сайтов незаметно! +>> page = StealthyFetcher.fetch('https://example.com', headless=True, network_idle=True) +>> print(page.status) +200 +>> products = page.css('.product', auto_save=True) # Скрапьте данные, которые переживут изменения дизайна сайта! +>> # Позже, если структура сайта изменится, передайте `adaptive=True` +>> products = page.css('.product', adaptive=True) # и Scrapling все равно их найдет! +``` + +# Спонсоры + + + + + + + + + + + + + +Хотите показать здесь свою рекламу? Нажмите [здесь](https://github.com/sponsors/D4Vinci) и выберите подходящий вам уровень! + +--- + +## Ключевые особенности + +### Продвинутая загрузка сайтов с поддержкой сессий +- **HTTP-запросы**: Быстрые и скрытные HTTP-запросы с классом `Fetcher`. Может имитировать TLS-отпечаток браузера, заголовки и использовать HTTP3. +- **Динамическая загрузка**: Загрузка динамических сайтов с полной автоматизацией браузера через класс `DynamicFetcher`, поддерживающий Chromium от Playwright, настоящий Chrome и пользовательский режим скрытности. +- **Обход анти-ботов**: Расширенные возможности скрытности с `StealthyFetcher`, использующим модифицированную версию Firefox и подмену отпечатков. Может легко обойти все типы Turnstile и Interstitial от Cloudflare с помощью автоматизации. +- **Управление сессиями**: Поддержка постоянных сессий с классами `FetcherSession`, `StealthySession` и `DynamicSession` для управления cookie и состоянием между запросами. +- **Поддержка асинхронности**: Полная асинхронная поддержка во всех фетчерах и выделенных асинхронных классах сессий. + +### Адаптивный скрапинг и интеграция с ИИ +- 🔄 **Умное отслеживание элементов**: Перемещайте элементы после изменений сайта с помощью интеллектуальных алгоритмов подобия. +- 🎯 **Умный гибкий выбор**: CSS-селекторы, XPath-селекторы, поиск на основе фильтров, текстовый поиск, поиск по регулярным выражениям и многое другое. +- 🔍 **Поиск похожих элементов**: Автоматически находите элементы, похожие на найденные элементы. +- 🤖 **MCP-сервер для использования с ИИ**: Встроенный MCP-сервер для веб-скрапинга с помощью ИИ и извлечения данных. MCP-сервер обладает пользовательскими, мощными возможностями, которые используют Scrapling для извлечения целевого контента перед передачей его ИИ (Claude/Cursor/и т.д.), тем самым ускоряя операции и снижая затраты за счет минимизации использования токенов. ([демо-видео](https://www.youtube.com/watch?v=qyFk3ZNwOxE)) + +### Высокопроизводительная и проверенная в боях архитектура +- 🚀 **Молниеносно быстро**: Оптимизированная производительность превосходит большинство библиотек скрапинга Python. +- 🔋 **Эффективное использование памяти**: Оптимизированные структуры данных и ленивая загрузка для минимального потребления памяти. +- ⚡ **Быстрая сериализация JSON**: В 10 раз быстрее, чем стандартная библиотека. +- 🏗️ **Проверено в боях**: Scrapling имеет не только 92% покрытия тестами и полное покрытие type hints, но и ежедневно использовался сотнями веб-скраперов в течение последнего года. + +### Удобный для разработчиков/веб-скраперов опыт +- 🎯 **Интерактивная оболочка веб-скрапинга**: Опциональная встроенная оболочка IPython с интеграцией Scrapling, ярлыками и новыми инструментами для ускорения разработки скриптов веб-скрапинга, такими как преобразование curl-запросов в Scrapling-запросы и просмотр результатов запросов в вашем браузере. +- 🚀 **Используйте прямо из терминала**: При желании вы можете использовать Scrapling для скрапинга URL без написания ни одной строки кода! +- 🛠️ **Богатый API навигации**: Расширенный обход DOM с методами навигации по родителям, братьям и детям. +- 🧬 **Улучшенная обработка текста**: Встроенные регулярные выражения, методы очистки и оптимизированные операции со строками. +- 📝 **Автоматическая генерация селекторов**: Генерация надежных CSS/XPath селекторов для любого элемента. +- 🔌 **Знакомый API**: Похож на Scrapy/BeautifulSoup с теми же псевдоэлементами, используемыми в Scrapy/Parsel. +- 📘 **Полное покрытие типами**: Полные подсказки типов для отличной поддержки IDE и автодополнения кода. +- 🔋 **Готовый Docker-образ**: С каждым релизом автоматически создается и отправляется Docker-образ, содержащий все браузеры. + +## Начало работы + +### Базовое использование +```python +from scrapling.fetchers import Fetcher, StealthyFetcher, DynamicFetcher +from scrapling.fetchers import FetcherSession, StealthySession, DynamicSession + +# HTTP-запросы с поддержкой сессий +with FetcherSession(impersonate='chrome') as session: # Используйте последнюю версию TLS-отпечатка Chrome + page = session.get('https://quotes.toscrape.com/', stealthy_headers=True) + quotes = page.css('.quote .text::text') + +# Или используйте одноразовые запросы +page = Fetcher.get('https://quotes.toscrape.com/') +quotes = page.css('.quote .text::text') + +# Расширенный режим скрытности (Держите браузер открытым до завершения) +with StealthySession(headless=True, solve_cloudflare=True) as session: + page = session.fetch('https://nopecha.com/demo/cloudflare', google_search=False) + data = page.css('#padded_content a') + +# Или используйте стиль одноразового запроса, открывает браузер для этого запроса, затем закрывает его после завершения +page = StealthyFetcher.fetch('https://nopecha.com/demo/cloudflare') +data = page.css('#padded_content a') + +# Полная автоматизация браузера (Держите браузер открытым до завершения) +with DynamicSession(headless=True) as session: + page = session.fetch('https://quotes.toscrape.com/', network_idle=True) + quotes = page.css('.quote .text::text') + +# Или используйте стиль одноразового запроса +page = DynamicFetcher.fetch('https://quotes.toscrape.com/', network_idle=True) +quotes = page.css('.quote .text::text') +``` + +### Выбор элементов +```python +# CSS-селекторы +page.css('a::text') # Извлечь текст +page.css('a::attr(href)') # Извлечь атрибуты +page.css('a', recursive=False) # Только прямые элементы +page.css('a', auto_save=True) # Автоматически сохранять позиции элементов + +# XPath +page.xpath('//a/text()') + +# Гибкий поиск +page.find_by_text('Python', first_match=True) # Найти по тексту +page.find_by_regex(r'\d{4}') # Найти по паттерну regex +page.find('div', {'class': 'container'}) # Найти по атрибутам + +# Навигация +element.parent # Получить родительский элемент +element.next_sibling # Получить следующего брата +element.children # Получить дочерние элементы + +# Похожие элементы +similar = page.get_similar(element) # Найти похожие элементы + +# Адаптивный скрапинг +saved_elements = page.css('.product', auto_save=True) +# Позже, когда сайт изменится: +page.css('.product', adaptive=True) # Найти элементы используя сохраненные позиции +``` + +### Использование сессий +```python +from scrapling.fetchers import FetcherSession, AsyncFetcherSession + +# Синхронная сессия +with FetcherSession() as session: + # Cookie автоматически сохраняются + page1 = session.get('https://quotes.toscrape.com/login') + page2 = session.post('https://quotes.toscrape.com/login', data={'username': 'admin', 'password': 'admin'}) + + # При необходимости переключите отпечаток браузера + page2 = session.get('https://quotes.toscrape.com/', impersonate='firefox135') + +# Использование асинхронной сессии +async with AsyncStealthySession(max_pages=2) as session: + tasks = [] + urls = ['https://example.com/page1', 'https://example.com/page2'] + + for url in urls: + task = session.fetch(url) + tasks.append(task) + + print(session.get_pool_stats()) # Опционально - Статус пула вкладок браузера (занят/свободен/ошибка) + results = await asyncio.gather(*tasks) + print(session.get_pool_stats()) +``` + +## CLI и интерактивная оболочка + +Scrapling v0.3 включает мощный интерфейс командной строки: + +[![asciicast](https://asciinema.org/a/736339.svg)](https://asciinema.org/a/736339) + +```bash +# Запустить интерактивную оболочку веб-скрапинга +scrapling shell + +# Извлечь страницы в файл напрямую без программирования (Извлекает содержимое внутри тега `body` по умолчанию) +# Если выходной файл заканчивается на `.txt`, то будет извлечено текстовое содержимое цели. +# Если заканчивается на `.md`, это будет markdown-представление HTML-содержимого, а `.html` будет непосредственно HTML-содержимым. +scrapling extract get 'https://example.com' content.md +scrapling extract get 'https://example.com' content.txt --css-selector '#fromSkipToProducts' --impersonate 'chrome' # Все элементы, соответствующие CSS-селектору '#fromSkipToProducts' +scrapling extract fetch 'https://example.com' content.md --css-selector '#fromSkipToProducts' --no-headless +scrapling extract stealthy-fetch 'https://nopecha.com/demo/cloudflare' captchas.html --css-selector '#padded_content a' --solve-cloudflare +``` + +> [!NOTE] +> Есть много дополнительных функций, но мы хотим сохранить эту страницу краткой, например, MCP-сервер и интерактивная оболочка веб-скрапинга. Ознакомьтесь с полной документацией [здесь](https://scrapling.readthedocs.io/en/latest/) + +## Тесты производительности + +Scrapling не только мощный - он также невероятно быстрый, и обновления с версии 0.3 обеспечили исключительные улучшения производительности во всех операциях. + +### Тест скорости извлечения текста (5000 вложенных элементов) + +| # | Библиотека | Время (мс) | vs Scrapling | +|---|:--------------------:|:----------:|:------------:| +| 1 | Scrapling | 1.92 | 1.0x | +| 2 | Parsel/Scrapy | 1.99 | 1.036x | +| 3 | Raw Lxml | 2.33 | 1.214x | +| 4 | PyQuery | 20.61 | ~11x | +| 5 | Selectolax | 80.65 | ~42x | +| 6 | BS4 с Lxml | 1283.21 | ~698x | +| 7 | MechanicalSoup | 1304.57 | ~679x | +| 8 | BS4 с html5lib | 3331.96 | ~1735x | + +### Производительность подобия элементов и текстового поиска + +Возможности адаптивного поиска элементов Scrapling значительно превосходят альтернативы: + +| Библиотека | Время (мс) | vs Scrapling | +|-------------|:----------:|:------------:| +| Scrapling | 1.87 | 1.0x | +| AutoScraper | 10.24 | 5.476x | + + +> Все тесты производительности представляют собой средние значения более 100 запусков. См. [benchmarks.py](https://github.com/D4Vinci/Scrapling/blob/main/benchmarks.py) для методологии. + +## Установка + +Scrapling требует Python 3.10 или выше: + +```bash +pip install scrapling +``` + +Начиная с v0.3.2, эта установка включает только движок парсера и его зависимости, без каких-либо фетчеров или зависимостей командной строки. + +### Опциональные зависимости + +1. Если вы собираетесь использовать какие-либо из дополнительных функций ниже, фетчеры или их классы, то вам нужно установить зависимости фетчеров, а затем установить их зависимости браузера с помощью + ```bash + pip install "scrapling[fetchers]" + + scrapling install + ``` + + Это загрузит все браузеры с их системными зависимостями и зависимостями манипуляции отпечатками. + +2. Дополнительные функции: + - Установить функцию MCP-сервера: + ```bash + pip install "scrapling[ai]" + ``` + - Установить функции оболочки (оболочка веб-скрапинга и команда `extract`): + ```bash + pip install "scrapling[shell]" + ``` + - Установить все: + ```bash + pip install "scrapling[all]" + ``` + Помните, что вам нужно установить зависимости браузера с помощью `scrapling install` после любого из этих дополнений (если вы еще этого не сделали) + +### Docker +Вы также можете установить Docker-образ со всеми дополнениями и браузерами с помощью следующей команды из DockerHub: +```bash +docker pull pyd4vinci/scrapling +``` +Или скачайте его из реестра GitHub: +```bash +docker pull ghcr.io/d4vinci/scrapling:latest +``` +Этот образ автоматически создается и отправляется через GitHub actions в основной ветке репозитория. + +## Вклад + +Мы приветствуем вклад! Пожалуйста, прочитайте наши [руководства по внесению вклада](https://github.com/D4Vinci/Scrapling/blob/main/CONTRIBUTING.md) перед началом работы. + +## Отказ от ответственности + +> [!CAUTION] +> Эта библиотека предоставляется только в образовательных и исследовательских целях. Используя эту библиотеку, вы соглашаетесь соблюдать местные и международные законы о скрапинге данных и конфиденциальности. Авторы и участники не несут ответственности за любое неправомерное использование этого программного обеспечения. Всегда уважайте условия обслуживания веб-сайтов и файлы robots.txt. + +## Лицензия + +Эта работа лицензирована по лицензии BSD-3-Clause. + +## Благодарности + +Этот проект включает код, адаптированный из: +- Parsel (лицензия BSD) — Используется для подмодуля [translator](https://github.com/D4Vinci/Scrapling/blob/main/scrapling/core/translator.py) + +## Благодарности и ссылки + +- Блестящая работа [Daijro](https://github.com/daijro) над [BrowserForge](https://github.com/daijro/browserforge) и [Camoufox](https://github.com/daijro/camoufox) +- Блестящая работа [Vinyzu](https://github.com/Vinyzu) над [Botright](https://github.com/Vinyzu/Botright) и [PatchRight](https://github.com/Kaliiiiiiiiii-Vinyzu/patchright) +- [brotector](https://github.com/kaliiiiiiiiii/brotector) за техники обхода обнаружения браузера +- [fakebrowser](https://github.com/kkoooqq/fakebrowser) и [BotBrowser](https://github.com/botswin/BotBrowser) за исследование отпечатков + +--- +
Разработано и создано с ❤️ Карим Шоаир.

\ No newline at end of file diff --git a/docs/cli/extract-commands.md b/docs/cli/extract-commands.md index 5c86373..ecf84da 100644 --- a/docs/cli/extract-commands.md +++ b/docs/cli/extract-commands.md @@ -4,6 +4,14 @@ The `scrapling extract` Command lets you download and extract content from websites directly from your terminal without writing any code. Ideal for beginners, researchers, and anyone requiring rapid web data extraction. +> 💡 **Prerequisites:** +> +> 1. You’ve completed or read the [Fetchers basics](../fetching/choosing.md) page to understand what the [Response object](../fetching/choosing.md#response-object) is and which fetcher to use. +> 2. You’ve completed or read the [Querying elements](../parsing/selection.md) page to understand how to find/extract elements from the [Selector](../parsing/main_classes.md#selector)/[Response](../fetching/choosing.md#response-object) object. +> 3. You’ve completed or read the [Main classes](../parsing/main_classes.md) page to know what properties/methods the [Response](../fetching/choosing.md#response-object) class is inheriting from the [Selector](../parsing/main_classes.md#selector) class. +> 4. You’ve completed or read at least one page from the fetchers section to use here for requests: [HTTP requests](../fetching/static.md), [Dynamic websites](../fetching/dynamic.md), or [Dynamic websites with hard protections](../fetching/stealthy.md). + + ## What is the Extract Command group? The extract command is a set of simple terminal tools that: diff --git a/docs/cli/interactive-shell.md b/docs/cli/interactive-shell.md index e56939b..e12945e 100644 --- a/docs/cli/interactive-shell.md +++ b/docs/cli/interactive-shell.md @@ -6,6 +6,14 @@ The Scrapling Interactive Shell is an enhanced IPython-based environment designed specifically for Web Scraping tasks. It provides instant access to all Scrapling features, clever shortcuts, automatic page management, and advanced tools like curl command conversion. +> 💡 **Prerequisites:** +> +> 1. You’ve completed or read the [Fetchers basics](../fetching/choosing.md) page to understand what the [Response object](../fetching/choosing.md#response-object) is and which fetcher to use. +> 2. You’ve completed or read the [Querying elements](../parsing/selection.md) page to understand how to find/extract elements from the [Selector](../parsing/main_classes.md#selector)/[Response](../fetching/choosing.md#response-object) object. +> 3. You’ve completed or read the [Main classes](../parsing/main_classes.md) page to know what properties/methods the [Response](../fetching/choosing.md#response-object) class is inheriting from the [Selector](../parsing/main_classes.md#selector) class. +> 4. You’ve completed or read at least one page from the fetchers section to use here for requests: [HTTP requests](../fetching/static.md), [Dynamic websites](../fetching/dynamic.md), or [Dynamic websites with hard protections](../fetching/stealthy.md). + + ## Why use the Interactive Shell? The interactive shell transforms web scraping from a slow script-and-run cycle into a fast, exploratory experience. It's perfect for: diff --git a/docs/contributing.md b/docs/contributing.md deleted file mode 100644 index 463ba85..0000000 --- a/docs/contributing.md +++ /dev/null @@ -1,102 +0,0 @@ -Thank you for your interest in contributing to Scrapling! - -Everybody is invited and welcome to contribute to Scrapling. - -Smaller changes have a better chance of getting included in a timely manner. Adding unit tests for new features or test cases for bugs you've fixed helps us to ensure that the Pull Request (PR) is acceptable. - -There is a lot to do... - -- If you are not a developer, you can help us improve the documentation. -- If you are a developer, most of the features I'm planning to add in the future are moved to [roadmap file](https://github.com/D4Vinci/Scrapling/blob/main/ROADMAP.md), so consider reading it. - -## Running tests -Scrapling includes a comprehensive test suite that can be executed with pytest, but first, you need to install all libraries and `pytest-plugins` inside `tests/requirements.txt`. Then, running the tests will result in an output like this: - ```bash - $ pytest tests - =============================== test session starts =============================== - platform darwin -- Python 3.12.8, pytest-8.3.3, pluggy-1.5.0 -- /Users//.venv/bin/python3.12 - cachedir: .pytest_cache - rootdir: /Users//scrapling - configfile: pytest.ini - plugins: cov-5.0.0, asyncio-0.25.0, base-url-2.1.0, httpbin-2.1.0, playwright-0.5.2, anyio-4.6.2.post1, xdist-3.6.1, typeguard-4.3.0 - asyncio: mode=Mode.AUTO, asyncio_default_fixture_loop_scope=function - collected 83 items - - ...... - - =============================== 83 passed in 157.52s (0:02:37) ===================== - ``` -Hence, you can add `-n auto` to the command above to run tests in threads to increase speed. - -Bonus: You can also see the test coverage with the pytest plugin below -```bash -pytest --cov=scrapling tests/ -``` - -## Installing the latest unstable version from the dev branch -```bash -pip3 install git+https://github.com/D4Vinci/Scrapling.git@dev -``` - -## Development -Setting the scrapling logging level to `debug` makes it easier to know what's happening in the background. - ```python - >>> import logging - >>> logging.getLogger("scrapling").setLevel(logging.DEBUG) - ``` -### Code Style - -We use: - -1. Type hints for better code clarity -2. Flake8, bandit, isort, and other hooks through `pre-commit`.
Please install the hooks before committing with: - ```bash - pip install pre-commit - pre-commit install - ``` - It will run automatically on the code you push with each commit. -3. Conventional commit messages format. We use the below format for commit messages - - | Prefix | When to use it | - |-------------|--------------------------| - | `feat:` | New feature added | - | `fix:` | Bug fix | - | `docs:` | Documentation change/add | - | `test:` | Tests | - | `refactor:` | Code refactoring | - | `chore:` | Maintenance tasks | - - Example: - ``` - feat: add `adaptive` for similar elements - - - Added find_similar() method - - Implemented pattern matching - - Added tests and documentation - ``` - -### Push changes to the library - -Then, the process is straightforward. - - - Read [How to get faster PR reviews](https://github.com/kubernetes/community/blob/master/contributors/guide/pull-requests.md#best-practices-for-faster-reviews) by Kubernetes (but skip step 0 and 1) - - Fork Scrapling [Git repository](https://github.com/D4Vinci/Scrapling.git). - - Make your changes, and don't forget to create a separate virtual environment for this project. - - Ensure all tests are passing. - - Create a Pull Request against the [**dev**](https://github.com/D4Vinci/Scrapling/tree/dev) branch of Scrapling. - -A bonus: if you have more than one version of Python installed, you can use tox to run tests on each version with: -```bash -pip install tox -tox -``` - -> Note: All tests are automatically run with each push on Github on all supported Python versions using tox, so ensure all tests pass, or your PR will not be accepted. - - -## Building Documentation -```bash -pip install mkdocs-material -mkdocs serve # Local preview -mkdocs build # Build the static site -``` \ No newline at end of file diff --git a/docs/fetching/choosing.md b/docs/fetching/choosing.md index 67cfcb4..50ddf37 100644 --- a/docs/fetching/choosing.md +++ b/docs/fetching/choosing.md @@ -1,7 +1,7 @@ ## Introduction -Fetchers are classes that can do requests or fetch pages for you easily in a single-line fashion with many features and then return a [Response](#response-object) object. Starting with v0.3, all fetchers have other classes to keep the session running, so for example, a fetcher that uses a browser will keep the browser open till you finish all your requests through it instead of opening multiple browsers. So it depends on your use case. +Fetchers are classes that can do requests or fetch pages for you easily in a single-line fashion with many features and then return a [Response](#response-object) object. Starting with v0.3, all fetchers have separate classes to keep the session running, so for example, a fetcher that uses a browser will keep the browser open till you finish all your requests through it instead of opening multiple browsers. So it depends on your use case. -This feature was introduced because, before v0.2, Scrapling was only a parsing engine; therefore, we wanted to gradually transition to become your one-stop shop for all Web Scraping needs. +This feature was introduced because, before v0.2, Scrapling was only a parsing engine. The target here is to gradually become the one-stop shop for all Web Scraping needs. > Fetchers are not wrappers built on top of other libraries. However, they utilize these libraries as an engine to request/fetch pages easily for you, while fully leveraging that engine and adding features for you. Some fetchers don't even use the official library for requests; instead, they use their own custom version. For example, `StealthyFetcher` utilizes `Camoufox` browser directly, without relying on its Python library for anything except launch options. This last part might change soon as well. @@ -38,13 +38,13 @@ Then you use it right away without initializing like this, and it will use the d If you want to configure the parser ([Selector class](../parsing/main_classes.md#selector)) that will be used on the response before returning it for you, then do this first: ```python >>> from scrapling.fetchers import Fetcher ->>> Fetcher.configure(adaptive=True, encoding="utf8", keep_comments=False, keep_cdata=False) # and the rest +>>> Fetcher.configure(adaptive=True, encoding="utf-8", keep_comments=False, keep_cdata=False) # and the rest ``` or ```python >>> from scrapling.fetchers import Fetcher >>> Fetcher.adaptive=True ->>> Fetcher.encoding="utf8" +>>> Fetcher.encoding="utf-8" >>> Fetcher.keep_comments=False >>> Fetcher.keep_cdata=False # and the rest ``` @@ -71,7 +71,7 @@ The `Response` object is the same as the [Selector](../parsing/main_classes.md#s >>> page.headers # Response headers >>> page.request_headers # Request headers >>> page.history # Response history of redirections, if any ->>> page.body # Raw response body +>>> page.body # Raw response body without any processing >>> page.encoding # Response encoding ``` All fetchers return the `Response` object. \ No newline at end of file diff --git a/docs/fetching/dynamic.md b/docs/fetching/dynamic.md index fbdb387..066d12c 100644 --- a/docs/fetching/dynamic.md +++ b/docs/fetching/dynamic.md @@ -4,6 +4,12 @@ Here, we will discuss the `DynamicFetcher` class (previously known as `PlayWrigh As we will explain later, to automate the page, you need some knowledge of [Playwright's Page API](https://playwright.dev/python/docs/api/class-page). +> 💡 **Prerequisites:** +> +> 1. You’ve completed or read the [Fetchers basics](../fetching/choosing.md) page to understand what the [Response object](../fetching/choosing.md#response-object) is and which fetcher to use. +> 2. You’ve completed or read the [Querying elements](../parsing/selection.md) page to understand how to find/extract elements from the [Selector](../parsing/main_classes.md#selector)/[Response](../fetching/choosing.md#response-object) object. +> 3. You’ve completed or read the [Main classes](../parsing/main_classes.md) page to know what properties/methods the [Response](../fetching/choosing.md#response-object) class is inheriting from the [Selector](../parsing/main_classes.md#selector) class. + ## Basic Usage You have one primary way to import this Fetcher, which is the same for all fetchers. @@ -125,6 +131,17 @@ page = DynamicFetcher.fetch( ) ``` +### Downloading Files + +```python +page = DynamicFetcher.fetch('https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/poster.png') + +with open(file='poster.png', mode='wb') as f: + f.write(page.body) +``` + +The `body` attribute of the `Response` object is a `bytes` object containing the response body in case of Non-HTML responses. + ### Browser Automation This is where your knowledge about [Playwright's Page API](https://playwright.dev/python/docs/api/class-page) comes into play. The function you pass here takes the page object from Playwright's API, performs the desired action, and then the fetcher continues. @@ -275,7 +292,7 @@ async def scrape_multiple_sites(): return pages ``` -You may have noticed the `max_pages` argument. This is a new argument that enables the fetcher to create a **rotating pool of Browser tabs**. Instead of using a single tab for all your requests, you set a limit on the maximum number of pages. With each request, the library will close all tabs that have finished their task and check if the number of the current tabs is lower than the maximum allowed number of pages/tabs, then: +You may have noticed the `max_pages` argument. This is a new argument that enables the fetcher to create a **rotating pool of Browser tabs**. Instead of using a single tab for all your requests, you set a limit on the maximum number of pages that can be displayed at once. With each request, the library will close all tabs that have finished their task and check if the number of the current tabs is lower than the maximum allowed number of pages/tabs, then: 1. If you are within the allowed range, the fetcher will create a new tab for you, and then all is as normal. 2. Otherwise, it will keep checking every subsecond if creating a new tab is allowed or not for 60 seconds, then raise `TimeoutError`. This can happen when the website you are fetching becomes unresponsive. @@ -301,4 +318,4 @@ Use DynamicFetcher when: - Need custom browser config - Want flexible stealth options -If you want more stealth and control without much config, check out the [StealthyFetcher](stealthy.md). +If you want more stealth and control without much config, check out the [StealthyFetcher](stealthy.md). \ No newline at end of file diff --git a/docs/fetching/static.md b/docs/fetching/static.md index 524e63e..43a2d0e 100644 --- a/docs/fetching/static.md +++ b/docs/fetching/static.md @@ -2,6 +2,12 @@ The `Fetcher` class provides rapid and lightweight HTTP requests using the high-performance `curl_cffi` library with a lot of stealth capabilities. +> 💡 **Prerequisites:** +> +> 1. You’ve completed or read the [Fetchers basics](../fetching/choosing.md) page to understand what the [Response object](../fetching/choosing.md#response-object) is and which fetcher to use. +> 2. You’ve completed or read the [Querying elements](../parsing/selection.md) page to understand how to find/extract elements from the [Selector](../parsing/main_classes.md#selector)/[Response](../fetching/choosing.md#response-object) object. +> 3. You’ve completed or read the [Main classes](../parsing/main_classes.md) page to know what properties/methods the [Response](../fetching/choosing.md#response-object) class is inheriting from the [Selector](../parsing/main_classes.md#selector) class. + ## Basic Usage You have one primary way to import this Fetcher, which is the same for all fetchers. @@ -263,6 +269,16 @@ def scrape_products(): return results ``` +### Downloading Files + +```python +from scrapling.fetchers import Fetcher + +page = Fetcher.get('https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/poster.png') +with open(file='poster.png', mode='wb') as f: + f.write(page.body) +``` + ### Pagination Handling ```python diff --git a/docs/fetching/stealthy.md b/docs/fetching/stealthy.md index ddf8410..d53e930 100644 --- a/docs/fetching/stealthy.md +++ b/docs/fetching/stealthy.md @@ -4,6 +4,12 @@ Here, we will discuss the `StealthyFetcher` class. This class is similar to [Dyn As with [DynamicFetcher](dynamic.md#introduction), you will need some knowledge about [Playwright's Page API](https://playwright.dev/python/docs/api/class-page) to automate the page, as we will explain later. +> 💡 **Prerequisites:** +> +> 1. You’ve completed or read the [Fetchers basics](../fetching/choosing.md) page to understand what the [Response object](../fetching/choosing.md#response-object) is and which fetcher to use. +> 2. You’ve completed or read the [Querying elements](../parsing/selection.md) page to understand how to find/extract elements from the [Selector](../parsing/main_classes.md#selector)/[Response](../fetching/choosing.md#response-object) object. +> 3. You’ve completed or read the [Main classes](../parsing/main_classes.md) page to know what properties/methods the [Response](../fetching/choosing.md#response-object) class is inheriting from the [Selector](../parsing/main_classes.md#selector) class. + ## Basic Usage You have one primary way to import this Fetcher, which is the same for all fetchers. @@ -146,6 +152,17 @@ page = StealthyFetcher.fetch( ) ``` +### Downloading Files + +```python +page = StealthyFetcher.fetch('https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/poster.png') + +with open(file='poster.png', mode='wb') as f: + f.write(page.body) +``` + +The `body` attribute of the `Response` object is a `bytes` object containing the response body in case of Non-HTML responses. + ### Browser Automation This is where your knowledge about [Playwright's Page API](https://playwright.dev/python/docs/api/class-page) comes into play. The function you pass here takes the page object from Playwright's API, performs the desired action, and then the fetcher continues. diff --git a/docs/index.md b/docs/index.md index ea89264..11a2c55 100644 --- a/docs/index.md +++ b/docs/index.md @@ -82,33 +82,10 @@ Scrapling’s GitHub stars have grown steadily since its release (see chart belo - ## Installation Scrapling requires Python 3.10 or higher: diff --git a/docs/parsing/adaptive.md b/docs/parsing/adaptive.md index 0a215fd..6ba9288 100644 --- a/docs/parsing/adaptive.md +++ b/docs/parsing/adaptive.md @@ -1,4 +1,11 @@ ## Introduction + +> 💡 **Prerequisites:** +> +> 1. You’ve completed or read the [Querying elements](../parsing/selection.md) page to understand how to find/extract elements from the [Selector](../parsing/main_classes.md#selector) object. +> 2. You’ve completed or read the [Main classes](../parsing/main_classes.md) page to understand the [Selector](../parsing/main_classes.md#selector) class. +>

+ Adaptive scraping (previously known as automatch) is one of Scrapling's most powerful features. It allows your scraper to survive website changes by intelligently tracking and relocating elements. Let's say you are scraping a page with a structure like this: diff --git a/docs/parsing/main_classes.md b/docs/parsing/main_classes.md index 9d4260d..fb1ccf0 100644 --- a/docs/parsing/main_classes.md +++ b/docs/parsing/main_classes.md @@ -1,7 +1,13 @@ ## Introduction -After exploring the various ways to select elements with Scrapling and related features, let's take a step back and examine the [Selector](#selector) class generally and other objects to better understand the parsing engine. -The [Selector](#selector) class is the core parsing engine in Scrapling that provides HTML parsing and element selection capabilities. You can always import it with any of the following imports +> 💡 **Prerequisites:** +> +> - You’ve completed or read the [Querying elements](../parsing/selection.md) page to understand how to find/extract elements from the [Selector](../parsing/main_classes.md#selector) object. +>

+ +After exploring the various ways to select elements with Scrapling and its related features, let's take a step back and examine the [Selector](#selector) class in general, as well as other objects, to gain a better understanding of the parsing engine. + +The [Selector](#selector) class is the core parsing engine in Scrapling, providing HTML parsing and element selection capabilities. You can always import it with any of the following imports ```python from scrapling import Selector from scrapling.parser import Selector @@ -133,7 +139,7 @@ Getting the attributes of the element >>> print(article.attrib) {'class': 'product', 'data-id': '1'} ``` -Access a specific attribute with any method of the following +Access a specific attribute with any of the following ```python >>> article.attrib['class'] >>> article.attrib.get('class') @@ -151,14 +157,16 @@ Get the HTML content of the element ``` Get the prettified version of the element's HTML content ```python ->>> print(article.prettify()) +print(article.prettify()) +``` +```html

Product 1

This is product 1

$10.99
``` -Use `.body` property to get the raw content of page +Use the `.body` property to get the raw content of the page ```python >>> page.body '\n \n Some page\n \n \n
\n
\n

Product 1

\n

This is product 1

\n $10.99\n \n
\n\n
\n

Product 2

\n

This is product 2

\n $20.99\n \n
\n\n
\n

Product 3

\n

This is product 3

\n $15.99\n \n
\n
\n\n \n \n' @@ -192,7 +200,7 @@ If you are unfamiliar with the DOM tree or the tree data structure in general, t If you are too lazy to search about it, here's a quick explanation to give you a good idea.
In simple words, the `html` element is the root of the website's tree, as every page starts with an `html` element.
-This element will be directly above elements like `head` and `body`. These are considered "children" of the `html` element, and the `html` element is considered their "parent." The element `body` is a "sibling" of the element `head` and vice versa. +This element will be positioned directly above elements such as `head` and `body`. These are considered "children" of the `html` element, and the `html` element is considered their "parent". The element `body` is a "sibling" of the element `head` and vice versa. Accessing the parent of an element ```python @@ -302,7 +310,7 @@ In the [Selector](#selector) class, all methods/properties that should return a Let's see what [Selectors](#selectors) class adds to the table with that out of the way. ### Properties -Apart from the normal operations on Python lists like iteration, slicing, etc... +Apart from the normal operations on Python lists, such as iteration and slicing, etc. You can do the following: @@ -326,9 +334,9 @@ Execute CSS and XPath selectors directly on the [Selector](#selector) instances , ...] ``` -Run the `re` and `re_first` methods directly. They take the same arguments passed to the [Selector](#selector) class. I'm still leaving these methods to be explained in the [TextHandler](#texthandler) section below. +Run the `re` and `re_first` methods directly. They take the same arguments passed to the [Selector](#selector) class. I will still leave these methods to be explained in the [TextHandler](#texthandler) section below. -However, in this class, the `re_first` behaves differently as it runs `re` on each [Selector](#selector) within and returns the first one with a result. The `re` method will return a [TextHandlers](#texthandlers) object as normal, that has all the [TextHandler](#texthandler) instances combined in one [TextHandlers](#texthandlers) instance. +However, in this class, the `re_first` behaves differently as it runs `re` on each [Selector](#selector) within and returns the first one with a result. The `re` method will return a [TextHandlers](#texthandlers) object as normal, which combines all the [TextHandler](#texthandler) instances into one [TextHandlers](#texthandlers) instance. ```python >>> page.css('.price_color').re(r'[\d\.]+') ['51.77', @@ -381,15 +389,15 @@ Of course, TextHandler provides extra methods and properties that standard Pytho ### Usage First, before discussing the added methods, you need to know that all operations on it, like slicing, accessing by index, etc., and methods like `split`, `replace`, `strip`, etc., all return a `TextHandler` again, so you can chain them as you want. If you find a method or property that returns a standard string instead of `TextHandler`, please open an issue, and we will override it as well. -First, we start with the `re` and `re_first` methods. These are the same methods that exist in the rest of the classes ([Selector](#selector), [Selectors](#selectors), and [TextHandlers](#texthandlers)), so they will take the same arguments as well. +First, we start with the `re` and `re_first` methods. These are the same methods that exist in the other classes ([Selector](#selector), [Selectors](#selectors), and [TextHandlers](#texthandlers)), so they will accept the same arguments as well. - The `re` method takes a string/compiled regex pattern as the first argument. It searches the data for all strings matching the regex and returns them as a [TextHandlers](#texthandlers) instance. The `re_first` method takes the same arguments and behaves similarly, but as you probably figured out from the naming, it returns the first result only as a `TextHandler` instance. Also, it takes other helpful arguments, which are: - **replace_entities**: This is enabled by default. It replaces character entity references with their corresponding characters. - - **clean_match**: It's disabled by default. This makes the method ignore all whitespaces and consecutive spaces while matching. - - **case_sensitive**: It's enabled by default. As the name implies, disabling it will make the regex ignore the case of letters while compiling it. + - **clean_match**: It's disabled by default. This causes the method to ignore all whitespace and consecutive spaces while matching. + - **case_sensitive**: It's enabled by default. As the name implies, disabling it will cause the regex to ignore the case of letters while compiling. You have seen these examples before; the return result is [TextHandlers](#texthandlers) because we used the `re` method. ```python @@ -484,7 +492,7 @@ First, we start with the `re` and `re_first` methods. These are the same methods >>> page.json() {'some_key': 'some_value'} ``` - You might wonder how this happened while the `html` tag doesn't have direct text?
+ You might wonder how this happened, given that the `html` tag doesn't contain direct text.
Well, for cases like JSON responses, I made the [Selector](#selector) class maintain a raw copy of the content passed to it. This way, when you use the `.json()` method, it checks for that raw copy and then converts it to JSON. If the raw copy is not available like the case with the elements, it checks for the current element text content, or otherwise it used the `get_all_text` method directly.

This might sound hacky a bit but remember, Scrapling is currently optimized to work with HTML pages only so that's the best way till now to handle JSON responses currently without sacrificing speed. This will be changed in the upcoming versions. - Another handy method is `.clean()`, which will remove all white spaces and consecutive spaces for you and return a new `TextHandler` instance @@ -492,6 +500,7 @@ First, we start with the `re` and `re_first` methods. These are the same methods >>> TextHandler('\n wonderful idea, \reh?').clean() 'wonderful idea, eh?' ``` +Also, you can pass `remove_entities` argument to make `clean` replace HTML entities with their corresponding characters. - Another method that might be helpful in some cases is the `.sort()` method to sort the string for you, as you do with lists ```python @@ -509,10 +518,10 @@ Other methods and properties will be added over time, but remember that this cla ## TextHandlers You probably guessed it: This class is similar to [Selectors](#selectors) and [Selector](#selector), but here it inherits the same logic and method as standard lists, with only `re` and `re_first` as new methods. -The only difference is that the `re_first` method logic here does `re` on each [TextHandler](#texthandler) within and returns the first result it has or `None`. Nothing is new to explain here, but new methods will be added over time. +The only difference is that the `re_first` method logic here does `re` on each [TextHandler](#texthandler) within and returns the first result it has or `None`. Nothing new needs to be explained here, but new methods will be added over time. ## AttributesHandler -This is a read-only version of Python's standard dictionary or `dict` that's only used to store the attributes of each element or each [Selector](#selector) instance, in other words. +This is a read-only version of Python's standard dictionary, or `dict`, that is used solely to store the attributes of each element or each [Selector](#selector) instance. ```python >>> print(page.find('script').attrib) {'id': 'page-data', 'type': 'application/json'} @@ -525,7 +534,7 @@ It currently adds two extra simple methods: - The `search_values` method - In standard dictionaries, you can do `dict.get("key_name")` to check if a key exists. However, if you want to search by values instead of keys, it will take you some code lines. This method does that for you. It allows you to search the current attributes by values and returns a dictionary of each matching item. + In standard dictionaries, you can do `dict.get("key_name")` to check if a key exists. However, if you want to search by values instead of keys, it will require some additional code lines. This method does that for you. It allows you to search the current attributes by values and returns a dictionary of each matching item. A simple example would be ```python @@ -552,8 +561,9 @@ It currently adds two extra simple methods: - The `json_string` property - This property converts current attributes to a JSON string if the attributes are JSON serializable; otherwise, it throws an error - ```python + This property converts current attributes to a JSON string if the attributes are JSON serializable; otherwise, it throws an error + + ```python >>>page.find('script').attrib.json_string - b'{"id":"page-data","type":"application/json"}' - ``` \ No newline at end of file + b'{"id":"page-data","type":"application/json"}' + ``` \ No newline at end of file diff --git a/mkdocs.yml b/mkdocs.yml index 6a16483..ea2dd7a 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -12,24 +12,9 @@ theme: logo: assets/logo.png favicon: assets/favicon.ico palette: - - media: "(prefers-color-scheme)" - toggle: - icon: material/link - name: Switch to light mode - - media: "(prefers-color-scheme: light)" - scheme: default - primary: indigo - accent: indigo - toggle: - icon: material/toggle-switch - name: Switch to dark mode - - media: "(prefers-color-scheme: dark)" - scheme: slate - primary: black - accent: indigo - toggle: - icon: material/toggle-switch-off - name: Switch to system preference + scheme: slate + primary: black + accent: deep purple font: text: Open Sans code: JetBrains Mono @@ -70,10 +55,10 @@ nav: - Main classes: parsing/main_classes.md - Adaptive scraping: parsing/adaptive.md - Fetching: - - Choosing a fetcher: fetching/choosing.md - - Static requests: fetching/static.md - - Dynamically loaded websites: fetching/dynamic.md - - Fully bypass protections while fetching: fetching/stealthy.md + - Fetchers basics: fetching/choosing.md + - HTTP requests: fetching/static.md + - Dynamic websites: fetching/dynamic.md + - Dynamic websites with hard protections: fetching/stealthy.md - Command Line Interface: - Overview: cli/overview.md - Interactive shell: cli/interactive-shell.md @@ -94,7 +79,7 @@ nav: - Writing your retrieval system: development/adaptive_storage_system.md - Using Scrapling's custom types: development/scrapling_custom_types.md - Support and Advertisement: donate.md - - Contributing: contributing.md + - Contributing: 'https://github.com/D4Vinci/Scrapling/blob/main/CONTRIBUTING.md' - Changelog: 'https://github.com/D4Vinci/Scrapling/releases' markdown_extensions: @@ -118,10 +103,10 @@ markdown_extensions: plugins: - search - - social: - cards_layout_options: - background_color: "#1f1f1f" - font_family: Roboto +# - social: +# cards_layout_options: +# background_color: "#1f1f1f" +# font_family: Roboto - mkdocstrings: handlers: python: diff --git a/pyproject.toml b/pyproject.toml index 2dbbb92..558bf55 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,9 +5,9 @@ build-backend = "setuptools.build_meta" [project] name = "scrapling" # Static version instead of a dynamic version so we can get better layer caching while building docker, check the docker file to understand -version = "0.3.8" +version = "0.3.9" description = "Scrapling is an undetectable, powerful, flexible, high-performance Python library that makes Web Scraping easy and effortless as it should be!" -readme = {file = "README.md", content-type = "text/markdown"} +readme = {file = "docs/README.md", content-type = "text/markdown"} license = {file = "LICENSE"} authors = [ {name = "Karim Shoair", email = "karim.shoair@pm.me"} diff --git a/pytest.ini b/pytest.ini index cb0da7d..083e39a 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,5 +1,5 @@ [pytest] -asyncio_mode = auto +asyncio_mode = strict asyncio_default_fixture_loop_scope = function addopts = -p no:warnings --doctest-modules --ignore=setup.py --verbose markers = diff --git a/scrapling/__init__.py b/scrapling/__init__.py index 2122396..bbbb02b 100644 --- a/scrapling/__init__.py +++ b/scrapling/__init__.py @@ -1,5 +1,5 @@ __author__ = "Karim Shoair (karim.shoair@pm.me)" -__version__ = "0.3.8" +__version__ = "0.3.9" __copyright__ = "Copyright (c) 2024 Karim Shoair" from typing import Any, TYPE_CHECKING diff --git a/scrapling/cli.py b/scrapling/cli.py index 25fdce3..22f2dd1 100644 --- a/scrapling/cli.py +++ b/scrapling/cli.py @@ -2,6 +2,8 @@ from pathlib import Path from subprocess import check_output from sys import executable as python_executable +from curl_cffi.requests import impersonate + from scrapling.core.utils import log from scrapling.engines.toolbelt.custom import Response from scrapling.core.utils._shell import _CookieParser, _ParseHeaders @@ -84,7 +86,7 @@ def __BuildRequest(headers: List[str], cookies: str, params: str, json: Optional # Parse parameters parsed_headers, parsed_cookies, parsed_params, parsed_json = __ParseExtractArguments(headers, cookies, params, json) # Build request arguments - request_kwargs = { + request_kwargs: Dict[str, Any] = { "headers": parsed_headers if parsed_headers else None, "cookies": parsed_cookies if parsed_cookies else None, } @@ -95,6 +97,10 @@ def __BuildRequest(headers: List[str], cookies: str, params: str, json: Optional if "proxy" in kwargs: request_kwargs["proxy"] = kwargs.pop("proxy") + # Parse impersonate parameter if it contains commas (for random selection) + if "impersonate" in kwargs and "," in (kwargs.get("impersonate") or ""): + kwargs["impersonate"] = [browser.strip() for browser in kwargs["impersonate"].split(",")] + return {**request_kwargs, **kwargs} @@ -225,7 +231,10 @@ def extract(): default=True, help="Whether to verify SSL certificates (default: True)", ) -@option("--impersonate", help="Browser to impersonate (e.g., chrome, firefox).") +@option( + "--impersonate", + help="Browser to impersonate. Can be a single browser (e.g., chrome) or comma-separated list for random selection (e.g., chrome,firefox,safari).", +) @option( "--stealthy-headers/--no-stealthy-headers", default=True, @@ -318,7 +327,10 @@ def get( default=True, help="Whether to verify SSL certificates (default: True)", ) -@option("--impersonate", help="Browser to impersonate (e.g., chrome, firefox).") +@option( + "--impersonate", + help="Browser to impersonate. Can be a single browser (e.g., chrome) or comma-separated list for random selection (e.g., chrome,firefox,safari).", +) @option( "--stealthy-headers/--no-stealthy-headers", default=True, @@ -412,7 +424,10 @@ def post( default=True, help="Whether to verify SSL certificates (default: True)", ) -@option("--impersonate", help="Browser to impersonate (e.g., chrome, firefox).") +@option( + "--impersonate", + help="Browser to impersonate. Can be a single browser (e.g., chrome) or comma-separated list for random selection (e.g., chrome,firefox,safari).", +) @option( "--stealthy-headers/--no-stealthy-headers", default=True, @@ -504,7 +519,10 @@ def put( default=True, help="Whether to verify SSL certificates (default: True)", ) -@option("--impersonate", help="Browser to impersonate (e.g., chrome, firefox).") +@option( + "--impersonate", + help="Browser to impersonate. Can be a single browser (e.g., chrome) or comma-separated list for random selection (e.g., chrome,firefox,safari).", +) @option( "--stealthy-headers/--no-stealthy-headers", default=True, diff --git a/scrapling/core/ai.py b/scrapling/core/ai.py index d497e57..4d5277a 100644 --- a/scrapling/core/ai.py +++ b/scrapling/core/ai.py @@ -5,6 +5,7 @@ from pydantic import BaseModel, Field from scrapling.core.shell import Convertor from scrapling.engines.toolbelt.custom import Response as _ScraplingResponse +from scrapling.engines.static import ImpersonateType from scrapling.fetchers import ( Fetcher, FetcherSession, @@ -24,9 +25,6 @@ from scrapling.core._types import ( SelectorWaitStates, Generator, ) -from curl_cffi.requests import ( - BrowserTypeLiteral, -) class ResponseModel(BaseModel): @@ -46,7 +44,7 @@ class ScraplingMCPServer: @staticmethod def get( url: str, - impersonate: Optional[BrowserTypeLiteral] = "chrome", + impersonate: ImpersonateType = "chrome", extraction_type: extraction_types = "markdown", css_selector: Optional[str] = None, main_content_only: bool = True, @@ -124,7 +122,7 @@ class ScraplingMCPServer: @staticmethod async def bulk_get( urls: Tuple[str, ...], - impersonate: Optional[BrowserTypeLiteral] = "chrome", + impersonate: ImpersonateType = "chrome", extraction_type: extraction_types = "markdown", css_selector: Optional[str] = None, main_content_only: bool = True, diff --git a/scrapling/core/custom_types.py b/scrapling/core/custom_types.py index 3675372..d437a23 100644 --- a/scrapling/core/custom_types.py +++ b/scrapling/core/custom_types.py @@ -103,9 +103,11 @@ class TextHandler(str): """Return a sorted version of the string""" return self.__class__("".join(sorted(self, reverse=reverse))) - def clean(self) -> Union[str, "TextHandler"]: + def clean(self, remove_entities=False) -> Union[str, "TextHandler"]: """Return a new version of the string after removing all white spaces and consecutive spaces""" data = self.translate(__CLEANING_TABLE__) + if remove_entities: + data = _replace_entities(data) return self.__class__(__CONSECUTIVE_SPACES_REGEX__.sub(" ", data).strip()) # For easy copy-paste from Scrapy/parsel code when needed :) diff --git a/scrapling/engines/_browsers/_camoufox.py b/scrapling/engines/_browsers/_camoufox.py index 4ac0558..8d70dbd 100644 --- a/scrapling/engines/_browsers/_camoufox.py +++ b/scrapling/engines/_browsers/_camoufox.py @@ -343,7 +343,7 @@ class StealthySession(StealthySessionMixin, SyncSession): page_info.page.wait_for_timeout(params.wait) response = ResponseFactory.from_playwright_response( - page_info.page, first_response, final_response[0], params.selector_config, bool(params.page_action) + page_info.page, first_response, final_response[0], params.selector_config ) # Close the page to free up resources @@ -636,7 +636,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession): # Create response object response = await ResponseFactory.from_async_playwright_response( - page_info.page, first_response, final_response[0], params.selector_config, bool(params.page_action) + page_info.page, first_response, final_response[0], params.selector_config ) # Close the page to free up resources diff --git a/scrapling/engines/_browsers/_controllers.py b/scrapling/engines/_browsers/_controllers.py index bcfd0ac..9fc3490 100644 --- a/scrapling/engines/_browsers/_controllers.py +++ b/scrapling/engines/_browsers/_controllers.py @@ -268,7 +268,7 @@ class DynamicSession(DynamicSessionMixin, SyncSession): # Create response object response = ResponseFactory.from_playwright_response( - page_info.page, first_response, final_response[0], params.selector_config, bool(params.page_action) + page_info.page, first_response, final_response[0], params.selector_config ) # Close the page to free up resources @@ -492,7 +492,7 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession): # Create response object response = await ResponseFactory.from_async_playwright_response( - page_info.page, first_response, final_response[0], params.selector_config, bool(params.page_action) + page_info.page, first_response, final_response[0], params.selector_config ) # Close the page to free up resources diff --git a/scrapling/engines/static.py b/scrapling/engines/static.py index dd54168..6f68737 100644 --- a/scrapling/engines/static.py +++ b/scrapling/engines/static.py @@ -1,4 +1,5 @@ from abc import ABC +from random import choice from time import sleep as time_sleep from asyncio import sleep as asyncio_sleep @@ -31,12 +32,29 @@ from .toolbelt.fingerprints import generate_convincing_referer, generate_headers _UNSET: Any = object() _NO_SESSION: Any = object() +# Type alias for `impersonate` parameter - accepts a single browser or list of browsers +ImpersonateType = BrowserTypeLiteral | List[BrowserTypeLiteral] | None + + +def _select_random_browser(impersonate: ImpersonateType) -> Optional[BrowserTypeLiteral]: + """ + Handle browser selection logic for the ` impersonate ` parameter. + + If impersonate is a list, randomly select one browser from it. + If it's a string or None, return as is. + """ + if isinstance(impersonate, list): + if not impersonate: + return None + return choice(impersonate) + return impersonate + class _ConfigurationLogic(ABC): # Core Logic Handler (Internal Engine) def __init__( self, - impersonate: Optional[BrowserTypeLiteral] = "chrome", + impersonate: ImpersonateType = "chrome", http3: Optional[bool] = False, stealthy_headers: Optional[bool] = True, proxies: Optional[Dict[str, str]] = None, @@ -76,7 +94,9 @@ class _ConfigurationLogic(ABC): def _merge_request_args(self, **method_kwargs) -> Dict[str, Any]: """Merge request-specific arguments with default session arguments.""" url = method_kwargs.pop("url") - impersonate = self._get_with_precedence(method_kwargs.pop("impersonate"), self._default_impersonate) + impersonate = _select_random_browser( + self._get_with_precedence(method_kwargs.pop("impersonate"), self._default_impersonate) + ) http3_enabled = self._get_with_precedence(method_kwargs.pop("http3"), self._default_http3) final_args = { "url": url, @@ -146,7 +166,7 @@ class _ConfigurationLogic(ABC): class _SyncSessionLogic(_ConfigurationLogic): def __init__( self, - impersonate: Optional[BrowserTypeLiteral] = "chrome", + impersonate: ImpersonateType = "chrome", http3: Optional[bool] = False, stealthy_headers: Optional[bool] = True, proxies: Optional[Dict[str, str]] = None, @@ -261,7 +281,7 @@ class _SyncSessionLogic(_ConfigurationLogic): auth: Optional[Tuple[str, str]] = None, verify: Optional[bool] = _UNSET, cert: Optional[str | Tuple[str, str]] = _UNSET, - impersonate: Optional[BrowserTypeLiteral] = _UNSET, + impersonate: ImpersonateType = _UNSET, http3: Optional[bool] = _UNSET, stealthy_headers: Optional[bool] = _UNSET, **kwargs, @@ -334,7 +354,7 @@ class _SyncSessionLogic(_ConfigurationLogic): auth: Optional[Tuple[str, str]] = None, verify: Optional[bool] = _UNSET, cert: Optional[str | Tuple[str, str]] = _UNSET, - impersonate: Optional[BrowserTypeLiteral] = _UNSET, + impersonate: ImpersonateType = _UNSET, http3: Optional[bool] = _UNSET, stealthy_headers: Optional[bool] = _UNSET, **kwargs, @@ -411,7 +431,7 @@ class _SyncSessionLogic(_ConfigurationLogic): auth: Optional[Tuple[str, str]] = None, verify: Optional[bool] = _UNSET, cert: Optional[str | Tuple[str, str]] = _UNSET, - impersonate: Optional[BrowserTypeLiteral] = _UNSET, + impersonate: ImpersonateType = _UNSET, http3: Optional[bool] = _UNSET, stealthy_headers: Optional[bool] = _UNSET, **kwargs, @@ -488,7 +508,7 @@ class _SyncSessionLogic(_ConfigurationLogic): auth: Optional[Tuple[str, str]] = None, verify: Optional[bool] = _UNSET, cert: Optional[str | Tuple[str, str]] = _UNSET, - impersonate: Optional[BrowserTypeLiteral] = _UNSET, + impersonate: ImpersonateType = _UNSET, http3: Optional[bool] = _UNSET, stealthy_headers: Optional[bool] = _UNSET, **kwargs, @@ -552,7 +572,7 @@ class _SyncSessionLogic(_ConfigurationLogic): class _ASyncSessionLogic(_ConfigurationLogic): def __init__( self, - impersonate: Optional[BrowserTypeLiteral] = "chrome", + impersonate: ImpersonateType = "chrome", http3: Optional[bool] = False, stealthy_headers: Optional[bool] = True, proxies: Optional[Dict[str, str]] = None, @@ -669,7 +689,7 @@ class _ASyncSessionLogic(_ConfigurationLogic): auth: Optional[Tuple[str, str]] = None, verify: Optional[bool] = _UNSET, cert: Optional[str | Tuple[str, str]] = _UNSET, - impersonate: Optional[BrowserTypeLiteral] = _UNSET, + impersonate: ImpersonateType = _UNSET, http3: Optional[bool] = _UNSET, stealthy_headers: Optional[bool] = _UNSET, **kwargs, @@ -742,7 +762,7 @@ class _ASyncSessionLogic(_ConfigurationLogic): auth: Optional[Tuple[str, str]] = None, verify: Optional[bool] = _UNSET, cert: Optional[str | Tuple[str, str]] = _UNSET, - impersonate: Optional[BrowserTypeLiteral] = _UNSET, + impersonate: ImpersonateType = _UNSET, http3: Optional[bool] = _UNSET, stealthy_headers: Optional[bool] = _UNSET, **kwargs, @@ -819,7 +839,7 @@ class _ASyncSessionLogic(_ConfigurationLogic): auth: Optional[Tuple[str, str]] = None, verify: Optional[bool] = _UNSET, cert: Optional[str | Tuple[str, str]] = _UNSET, - impersonate: Optional[BrowserTypeLiteral] = _UNSET, + impersonate: ImpersonateType = _UNSET, http3: Optional[bool] = _UNSET, stealthy_headers: Optional[bool] = _UNSET, **kwargs, @@ -896,7 +916,7 @@ class _ASyncSessionLogic(_ConfigurationLogic): auth: Optional[Tuple[str, str]] = None, verify: Optional[bool] = _UNSET, cert: Optional[str | Tuple[str, str]] = _UNSET, - impersonate: Optional[BrowserTypeLiteral] = _UNSET, + impersonate: ImpersonateType = _UNSET, http3: Optional[bool] = _UNSET, stealthy_headers: Optional[bool] = _UNSET, **kwargs, @@ -970,7 +990,7 @@ class FetcherSession: def __init__( self, - impersonate: Optional[BrowserTypeLiteral] = "chrome", + impersonate: ImpersonateType = "chrome", http3: Optional[bool] = False, stealthy_headers: Optional[bool] = True, proxies: Optional[Dict[str, str]] = None, @@ -987,7 +1007,7 @@ class FetcherSession: selector_config: Optional[Dict] = None, ): """ - :param impersonate: Browser version to impersonate. Automatically defaults to the latest available Chrome version. + :param impersonate: Browser version to impersonate. Can be a single browser string or a list of browser strings for random selection. (Default: latest available Chrome version) :param http3: Whether to use HTTP3. Defaults to False. It might be problematic if used it with `impersonate`. :param stealthy_headers: If enabled (default), it creates and adds real browser headers. It also sets the referer header as if this request came from a Google search of URL's domain. :param proxies: Dict of proxies to use. Format: {"http": proxy_url, "https": proxy_url}. @@ -1004,7 +1024,7 @@ class FetcherSession: :param cert: Tuple of (cert, key) filenames for the client certificate. :param selector_config: Arguments passed when creating the final Selector class. """ - self._default_impersonate: Optional[BrowserTypeLiteral] = impersonate + self._default_impersonate: ImpersonateType = impersonate self._stealth = stealthy_headers self._default_proxies = proxies or {} self._default_proxy = proxy or None diff --git a/scrapling/engines/toolbelt/convertor.py b/scrapling/engines/toolbelt/convertor.py index 5bb4a49..ef28acf 100644 --- a/scrapling/engines/toolbelt/convertor.py +++ b/scrapling/engines/toolbelt/convertor.py @@ -85,7 +85,6 @@ class ResponseFactory: first_response: SyncResponse, final_response: Optional[SyncResponse], parser_arguments: Dict, - automated_page: bool = False, ) -> Response: """ Transforms a Playwright response into an internal `Response` object, encapsulating @@ -101,7 +100,6 @@ class ResponseFactory: :param first_response: An earlier or initial Playwright `Response` object that may serve as a fallback response in the absence of the final one. :param parser_arguments: A dictionary containing additional arguments needed for parsing or further customization of the returned `Response`. These arguments are dynamically unpacked into the `Response` object. - :param automated_page: If True, it means the `page_action` argument was being used, so the response retrieving method changes to use Playwright's page instead of the final response. :return: A fully populated `Response` object containing the page's URL, content, status, headers, cookies, and other derived metadata. :rtype: Response @@ -117,7 +115,10 @@ class ResponseFactory: history = cls._process_response_history(first_response, parser_arguments) try: - page_content = final_response.text() if not automated_page else cls._get_page_content(page) + if "html" in final_response.all_headers().get("content-type", ""): + page_content = cls._get_page_content(page) + else: + page_content = final_response.body() except Exception as e: # pragma: no cover log.error(f"Error getting page content: {e}") page_content = "" @@ -219,7 +220,6 @@ class ResponseFactory: first_response: AsyncResponse, final_response: Optional[AsyncResponse], parser_arguments: Dict, - automated_page: bool = False, ) -> Response: """ Transforms a Playwright response into an internal `Response` object, encapsulating @@ -235,7 +235,6 @@ class ResponseFactory: :param first_response: An earlier or initial Playwright `Response` object that may serve as a fallback response in the absence of the final one. :param parser_arguments: A dictionary containing additional arguments needed for parsing or further customization of the returned `Response`. These arguments are dynamically unpacked into the `Response` object. - :param automated_page: If True, it means the `page_action` argument was being used, so the response retrieving method changes to use Playwright's page instead of the final response. :return: A fully populated `Response` object containing the page's URL, content, status, headers, cookies, and other derived metadata. :rtype: Response @@ -251,7 +250,10 @@ class ResponseFactory: history = await cls._async_process_response_history(first_response, parser_arguments) try: - page_content = await (final_response.text() if not automated_page else cls._get_async_page_content(page)) + if "html" in (await final_response.all_headers()).get("content-type", ""): + page_content = await cls._get_async_page_content(page) + else: + page_content = await final_response.body() except Exception as e: # pragma: no cover log.error(f"Error getting page content in async: {e}") page_content = "" diff --git a/scrapling/engines/toolbelt/custom.py b/scrapling/engines/toolbelt/custom.py index 1f20038..d1f95b7 100644 --- a/scrapling/engines/toolbelt/custom.py +++ b/scrapling/engines/toolbelt/custom.py @@ -18,7 +18,7 @@ from scrapling.parser import Selector, SQLiteStorageSystem class Response(Selector): - """This class is returned by all engines as a way to unify response type between different libraries.""" + """This class is returned by all engines as a way to unify the response type between different libraries.""" def __init__( self, @@ -132,7 +132,7 @@ class BaseFetcher: class StatusText: - """A class that gets the status text of response status code. + """A class that gets the status text of the response status code. Reference: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status """ diff --git a/scrapling/parser.py b/scrapling/parser.py index 6a4934c..ac97b7d 100644 --- a/scrapling/parser.py +++ b/scrapling/parser.py @@ -220,16 +220,6 @@ class Selector(SelectorsGeneration): # Faster than checking `element.is_attribute or element.is_text or element.is_tail` return issubclass(type(element), _ElementUnicodeResult) - @staticmethod - def __content_convertor( - element: HtmlElement | _ElementUnicodeResult, - ) -> TextHandler: - """Used internally to convert a single element's text content to TextHandler directly without checks - - This single line has been isolated like this, so when it's used with `map` we get that slight performance boost vs. list comprehension - """ - return TextHandler(element) - def __element_convertor(self, element: HtmlElement) -> "Selector": """Used internally to convert a single HtmlElement to Selector directly without checks""" db_instance = self._storage if (hasattr(self, "_storage") and self._storage) else None @@ -248,18 +238,6 @@ class Selector(SelectorsGeneration): def __elements_convertor(self, elements: List[HtmlElement]) -> "Selectors": return Selectors(map(self.__element_convertor, elements)) - def __handle_element( - self, element: Optional[HtmlElement | _ElementUnicodeResult] - ) -> Optional[Union[TextHandler, "Selector"]]: - """Used internally in all functions to convert a single element to type (Selector|TextHandler) when possible""" - if element is None: - return None - elif self._is_text_node(element): - # `_ElementUnicodeResult` basically inherit from `str` so it's fine - return self.__content_convertor(element) - else: - return self.__element_convertor(element) - def __handle_elements( self, result: List[HtmlElement | _ElementUnicodeResult] ) -> Union["Selectors", "TextHandlers"]: diff --git a/setup.cfg b/setup.cfg index fddbe8f..54350cc 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = scrapling -version = 0.3.8 +version = 0.3.9 author = Karim Shoair author_email = karim.shoair@pm.me description = Scrapling is an undetectable, powerful, flexible, high-performance Python library that makes Web Scraping easy and effortless as it should be! diff --git a/tests/cli/test_cli.py b/tests/cli/test_cli.py index 1f98bfb..4a79c25 100644 --- a/tests/cli/test_cli.py +++ b/tests/cli/test_cli.py @@ -193,3 +193,51 @@ class TestCLI: [html_url, 'output.invalid'] ) # Should handle the error gracefully + + def test_impersonate_comma_separated(self, runner, tmp_path, html_url): + """Test that comma-separated impersonate values are parsed correctly""" + output_file = tmp_path / "output.md" + + with patch('scrapling.fetchers.Fetcher.get') as mock_get: + mock_response = configure_selector_mock() + mock_response.status = 200 + mock_get.return_value = mock_response + + result = runner.invoke( + get, + [ + html_url, + str(output_file), + '--impersonate', 'chrome,firefox,safari' + ] + ) + assert result.exit_code == 0 + + # Verify that the impersonate argument was converted to a list + call_kwargs = mock_get.call_args[1] + assert isinstance(call_kwargs['impersonate'], list) + assert call_kwargs['impersonate'] == ['chrome', 'firefox', 'safari'] + + def test_impersonate_single_browser(self, runner, tmp_path, html_url): + """Test that single impersonate value remains as string""" + output_file = tmp_path / "output.md" + + with patch('scrapling.fetchers.Fetcher.get') as mock_get: + mock_response = configure_selector_mock() + mock_response.status = 200 + mock_get.return_value = mock_response + + result = runner.invoke( + get, + [ + html_url, + str(output_file), + '--impersonate', 'chrome' + ] + ) + assert result.exit_code == 0 + + # Verify that the impersonate argument remains a string + call_kwargs = mock_get.call_args[1] + assert isinstance(call_kwargs['impersonate'], str) + assert call_kwargs['impersonate'] == 'chrome' diff --git a/tests/fetchers/test_impersonate_list.py b/tests/fetchers/test_impersonate_list.py new file mode 100644 index 0000000..d79a06b --- /dev/null +++ b/tests/fetchers/test_impersonate_list.py @@ -0,0 +1,153 @@ +"""Test suite for list-based impersonate parameter functionality.""" +import pytest +import pytest_httpbin +from unittest.mock import patch, MagicMock + +from scrapling import Fetcher +from scrapling.fetchers import FetcherSession +from scrapling.engines.static import _select_random_browser + + +class TestRandomBrowserSelection: + """Test the random browser selection helper function.""" + + def test_select_random_browser_with_single_string(self): + """Test that single browser string is returned as-is.""" + result = _select_random_browser("chrome") + assert result == "chrome" + + def test_select_random_browser_with_none(self): + """Test that None is returned as-is.""" + result = _select_random_browser(None) + assert result is None + + def test_select_random_browser_with_list(self): + """Test that a browser is randomly selected from a list.""" + browsers = ["chrome", "firefox", "safari"] + result = _select_random_browser(browsers) + assert result in browsers + + def test_select_random_browser_with_empty_list(self): + """Test that empty list returns None.""" + result = _select_random_browser([]) + assert result is None + + def test_select_random_browser_with_single_item_list(self): + """Test that single-item list returns that item.""" + result = _select_random_browser(["chrome"]) + assert result == "chrome" + + +@pytest_httpbin.use_class_based_httpbin +class TestFetcherWithImpersonateList: + """Test Fetcher with list-based impersonate parameter.""" + + @pytest.fixture(autouse=True) + def setup_urls(self, httpbin): + """Fixture to set up URLs for testing.""" + self.basic_url = f"{httpbin.url}/get" + + def test_get_with_impersonate_list(self): + """Test that GET request works with impersonate as a list.""" + browsers = ["chrome", "firefox"] + response = Fetcher.get(self.basic_url, impersonate=browsers) + assert response.status == 200 + + def test_get_with_single_impersonate(self): + """Test that GET request still works with single browser string.""" + response = Fetcher.get(self.basic_url, impersonate="chrome") + assert response.status == 200 + + def test_post_with_impersonate_list(self): + """Test that POST request works with impersonate as a list.""" + browsers = ["chrome", "firefox"] + post_url = self.basic_url.replace("/get", "/post") + response = Fetcher.post(post_url, data={"key": "value"}, impersonate=browsers) + assert response.status == 200 + + def test_put_with_impersonate_list(self): + """Test that PUT request works with impersonate as a list.""" + browsers = ["chrome", "safari"] + put_url = self.basic_url.replace("/get", "/put") + response = Fetcher.put(put_url, data={"key": "value"}, impersonate=browsers) + assert response.status == 200 + + def test_delete_with_impersonate_list(self): + """Test that DELETE request works with impersonate as a list.""" + browsers = ["chrome", "edge"] + delete_url = self.basic_url.replace("/get", "/delete") + response = Fetcher.delete(delete_url, impersonate=browsers) + assert response.status == 200 + + +@pytest_httpbin.use_class_based_httpbin +class TestFetcherSessionWithImpersonateList: + """Test FetcherSession with list-based impersonate parameter.""" + + @pytest.fixture(autouse=True) + def setup_urls(self, httpbin): + """Fixture to set up URLs for testing.""" + self.basic_url = f"{httpbin.url}/get" + + def test_session_init_with_impersonate_list(self): + """Test that FetcherSession can be initialized with impersonate as a list.""" + browsers = ["chrome", "firefox", "safari"] + session = FetcherSession(impersonate=browsers) + assert session._default_impersonate == browsers + + def test_session_request_with_impersonate_list(self): + """Test that session request works with impersonate as a list.""" + browsers = ["chrome", "firefox"] + with FetcherSession(impersonate=browsers) as session: + response = session.get(self.basic_url) + assert response.status == 200 + + def test_session_multiple_requests_with_impersonate_list(self): + """Test that multiple requests in a session work with impersonate list.""" + browsers = ["chrome110", "chrome120", "chrome131"] + with FetcherSession(impersonate=browsers) as session: + response1 = session.get(self.basic_url) + response2 = session.get(self.basic_url) + assert response1.status == 200 + assert response2.status == 200 + + def test_session_request_level_impersonate_override(self): + """Test that request-level impersonate overrides session-level.""" + session_browsers = ["chrome", "firefox"] + request_browser = "safari" + + with FetcherSession(impersonate=session_browsers) as session: + response = session.get(self.basic_url, impersonate=request_browser) + assert response.status == 200 + + def test_session_request_level_impersonate_list_override(self): + """Test that request-level impersonate list overrides session-level.""" + session_browsers = ["chrome", "firefox"] + request_browsers = ["safari", "edge"] + + with FetcherSession(impersonate=session_browsers) as session: + response = session.get(self.basic_url, impersonate=request_browsers) + assert response.status == 200 + + +class TestImpersonateTypeValidation: + """Test type validation for impersonate parameter.""" + + def test_impersonate_accepts_string(self): + """Test that impersonate accepts string type.""" + # This should not raise any type errors + session = FetcherSession(impersonate="chrome") + assert session._default_impersonate == "chrome" + + def test_impersonate_accepts_list(self): + """Test that impersonate accepts list type.""" + # This should not raise any type errors + browsers = ["chrome", "firefox"] + session = FetcherSession(impersonate=browsers) + assert session._default_impersonate == browsers + + def test_impersonate_accepts_none(self): + """Test that impersonate accepts None.""" + # This should not raise any type errors + session = FetcherSession(impersonate=None) + assert session._default_impersonate is None diff --git a/tox.ini b/tox.ini index 758939f..0109863 100644 --- a/tox.ini +++ b/tox.ini @@ -15,6 +15,11 @@ deps = camoufox -r{toxinidir}/tests/requirements.txt extras = ai,shell +commands_pre = + # Install browsers in the tox virtual environment + python -m playwright install chromium + python -m playwright install-deps chromium firefox + python -m camoufox fetch --browserforge commands = # Run browser tests without parallelization (avoid browser conflicts) pytest --cov=scrapling --cov-report=xml -k "DynamicFetcher or StealthyFetcher" --verbose