diff --git a/.github/workflows/code-quality.yml b/.github/workflows/code-quality.yml index ada0ec2..5c32328 100644 --- a/.github/workflows/code-quality.yml +++ b/.github/workflows/code-quality.yml @@ -37,12 +37,12 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@v6 with: fetch-depth: 0 # Full history for better analysis - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: '3.10' cache: 'pip' @@ -177,7 +177,7 @@ jobs: - name: Upload Bandit report if: always() && steps.bandit.outcome != 'skipped' - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v6 with: name: bandit-security-report path: bandit-report.json diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 3c8ddcd..2c0948c 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -25,7 +25,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v6 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 @@ -70,7 +70,7 @@ jobs: org.opencontainers.image.documentation=https://scrapling.readthedocs.io/en/latest/ - name: Build and push Docker image - uses: docker/build-push-action@v5 + uses: docker/build-push-action@v6 with: context: . platforms: linux/amd64,linux/arm64 diff --git a/.github/workflows/release-and-publish.yml b/.github/workflows/release-and-publish.yml index 81705b4..d435088 100644 --- a/.github/workflows/release-and-publish.yml +++ b/.github/workflows/release-and-publish.yml @@ -18,7 +18,7 @@ jobs: contents: write id-token: write steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: fetch-depth: 0 @@ -27,7 +27,7 @@ jobs: run: echo "title=${{ github.event.pull_request.title }}" >> $GITHUB_OUTPUT - name: Save PR body to file - uses: actions/github-script@v6 + uses: actions/github-script@v8 with: script: | const fs = require('fs'); @@ -46,7 +46,7 @@ jobs: fi - name: Create Release - uses: softprops/action-gh-release@v1 + uses: softprops/action-gh-release@v2 with: tag_name: ${{ steps.extract_version.outputs.version }} name: Release ${{ steps.extract_version.outputs.version }} @@ -57,7 +57,7 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: 3.12 diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index be3bbca..75d6ab1 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -44,10 +44,10 @@ jobs: TOXENV: py313 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} cache: 'pip' @@ -69,7 +69,7 @@ jobs: - name: Retrieve Playwright browsers from cache if any id: playwright-cache - uses: actions/cache@v4 + uses: actions/cache@v5 with: path: | ~/.cache/ms-playwright @@ -92,7 +92,7 @@ jobs: # Cache tox environments - name: Cache tox environments - uses: actions/cache@v4 + uses: actions/cache@v5 with: path: .tox # Include python version and os in the cache key diff --git a/README.md b/README.md index 27474e4..aae6045 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,8 @@

+ D4Vinci%2FScrapling | Trendshift +
العربيه | Español | Deutsch | 简体中文 | 日本語 | Русский
@@ -74,7 +76,6 @@ class MySpider(Spider): MySpider().start() ``` - # Platinum Sponsors # Sponsors @@ -88,6 +89,7 @@ MySpider().start() + diff --git a/docs/README_AR.md b/docs/README_AR.md index 0700110..35a47aa 100644 --- a/docs/README_AR.md +++ b/docs/README_AR.md @@ -35,11 +35,13 @@ · اختيار Fetcher · + العناكب + · + تدوير البروكسي + · واجهة سطر الأوامر · وضع MCP - · - الانتقال من Beautifulsoup

Scrapling هو إطار عمل تكيفي لـ Web Scraping يتعامل مع كل شيء من طلب واحد إلى زحف كامل النطاق. @@ -84,6 +86,7 @@ MySpider().start() + diff --git a/docs/README_CN.md b/docs/README_CN.md index a44e140..24112b3 100644 --- a/docs/README_CN.md +++ b/docs/README_CN.md @@ -35,11 +35,13 @@ · 选择Fetcher · + 爬虫 + · + 代理轮换 + · CLI · MCP模式 - · - 从Beautifulsoup迁移

Scrapling是一个自适应Web Scraping框架，能处理从单个请求到大规模爬取的一切需求。 @@ -84,6 +86,7 @@ MySpider().start() + diff --git a/docs/README_DE.md b/docs/README_DE.md index 8bef383..9b0e6a8 100644 --- a/docs/README_DE.md +++ b/docs/README_DE.md @@ -35,11 +35,13 @@ · Einen Fetcher wählen · + Spiders + · + Proxy-Rotation + · CLI · MCP-Modus - · - Migration von Beautifulsoup

Scrapling ist ein adaptives Web-Scraping-Framework, das alles abdeckt -- von einer einzelnen Anfrage bis hin zu einem umfassenden Crawl. @@ -84,6 +86,7 @@ MySpider().start() + diff --git a/docs/README_ES.md b/docs/README_ES.md index cccdf7c..5fe6834 100644 --- a/docs/README_ES.md +++ b/docs/README_ES.md @@ -31,15 +31,17 @@

- Metodos de seleccion + Métodos de selección · Elegir un fetcher · + Spiders + · + Rotación de proxy + · CLI · Modo MCP - · - Migrar desde Beautifulsoup

Scrapling es un framework de Web Scraping adaptativo que se encarga de todo, desde una sola solicitud hasta un rastreo a gran escala. @@ -84,6 +86,7 @@ MySpider().start() + diff --git a/docs/README_JP.md b/docs/README_JP.md index d131289..d423d9b 100644 --- a/docs/README_JP.md +++ b/docs/README_JP.md @@ -35,11 +35,13 @@ · Fetcherの選び方 · + スパイダー + · + プロキシローテーション + · CLI · MCPモード - · - Beautifulsoupからの移行

Scraplingは、単一のリクエストから本格的なクロールまですべてを処理する適応型Web Scrapingフレームワークです。 @@ -84,6 +86,7 @@ MySpider().start() + diff --git a/docs/README_RU.md b/docs/README_RU.md index 4f865e4..0343cc0 100644 --- a/docs/README_RU.md +++ b/docs/README_RU.md @@ -35,11 +35,13 @@ · Выбор Fetcher · + Пауки + · + Ротация прокси + · CLI · Режим MCP - · - Миграция с Beautifulsoup

Scrapling — это адаптивный фреймворк для Web Scraping, который берёт на себя всё: от одного запроса до полномасштабного обхода сайтов. @@ -84,6 +86,7 @@ MySpider().start() + diff --git a/docs/donate.md b/docs/donate.md index 549a794..9c8349e 100644 --- a/docs/donate.md +++ b/docs/donate.md @@ -27,4 +27,4 @@ Perks: 2. The same logo will be featured at [the top of Scrapling's PyPI page](https://pypi.org/project/scrapling/) and [the top of Docker's image page](https://hub.docker.com/r/pyd4vinci/scrapling), the same way it was placed on the project's page. 3. Your logo will have a special placement as a top sponsor on [Scrapling's website](https://scrapling.readthedocs.io/en/latest/) main page. 4. A partner role at our Discord server. -5. A Shoutout at the end of each [Release notes](https://github.com/D4Vinci/Scrapling/releases). +5. A Shoutout at the end of each [Release notes](https://github.com/D4Vinci/Scrapling/releases). \ No newline at end of file diff --git a/docs/index.md b/docs/index.md index 3e53e33..15f7b5d 100644 --- a/docs/index.md +++ b/docs/index.md @@ -55,6 +55,7 @@ MySpider().start() + diff --git a/docs/tutorials/external.md b/docs/tutorials/external.md deleted file mode 100644 index 15a5f9b..0000000 --- a/docs/tutorials/external.md +++ /dev/null @@ -1,34 +0,0 @@ - -If you have issues with the browser installation, such as resource management, we recommend you try the Cloud Browser from [Scrapeless](https://www.scrapeless.com/en/product/scraping-browser?utm_source=official&utm_term=scrapling) for free! - -The usage is straightforward: create an account and [get your API key](https://docs.scrapeless.com/en/scraping-browser/quickstart/getting-started/?utm_source=official&utm_term=scrapling), then pass it to the `DynamicSession` like this: - -```python -from urllib.parse import urlencode - -from scrapling.fetchers import DynamicSession - -# Configure your browser session -config = { - "token": "YOUR_API_KEY", - "sessionName": "scrapling-session", - "sessionTTL": "300", # 5 minutes - "proxyCountry": "ANY", - "sessionRecording": "false", -} - -# Build WebSocket URL -ws_endpoint = f"wss://browser.scrapeless.com/api/v2/browser?{urlencode(config)}" -print('Connecting to Scrapeless...') - -with DynamicSession(cdp_url=ws_endpoint, disable_resources=True) as s: - print("Connected!") - page = s.fetch("https://httpbin.org/headers", network_idle=True) - print(f"Page loaded, content length: {len(page.body)}") - print(page.json()) -``` -The `DynamicSession` class instance will work as usual, so no further explanation is needed. - -However, the Scrapeless Cloud Browser can be configured with proxy options, like the proxy country in the config above, [custom fingerprint](https://docs.scrapeless.com/en/scraping-browser/features/advanced-privacy-anti-detection/custom-fingerprint/?utm_source=official&utm_term=scrapling) configuration, [captcha solving](https://docs.scrapeless.com/en/scraping-browser/features/advanced-privacy-anti-detection/supported-captchas/?utm_source=official&utm_term=scrapling), and more. - -Check out the [Scrapeless's browser documentation](https://docs.scrapeless.com/en/scraping-browser/quickstart/introduction/?utm_source=official&utm_term=scrapling) for more details. \ No newline at end of file diff --git a/images/ProxyEmpire.png b/images/ProxyEmpire.png new file mode 100644 index 0000000..d407789 Binary files /dev/null and b/images/ProxyEmpire.png differ diff --git a/scrapling/engines/_browsers/_stealth.py b/scrapling/engines/_browsers/_stealth.py index 3c9ea58..fe71865 100644 --- a/scrapling/engines/_browsers/_stealth.py +++ b/scrapling/engines/_browsers/_stealth.py @@ -22,7 +22,7 @@ from scrapling.engines._browsers._types import StealthSession, StealthFetchParam from scrapling.engines._browsers._base import SyncSession, AsyncSession, StealthySessionMixin from scrapling.engines._browsers._validators import validate_fetch as _validate, StealthConfig -__CF_PATTERN__ = re_compile("challenges.cloudflare.com/cdn-cgi/challenge-platform/.*") +__CF_PATTERN__ = re_compile(r"^https?://challenges\.cloudflare\.com/cdn-cgi/challenge-platform/.*") class StealthySession(SyncSession, StealthySessionMixin): diff --git a/zensical.toml b/zensical.toml index 06b5575..0b52de7 100644 --- a/zensical.toml +++ b/zensical.toml @@ -50,8 +50,7 @@ nav = [ ]}, {Tutorials = [ {"A Free Alternative to AI for Robust Web Scraping" = "tutorials/replacing_ai.md"}, - {"Migrating from BeautifulSoup" = "tutorials/migrating_from_beautifulsoup.md"}, - {"Using Scrapeless browser" = "tutorials/external.md"} + {"Migrating from BeautifulSoup" = "tutorials/migrating_from_beautifulsoup.md"} ]}, {Development = [ {"API Reference" = [