From 60eea3ba68f38f0832b11e532c337b484d951921 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Wed, 18 Feb 2026 00:04:53 +0200 Subject: [PATCH 01/22] docs: add media preview image --- docs/overrides/main.html | 8 ++++++++ zensical.toml | 1 + 2 files changed, 9 insertions(+) create mode 100644 docs/overrides/main.html diff --git a/docs/overrides/main.html b/docs/overrides/main.html new file mode 100644 index 0000000..a43e1d4 --- /dev/null +++ b/docs/overrides/main.html @@ -0,0 +1,8 @@ +{% extends "base.html" %} + +{% block extrahead %} + + + + +{% endblock %} diff --git a/zensical.toml b/zensical.toml index 402bd49..06b5575 100644 --- a/zensical.toml +++ b/zensical.toml @@ -73,6 +73,7 @@ nav = [ [project.theme] language = "en" +custom_dir = "docs/overrides" logo = "assets/logo.png" favicon = "assets/favicon.ico" features = [ From aa90dda43992620a9271263e170e4d5b844457a5 Mon Sep 17 00:00:00 2001 From: RinZ27 <222222878+RinZ27@users.noreply.github.com> Date: Wed, 18 Feb 2026 13:53:31 +0700 Subject: [PATCH 02/22] fix: improve regex precision for Cloudflare challenge detection --- scrapling/engines/_browsers/_stealth.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapling/engines/_browsers/_stealth.py b/scrapling/engines/_browsers/_stealth.py index 3c9ea58..fe71865 100644 --- a/scrapling/engines/_browsers/_stealth.py +++ b/scrapling/engines/_browsers/_stealth.py @@ -22,7 +22,7 @@ from scrapling.engines._browsers._types import StealthSession, StealthFetchParam from scrapling.engines._browsers._base import SyncSession, AsyncSession, StealthySessionMixin from scrapling.engines._browsers._validators import validate_fetch as _validate, StealthConfig -__CF_PATTERN__ = re_compile("challenges.cloudflare.com/cdn-cgi/challenge-platform/.*") +__CF_PATTERN__ = re_compile(r"^https?://challenges\.cloudflare\.com/cdn-cgi/challenge-platform/.*") class StealthySession(SyncSession, StealthySessionMixin): From 1f5a7d5d2c448e3a223eb04ca9ffaf5b86a48755 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Mon, 23 Feb 2026 02:31:28 +0200 Subject: [PATCH 03/22] ops: add mcp registry file --- README.md | 2 ++ server.json | 50 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+) create mode 100644 server.json diff --git a/README.md b/README.md index 3c5e5ca..6fe8b37 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,5 @@ + +

diff --git a/server.json b/server.json new file mode 100644 index 0000000..c88d9d1 --- /dev/null +++ b/server.json @@ -0,0 +1,50 @@ +{ + "$schema": "https://static.modelcontextprotocol.io/schemas/2025-12-11/server.schema.json", + "name": "io.github.D4Vinci/Scrapling", + "title": "Scrapling MCP Server", + "description": "Web scraping with stealth HTTP, real browsers, and Cloudflare bypass. CSS selectors supported.", + "websiteUrl": "https://scrapling.readthedocs.io/en/latest/ai/mcp-server.html", + "repository": { + "url": "https://github.com/D4Vinci/Scrapling", + "source": "github" + }, + "icons": [ + { + "src": "https://raw.githubusercontent.com/D4Vinci/Scrapling/main/docs/assets/logo.png", + "mimeType": "image/png" + } + ], + "version": "0.4.0", + "packages": [ + { + "registryType": "pypi", + "identifier": "scrapling", + "version": "0.4.0", + "runtimeHint": "uvx", + "packageArguments": [ + { + "type": "positional", + "valueHint": "mcp", + "isFixed": true + } + ], + "transport": { + "type": "stdio" + } + }, + { + "registryType": "oci", + "identifier": "ghcr.io/d4vinci/scrapling", + "packageArguments": [ + { + "type": "positional", + "valueHint": "mcp", + "isFixed": true + } + ], + "transport": { + "type": "stdio" + } + } + ] +} \ No newline at end of file From 4072c5e83d91c9ebb5d441dda5ca5757645b243e Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Mon, 23 Feb 2026 02:45:37 +0200 Subject: [PATCH 04/22] docs: update readme URLs --- README.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 6fe8b37..79a6a63 100644 --- a/README.md +++ b/README.md @@ -35,13 +35,15 @@

Selection methods · - Choosing a fetcher + Fetchers + · + Spiders + · + Proxy Rotation · CLI · - MCP mode - · - Migrating from Beautifulsoup + MCP

Scrapling is an adaptive Web Scraping framework that handles everything from a single request to a full-scale crawl. From a6beaaba891d74f4c57cb9a3e55e50bfc8a625fe Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Mon, 23 Feb 2026 02:58:25 +0200 Subject: [PATCH 05/22] docs: adjustments to sponsors placement --- README.md | 4 ++++ docs/README_AR.md | 6 ++++++ docs/README_CN.md | 6 ++++++ docs/README_DE.md | 6 ++++++ docs/README_ES.md | 6 ++++++ docs/README_JP.md | 6 ++++++ docs/README_RU.md | 6 ++++++ 7 files changed, 40 insertions(+) diff --git a/README.md b/README.md index 79a6a63..1e19415 100644 --- a/README.md +++ b/README.md @@ -75,6 +75,8 @@ MySpider().start() ``` +# Platinum Sponsors + # Sponsors @@ -87,6 +89,8 @@ MySpider().start() + + diff --git a/docs/README_AR.md b/docs/README_AR.md index 4ac86da..2d7400d 100644 --- a/docs/README_AR.md +++ b/docs/README_AR.md @@ -1,3 +1,5 @@ + +

@@ -69,6 +71,8 @@ MySpider().start() ``` +# الرعاة البلاتينيون + # الرعاة @@ -81,6 +85,8 @@ MySpider().start() + + diff --git a/docs/README_CN.md b/docs/README_CN.md index b33c422..9147296 100644 --- a/docs/README_CN.md +++ b/docs/README_CN.md @@ -1,3 +1,5 @@ + +

@@ -69,6 +71,8 @@ MySpider().start() ``` +# 铂金赞助商 + # 赞助商 @@ -81,6 +85,8 @@ MySpider().start() + + diff --git a/docs/README_DE.md b/docs/README_DE.md index 7124483..6a1cb14 100644 --- a/docs/README_DE.md +++ b/docs/README_DE.md @@ -1,3 +1,5 @@ + +

@@ -69,6 +71,8 @@ MySpider().start() ``` +# Platin-Sponsoren + # Sponsoren @@ -81,6 +85,8 @@ MySpider().start() + + diff --git a/docs/README_ES.md b/docs/README_ES.md index 8957c41..328d40b 100644 --- a/docs/README_ES.md +++ b/docs/README_ES.md @@ -1,3 +1,5 @@ + +

@@ -69,6 +71,8 @@ MySpider().start() ``` +# Patrocinadores Platino + # Patrocinadores @@ -81,6 +85,8 @@ MySpider().start() + + diff --git a/docs/README_JP.md b/docs/README_JP.md index 3423f49..6cadb43 100644 --- a/docs/README_JP.md +++ b/docs/README_JP.md @@ -1,3 +1,5 @@ + +

@@ -69,6 +71,8 @@ MySpider().start() ``` +# プラチナスポンサー + # スポンサー @@ -81,6 +85,8 @@ MySpider().start() + + diff --git a/docs/README_RU.md b/docs/README_RU.md index dc59ba6..67b8a0a 100644 --- a/docs/README_RU.md +++ b/docs/README_RU.md @@ -1,3 +1,5 @@ + +

@@ -69,6 +71,8 @@ MySpider().start() ``` +# Платиновые спонсоры + # Спонсоры @@ -81,6 +85,8 @@ MySpider().start() + + From f5376139211f277d04a999b3a3d927a9a39e0a43 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Mon, 23 Feb 2026 03:04:27 +0200 Subject: [PATCH 06/22] docs: shortening the demo code in the README --- README.md | 6 +++--- docs/README_AR.md | 6 +++--- docs/README_CN.md | 6 +++--- docs/README_DE.md | 6 +++--- docs/README_ES.md | 6 +++--- docs/README_JP.md | 6 +++--- docs/README_RU.md | 6 +++--- 7 files changed, 21 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 1e19415..27474e4 100644 --- a/README.md +++ b/README.md @@ -55,9 +55,9 @@ Blazing fast crawls with real-time stats and streaming. Built by Web Scrapers fo ```python from scrapling.fetchers import Fetcher, AsyncFetcher, StealthyFetcher, DynamicFetcher StealthyFetcher.adaptive = True -page = StealthyFetcher.fetch('https://example.com', headless=True, network_idle=True) # Fetch website under the radar! -products = page.css('.product', auto_save=True) # Scrape data that survives website design changes! -products = page.css('.product', adaptive=True) # Later, if the website structure changes, pass `adaptive=True` to find them! +p = StealthyFetcher.fetch('https://example.com', headless=True, network_idle=True) # Fetch website under the radar! +products = p.css('.product', auto_save=True) # Scrape data that survives website design changes! +products = p.css('.product', adaptive=True) # Later, if the website structure changes, pass `adaptive=True` to find them! ``` Or scale up to full crawls ```python diff --git a/docs/README_AR.md b/docs/README_AR.md index 2d7400d..0700110 100644 --- a/docs/README_AR.md +++ b/docs/README_AR.md @@ -51,9 +51,9 @@ Scrapling هو إطار عمل تكيفي لـ Web Scraping يتعامل مع ك ```python from scrapling.fetchers import Fetcher, AsyncFetcher, StealthyFetcher, DynamicFetcher StealthyFetcher.adaptive = True -page = StealthyFetcher.fetch('https://example.com', headless=True, network_idle=True) # احصل على الموقع بشكل خفي! -products = page.css('.product', auto_save=True) # استخرج بيانات تنجو من تغييرات تصميم الموقع! -products = page.css('.product', adaptive=True) # لاحقاً، إذا تغيرت بنية الموقع، مرر `adaptive=True` للعثور عليها! +p = StealthyFetcher.fetch('https://example.com', headless=True, network_idle=True) # احصل على الموقع بشكل خفي! +products = p.css('.product', auto_save=True) # استخرج بيانات تنجو من تغييرات تصميم الموقع! +products = p.css('.product', adaptive=True) # لاحقاً، إذا تغيرت بنية الموقع، مرر `adaptive=True` للعثور عليها! ``` أو توسع إلى عمليات زحف كاملة ```python diff --git a/docs/README_CN.md b/docs/README_CN.md index 9147296..a44e140 100644 --- a/docs/README_CN.md +++ b/docs/README_CN.md @@ -51,9 +51,9 @@ Scrapling是一个自适应Web Scraping框架，能处理从单个请求到大 ```python from scrapling.fetchers import Fetcher, AsyncFetcher, StealthyFetcher, DynamicFetcher StealthyFetcher.adaptive = True -page = StealthyFetcher.fetch('https://example.com', headless=True, network_idle=True) # 隐秘地获取网站！ -products = page.css('.product', auto_save=True) # 抓取在网站设计变更后仍能存活的数据！ -products = page.css('.product', adaptive=True) # 之后，如果网站结构改变，传递 `adaptive=True` 来找到它们！ +p = StealthyFetcher.fetch('https://example.com', headless=True, network_idle=True) # 隐秘地获取网站！ +products = p.css('.product', auto_save=True) # 抓取在网站设计变更后仍能存活的数据！ +products = p.css('.product', adaptive=True) # 之后，如果网站结构改变，传递 `adaptive=True` 来找到它们！ ``` 或扩展为完整爬取 ```python diff --git a/docs/README_DE.md b/docs/README_DE.md index 6a1cb14..8bef383 100644 --- a/docs/README_DE.md +++ b/docs/README_DE.md @@ -51,9 +51,9 @@ Blitzschnelle Crawls mit Echtzeit-Statistiken und Streaming. Von Web Scrapern f ```python from scrapling.fetchers import Fetcher, AsyncFetcher, StealthyFetcher, DynamicFetcher StealthyFetcher.adaptive = True -page = StealthyFetcher.fetch('https://example.com', headless=True, network_idle=True) # Website unbemerkt abrufen! -products = page.css('.product', auto_save=True) # Daten scrapen, die Website-Designänderungen überleben! -products = page.css('.product', adaptive=True) # Später, wenn sich die Website-Struktur ändert, `adaptive=True` übergeben, um sie zu finden! +p = StealthyFetcher.fetch('https://example.com', headless=True, network_idle=True) # Website unbemerkt abrufen! +products = p.css('.product', auto_save=True) # Daten scrapen, die Website-Designänderungen überleben! +products = p.css('.product', adaptive=True) # Später, wenn sich die Website-Struktur ändert, `adaptive=True` übergeben, um sie zu finden! ``` Oder auf vollständige Crawls hochskalieren ```python diff --git a/docs/README_ES.md b/docs/README_ES.md index 328d40b..cccdf7c 100644 --- a/docs/README_ES.md +++ b/docs/README_ES.md @@ -51,9 +51,9 @@ Rastreos ultrarrápidos con estadísticas en tiempo real y Streaming. Construido ```python from scrapling.fetchers import Fetcher, AsyncFetcher, StealthyFetcher, DynamicFetcher StealthyFetcher.adaptive = True -page = StealthyFetcher.fetch('https://example.com', headless=True, network_idle=True) # ¡Obtén el sitio web bajo el radar! -products = page.css('.product', auto_save=True) # ¡Extrae datos que sobreviven a cambios de diseño del sitio web! -products = page.css('.product', adaptive=True) # Más tarde, si la estructura del sitio web cambia, ¡pasa `adaptive=True` para encontrarlos! +p = StealthyFetcher.fetch('https://example.com', headless=True, network_idle=True) # ¡Obtén el sitio web bajo el radar! +products = p.css('.product', auto_save=True) # ¡Extrae datos que sobreviven a cambios de diseño del sitio web! +products = p.css('.product', adaptive=True) # Más tarde, si la estructura del sitio web cambia, ¡pasa `adaptive=True` para encontrarlos! ``` O escala a rastreos completos ```python diff --git a/docs/README_JP.md b/docs/README_JP.md index 6cadb43..d131289 100644 --- a/docs/README_JP.md +++ b/docs/README_JP.md @@ -51,9 +51,9 @@ Scraplingは、単一のリクエストから本格的なクロールまです ```python from scrapling.fetchers import Fetcher, AsyncFetcher, StealthyFetcher, DynamicFetcher StealthyFetcher.adaptive = True -page = StealthyFetcher.fetch('https://example.com', headless=True, network_idle=True) # レーダーの下でウェブサイトを取得！ -products = page.css('.product', auto_save=True) # ウェブサイトのデザイン変更に耐えるデータをスクレイプ！ -products = page.css('.product', adaptive=True) # 後でウェブサイトの構造が変わったら、`adaptive=True`を渡して見つける！ +p = StealthyFetcher.fetch('https://example.com', headless=True, network_idle=True) # レーダーの下でウェブサイトを取得！ +products = p.css('.product', auto_save=True) # ウェブサイトのデザイン変更に耐えるデータをスクレイプ！ +products = p.css('.product', adaptive=True) # 後でウェブサイトの構造が変わったら、`adaptive=True`を渡して見つける！ ``` または本格的なクロールへスケールアップ ```python diff --git a/docs/README_RU.md b/docs/README_RU.md index 67b8a0a..4f865e4 100644 --- a/docs/README_RU.md +++ b/docs/README_RU.md @@ -51,9 +51,9 @@ Scrapling — это адаптивный фреймворк для Web Scraping ```python from scrapling.fetchers import Fetcher, AsyncFetcher, StealthyFetcher, DynamicFetcher StealthyFetcher.adaptive = True -page = StealthyFetcher.fetch('https://example.com', headless=True, network_idle=True) # Загрузите сайт незаметно! -products = page.css('.product', auto_save=True) # Скрапьте данные, которые переживут изменения дизайна сайта! -products = page.css('.product', adaptive=True) # Позже, если структура сайта изменится, передайте `adaptive=True`, чтобы найти их! +p = StealthyFetcher.fetch('https://example.com', headless=True, network_idle=True) # Загрузите сайт незаметно! +products = p.css('.product', auto_save=True) # Скрапьте данные, которые переживут изменения дизайна сайта! +products = p.css('.product', adaptive=True) # Позже, если структура сайта изменится, передайте `adaptive=True`, чтобы найти их! ``` Или масштабируйте до полного обхода ```python From aed842c42eaa584d218a36c7c7b5cb60b4c2d8a0 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Mon, 23 Feb 2026 03:25:18 +0200 Subject: [PATCH 07/22] docs: update sponsorship plans --- docs/donate.md | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/docs/donate.md b/docs/donate.md index f5464a0..5143fc8 100644 --- a/docs/donate.md +++ b/docs/donate.md @@ -1,24 +1,29 @@ -I've been working on Scrapling and other public projects in my spare time and have invested considerable resources and effort to make them available to the community for free. By becoming a sponsor, you would directly fund my coffee reserves, helping me continuously update existing projects and create new ones. +I've been creating all of these projects in my spare time and have invested considerable resources & effort in providing them to the community for free. By becoming a sponsor, you'd be directly funding my coffee reserves, helping me fulfill my responsibilities, and enabling me to continuously update existing projects and potentially create new ones. You can sponsor me directly through the [GitHub Sponsors program](https://github.com/sponsors/D4Vinci) or [Buy Me a Coffee](https://buymeacoffee.com/d4vinci). Thank you, stay curious, and hack the planet! ❤️ ## Advertisement -If you are looking to **advertise** your business through Scrapling and take advantage of our target audience, check out the [available tiers](https://github.com/sponsors/D4Vinci): +If you are looking to **advertise** your business to our target audience, check out the [available tiers](https://github.com/sponsors/D4Vinci): -### [The Silver tier](https://github.com/sponsors/D4Vinci/sponsorships?tier_id=435496) ($50/month) +### 1. [The Silver tier](https://github.com/sponsors/D4Vinci/sponsorships?tier_id=435496) ($50/month) Perks: -- Your logo will be featured at [the top of Scrapling's project page](https://github.com/D4Vinci/Scrapling?tab=readme-ov-file#sponsors). -- The same logo will be featured at [the top of Scrapling's PyPI page](https://pypi.org/project/scrapling/). -- The same logo will be featured at [the top of Docker's image page](https://hub.docker.com/r/pyd4vinci/scrapling). +1. Your logo will be featured at [the top of Scrapling's project page](https://github.com/D4Vinci/Scrapling?tab=readme-ov-file#sponsors). +2. The same logo will be featured at [the top of Scrapling's PyPI page](https://pypi.org/project/scrapling/) and [the top of Docker's image page](https://hub.docker.com/r/pyd4vinci/scrapling), the same way it was placed on the project's page. -### [The Gold tier](https://github.com/sponsors/D4Vinci/sponsorships?tier_id=435495) ($100/month) +### 2. [The Gold tier](https://github.com/sponsors/D4Vinci/sponsorships?tier_id=435495) ($100/month) Perks: -- Your logo will be featured at [the top of Scrapling's project page](https://github.com/D4Vinci/Scrapling?tab=readme-ov-file#sponsors). -- The same logo will be featured at [the top of Scrapling's PyPI page](https://pypi.org/project/scrapling/). -- The same logo will be featured at [the top of Docker's image page](https://hub.docker.com/r/pyd4vinci/scrapling). -- Your logo will be featured as a top sponsor on [Scrapling's website](https://scrapling.readthedocs.io/en/latest/) main page. -- A Shoutout with each [Release note](https://github.com/D4Vinci/Scrapling/releases). +1. Your logo will be featured at [the top of Scrapling's project page](https://github.com/D4Vinci/Scrapling?tab=readme-ov-file#sponsors). +2. The same logo will be featured at [the top of Scrapling's PyPI page](https://pypi.org/project/scrapling/) and [the top of Docker's image page](https://hub.docker.com/r/pyd4vinci/scrapling), the same way it was placed on the project's page. +3. Your logo will be featured as a top sponsor on [Scrapling's website](https://scrapling.readthedocs.io/en/latest/) main page. + +### 3. [The Gold tier](https://github.com/sponsors/D4Vinci/sponsorships?tier_id=586646) ($100/month) +Perks: + +1. Your logo will have a special placement at [the very top of Scrapling's project page](https://github.com/D4Vinci/Scrapling?tab=readme-ov-file#sponsors) with an 80-word paragraph or less. +2. The same logo will be featured at [the top of Scrapling's PyPI page](https://pypi.org/project/scrapling/) and [the top of Docker's image page](https://hub.docker.com/r/pyd4vinci/scrapling), the same way it was placed on the project's page. +3. Your logo will have a special placement as a top sponsor on [Scrapling's website](https://scrapling.readthedocs.io/en/latest/) main page. +4. A Shoutout at the end of each [Release notes](https://github.com/D4Vinci/Scrapling/releases). From a0cc3857d01ee3d88b6c2ac4c5c59013ed1d2045 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Mon, 23 Feb 2026 03:27:43 +0200 Subject: [PATCH 08/22] docs: corrections to the sponsorship page --- docs/donate.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/donate.md b/docs/donate.md index 5143fc8..c8300f2 100644 --- a/docs/donate.md +++ b/docs/donate.md @@ -20,10 +20,10 @@ Perks: 2. The same logo will be featured at [the top of Scrapling's PyPI page](https://pypi.org/project/scrapling/) and [the top of Docker's image page](https://hub.docker.com/r/pyd4vinci/scrapling), the same way it was placed on the project's page. 3. Your logo will be featured as a top sponsor on [Scrapling's website](https://scrapling.readthedocs.io/en/latest/) main page. -### 3. [The Gold tier](https://github.com/sponsors/D4Vinci/sponsorships?tier_id=586646) ($100/month) +### 3. [The Platinum tier](https://github.com/sponsors/D4Vinci/sponsorships?tier_id=586646) ($300/month) Perks: -1. Your logo will have a special placement at [the very top of Scrapling's project page](https://github.com/D4Vinci/Scrapling?tab=readme-ov-file#sponsors) with an 80-word paragraph or less. +1. Your logo will have a special placement at [the very top of Scrapling's project page](https://github.com/D4Vinci/Scrapling?tab=readme-ov-file#platinum-sponsors) with an 80-word paragraph or less. 2. The same logo will be featured at [the top of Scrapling's PyPI page](https://pypi.org/project/scrapling/) and [the top of Docker's image page](https://hub.docker.com/r/pyd4vinci/scrapling), the same way it was placed on the project's page. 3. Your logo will have a special placement as a top sponsor on [Scrapling's website](https://scrapling.readthedocs.io/en/latest/) main page. 4. A Shoutout at the end of each [Release notes](https://github.com/D4Vinci/Scrapling/releases). From b503f537ce31b112d661899f18dc5e1ae79e1df2 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Mon, 23 Feb 2026 03:49:16 +0200 Subject: [PATCH 09/22] docs: corrections to the sponsorship page --- docs/donate.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/donate.md b/docs/donate.md index c8300f2..549a794 100644 --- a/docs/donate.md +++ b/docs/donate.md @@ -25,5 +25,6 @@ Perks: 1. Your logo will have a special placement at [the very top of Scrapling's project page](https://github.com/D4Vinci/Scrapling?tab=readme-ov-file#platinum-sponsors) with an 80-word paragraph or less. 2. The same logo will be featured at [the top of Scrapling's PyPI page](https://pypi.org/project/scrapling/) and [the top of Docker's image page](https://hub.docker.com/r/pyd4vinci/scrapling), the same way it was placed on the project's page. -3. Your logo will have a special placement as a top sponsor on [Scrapling's website](https://scrapling.readthedocs.io/en/latest/) main page. -4. A Shoutout at the end of each [Release notes](https://github.com/D4Vinci/Scrapling/releases). +3. Your logo will have a special placement as a top sponsor on [Scrapling's website](https://scrapling.readthedocs.io/en/latest/) main page. +4. A partner role at our Discord server. +5. A Shoutout at the end of each [Release notes](https://github.com/D4Vinci/Scrapling/releases). From b77a1b9419eb0d4ea8ff09d28a88b893446709d9 Mon Sep 17 00:00:00 2001 From: Robin Ede Date: Mon, 23 Feb 2026 19:49:39 -0600 Subject: [PATCH 10/22] fix: make MCP get schemas validator-safe Use JSON-object input types for cookies and basic-auth fields in get and bulk_get so strict MCP schema validators can register tools reliably. Normalize auth dictionaries to the tuple format expected by fetchers to preserve runtime behavior. --- scrapling/core/ai.py | 48 +++++++++++++++++++++++++++++++------------- 1 file changed, 34 insertions(+), 14 deletions(-) diff --git a/scrapling/core/ai.py b/scrapling/core/ai.py index 171359a..ad575f9 100644 --- a/scrapling/core/ai.py +++ b/scrapling/core/ai.py @@ -42,6 +42,20 @@ def _ContentTranslator(content: Generator[str, None, None], page: _ScraplingResp return ResponseModel(status=page.status, content=[result for result in content], url=page.url) +def _NormalizeCredentials(credentials: Optional[Dict[str, str]]) -> Optional[Tuple[str, str]]: + """Convert a credentials dictionary to a tuple accepted by fetchers.""" + if not credentials: + return None + + username = credentials.get("username") + password = credentials.get("password") + + if username is None or password is None: + return None + + return username, password + + class ScraplingMCPServer: @staticmethod def get( @@ -52,15 +66,15 @@ class ScraplingMCPServer: main_content_only: bool = True, params: Optional[Dict | List | Tuple] = None, headers: Optional[Mapping[str, Optional[str]]] = None, - cookies: Optional[Dict[str, str] | list[tuple[str, str]]] = None, + cookies: Optional[Dict[str, str]] = None, timeout: Optional[int | float] = 30, follow_redirects: bool = True, max_redirects: int = 30, retries: Optional[int] = 3, retry_delay: Optional[int] = 1, proxy: Optional[str] = None, - proxy_auth: Optional[Tuple[str, str]] = None, - auth: Optional[Tuple[str, str]] = None, + proxy_auth: Optional[Dict[str, str]] = None, + auth: Optional[Dict[str, str]] = None, verify: Optional[bool] = True, http3: Optional[bool] = False, stealthy_headers: Optional[bool] = True, @@ -87,20 +101,23 @@ class ScraplingMCPServer: :param retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second. :param proxy: Proxy URL to use. Format: "http://username:password@localhost:8030". Cannot be used together with the `proxies` parameter. - :param proxy_auth: HTTP basic auth for proxy, tuple of (username, password). - :param auth: HTTP basic auth tuple of (username, password). Only basic auth is supported. + :param proxy_auth: HTTP basic auth for proxy in dictionary format with `username` and `password` keys. + :param auth: HTTP basic auth in dictionary format with `username` and `password` keys. :param verify: Whether to verify HTTPS certificates. :param http3: Whether to use HTTP3. Defaults to False. It might be problematic if used it with `impersonate`. :param stealthy_headers: If enabled (default), it creates and adds real browser headers. It also sets the referer header as if this request came from a Google search of URL's domain. """ + normalized_proxy_auth = _NormalizeCredentials(proxy_auth) + normalized_auth = _NormalizeCredentials(auth) + page = Fetcher.get( url, - auth=auth, + auth=normalized_auth, proxy=proxy, http3=http3, verify=verify, params=params, - proxy_auth=proxy_auth, + proxy_auth=normalized_proxy_auth, retry_delay=retry_delay, stealthy_headers=stealthy_headers, impersonate=impersonate, @@ -130,15 +147,15 @@ class ScraplingMCPServer: main_content_only: bool = True, params: Optional[Dict | List | Tuple] = None, headers: Optional[Mapping[str, Optional[str]]] = None, - cookies: Optional[Dict[str, str] | list[tuple[str, str]]] = None, + cookies: Optional[Dict[str, str]] = None, timeout: Optional[int | float] = 30, follow_redirects: bool = True, max_redirects: int = 30, retries: Optional[int] = 3, retry_delay: Optional[int] = 1, proxy: Optional[str] = None, - proxy_auth: Optional[Tuple[str, str]] = None, - auth: Optional[Tuple[str, str]] = None, + proxy_auth: Optional[Dict[str, str]] = None, + auth: Optional[Dict[str, str]] = None, verify: Optional[bool] = True, http3: Optional[bool] = False, stealthy_headers: Optional[bool] = True, @@ -165,17 +182,20 @@ class ScraplingMCPServer: :param retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second. :param proxy: Proxy URL to use. Format: "http://username:password@localhost:8030". Cannot be used together with the `proxies` parameter. - :param proxy_auth: HTTP basic auth for proxy, tuple of (username, password). - :param auth: HTTP basic auth tuple of (username, password). Only basic auth is supported. + :param proxy_auth: HTTP basic auth for proxy in dictionary format with `username` and `password` keys. + :param auth: HTTP basic auth in dictionary format with `username` and `password` keys. :param verify: Whether to verify HTTPS certificates. :param http3: Whether to use HTTP3. Defaults to False. It might be problematic if used it with `impersonate`. :param stealthy_headers: If enabled (default), it creates and adds real browser headers. It also sets the referer header as if this request came from a Google search of URL's domain. """ + normalized_proxy_auth = _NormalizeCredentials(proxy_auth) + normalized_auth = _NormalizeCredentials(auth) + async with FetcherSession() as session: tasks: List[Any] = [ session.get( url, - auth=auth, + auth=normalized_auth, proxy=proxy, http3=http3, verify=verify, @@ -184,7 +204,7 @@ class ScraplingMCPServer: cookies=cookies, timeout=timeout, retries=retries, - proxy_auth=proxy_auth, + proxy_auth=normalized_proxy_auth, retry_delay=retry_delay, impersonate=impersonate, max_redirects=max_redirects, From 6943ec27f84306290ba084542bfb2ddb59785856 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Wed, 25 Feb 2026 02:23:36 +0200 Subject: [PATCH 11/22] docs: typo correction --- README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.md b/README.md index 84a569a..aae6045 100644 --- a/README.md +++ b/README.md @@ -78,8 +78,6 @@ MySpider().start() # Platinum Sponsors -# Platinum Sponsors - # Sponsors From 53dd3602962af730e87a24b13b46546cddfa3e14 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Thu, 26 Feb 2026 23:56:42 +0200 Subject: [PATCH 12/22] docs: add mcp integrated badge --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index aae6045..a39373a 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,8 @@ PyPI version PyPI Downloads + + MCP Integrated
Discord From 86e0f0f250be42d6e0187d3334eb322052c7b308 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Fri, 27 Feb 2026 00:11:02 +0200 Subject: [PATCH 13/22] docs: update deps for the website --- docs/requirements.txt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index ff7edb0..3495486 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,8 +1,8 @@ -zensical>=0.0.23 +zensical>=0.0.24 mkdocstrings>=1.0.3 -mkdocstrings-python>=2.0.2 -griffe-inherited-docstrings -griffe-runtime-objects -griffe-sphinx +mkdocstrings-python>=2.0.3 +griffe-inherited-docstrings>=1.1.3 +griffe-runtime-objects>=0.3.1 +griffe-sphinx>=0.2.1 black>=26.1.0 pngquant \ No newline at end of file From 921314c3144c8004042683fae3095c12adbf116e Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Fri, 27 Feb 2026 00:28:44 +0200 Subject: [PATCH 14/22] fix: complete MCP schema validation for remaining bare array types MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Fix params (Dict|List|Tuple → Dict), urls (Tuple[str,...] → List[str]), - Renamed _NormalizeCredentials/_ContentTranslator to snake_case. - Also, raise a ValueError on invalid credentials instead of returning None silently. --- scrapling/core/ai.py | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/scrapling/core/ai.py b/scrapling/core/ai.py index ad575f9..f637437 100644 --- a/scrapling/core/ai.py +++ b/scrapling/core/ai.py @@ -37,12 +37,12 @@ class ResponseModel(BaseModel): url: str = Field(description="The URL given by the user that resulted in this response.") -def _ContentTranslator(content: Generator[str, None, None], page: _ScraplingResponse) -> ResponseModel: +def _content_translator(content: Generator[str, None, None], page: _ScraplingResponse) -> ResponseModel: """Convert a content generator to a list of ResponseModel objects.""" return ResponseModel(status=page.status, content=[result for result in content], url=page.url) -def _NormalizeCredentials(credentials: Optional[Dict[str, str]]) -> Optional[Tuple[str, str]]: +def _normalize_credentials(credentials: Optional[Dict[str, str]]) -> Optional[Tuple[str, str]]: """Convert a credentials dictionary to a tuple accepted by fetchers.""" if not credentials: return None @@ -51,7 +51,7 @@ def _NormalizeCredentials(credentials: Optional[Dict[str, str]]) -> Optional[Tup password = credentials.get("password") if username is None or password is None: - return None + raise ValueError("Credentials dictionary must contain both 'username' and 'password' keys") return username, password @@ -64,7 +64,7 @@ class ScraplingMCPServer: extraction_type: extraction_types = "markdown", css_selector: Optional[str] = None, main_content_only: bool = True, - params: Optional[Dict | List | Tuple] = None, + params: Optional[Dict] = None, headers: Optional[Mapping[str, Optional[str]]] = None, cookies: Optional[Dict[str, str]] = None, timeout: Optional[int | float] = 30, @@ -107,8 +107,8 @@ class ScraplingMCPServer: :param http3: Whether to use HTTP3. Defaults to False. It might be problematic if used it with `impersonate`. :param stealthy_headers: If enabled (default), it creates and adds real browser headers. It also sets the referer header as if this request came from a Google search of URL's domain. """ - normalized_proxy_auth = _NormalizeCredentials(proxy_auth) - normalized_auth = _NormalizeCredentials(auth) + normalized_proxy_auth = _normalize_credentials(proxy_auth) + normalized_auth = _normalize_credentials(auth) page = Fetcher.get( url, @@ -128,7 +128,7 @@ class ScraplingMCPServer: max_redirects=max_redirects, follow_redirects=follow_redirects, ) - return _ContentTranslator( + return _content_translator( Convertor._extract_content( page, css_selector=css_selector, @@ -140,12 +140,12 @@ class ScraplingMCPServer: @staticmethod async def bulk_get( - urls: Tuple[str, ...], + urls: List[str], impersonate: ImpersonateType = "chrome", extraction_type: extraction_types = "markdown", css_selector: Optional[str] = None, main_content_only: bool = True, - params: Optional[Dict | List | Tuple] = None, + params: Optional[Dict] = None, headers: Optional[Mapping[str, Optional[str]]] = None, cookies: Optional[Dict[str, str]] = None, timeout: Optional[int | float] = 30, @@ -164,7 +164,7 @@ class ScraplingMCPServer: Note: This is only suitable for low-mid protection levels. For high-protection levels or websites that require JS loading, use the other tools directly. Note: If the `css_selector` resolves to more than one element, all the elements will be returned. - :param urls: A tuple of the URLs to request. + :param urls: A list of the URLs to request. :param impersonate: Browser version to impersonate its fingerprint. It's using the latest chrome version by default. :param extraction_type: The type of content to extract from the page. Defaults to "markdown". Options are: - Markdown will convert the page content to Markdown format. @@ -188,8 +188,8 @@ class ScraplingMCPServer: :param http3: Whether to use HTTP3. Defaults to False. It might be problematic if used it with `impersonate`. :param stealthy_headers: If enabled (default), it creates and adds real browser headers. It also sets the referer header as if this request came from a Google search of URL's domain. """ - normalized_proxy_auth = _NormalizeCredentials(proxy_auth) - normalized_auth = _NormalizeCredentials(auth) + normalized_proxy_auth = _normalize_credentials(proxy_auth) + normalized_auth = _normalize_credentials(auth) async with FetcherSession() as session: tasks: List[Any] = [ @@ -215,7 +215,7 @@ class ScraplingMCPServer: ] responses = await gather(*tasks) return [ - _ContentTranslator( + _content_translator( Convertor._extract_content( page, css_selector=css_selector, @@ -299,7 +299,7 @@ class ScraplingMCPServer: disable_resources=disable_resources, wait_selector_state=wait_selector_state, ) - return _ContentTranslator( + return _content_translator( Convertor._extract_content( page, css_selector=css_selector, @@ -311,7 +311,7 @@ class ScraplingMCPServer: @staticmethod async def bulk_fetch( - urls: Tuple[str, ...], + urls: List[str], extraction_type: extraction_types = "markdown", css_selector: Optional[str] = None, main_content_only: bool = True, @@ -336,7 +336,7 @@ class ScraplingMCPServer: Note: This is only suitable for low-mid protection levels. Note: If the `css_selector` resolves to more than one element, all the elements will be returned. - :param urls: A tuple of the URLs to request. + :param urls: A list of the URLs to request. :param extraction_type: The type of content to extract from the page. Defaults to "markdown". Options are: - Markdown will convert the page content to Markdown format. - HTML will return the raw HTML content of the page. @@ -384,7 +384,7 @@ class ScraplingMCPServer: tasks = [session.fetch(url) for url in urls] responses = await gather(*tasks) return [ - _ContentTranslator( + _content_translator( Convertor._extract_content( page, css_selector=css_selector, @@ -483,7 +483,7 @@ class ScraplingMCPServer: disable_resources=disable_resources, wait_selector_state=wait_selector_state, ) - return _ContentTranslator( + return _content_translator( Convertor._extract_content( page, css_selector=css_selector, @@ -495,7 +495,7 @@ class ScraplingMCPServer: @staticmethod async def bulk_stealthy_fetch( - urls: Tuple[str, ...], + urls: List[str], extraction_type: extraction_types = "markdown", css_selector: Optional[str] = None, main_content_only: bool = True, @@ -525,7 +525,7 @@ class ScraplingMCPServer: Note: This is the only suitable fetcher for high protection levels. Note: If the `css_selector` resolves to more than one element, all the elements will be returned. - :param urls: A tuple of the URLs to request. + :param urls: A list of the URLs to request. :param extraction_type: The type of content to extract from the page. Defaults to "markdown". Options are: - Markdown will convert the page content to Markdown format. - HTML will return the raw HTML content of the page. @@ -582,7 +582,7 @@ class ScraplingMCPServer: tasks = [session.fetch(url) for url in urls] responses = await gather(*tasks) return [ - _ContentTranslator( + _content_translator( Convertor._extract_content( page, css_selector=css_selector, From a8359a4878fb09e223d274a4ae1606695800648a Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Fri, 27 Feb 2026 00:32:25 +0200 Subject: [PATCH 15/22] build: pump version up and the deps --- pyproject.toml | 4 ++-- scrapling/__init__.py | 2 +- setup.cfg | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 90457e9..e16db98 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta" [project] name = "scrapling" # Static version instead of a dynamic version so we can get better layer caching while building docker, check the docker file to understand -version = "0.4" +version = "0.4.1" description = "Scrapling is an undetectable, powerful, flexible, high-performance Python library that makes Web Scraping easy and effortless as it should be!" readme = {file = "README.md", content-type = "text/markdown"} license = {file = "LICENSE"} @@ -80,7 +80,7 @@ fetchers = [ "anyio>=4.12.1" ] ai = [ - "mcp>=1.24.0", + "mcp>=1.26.0", "markdownify>=1.2.0", "scrapling[fetchers]", ] diff --git a/scrapling/__init__.py b/scrapling/__init__.py index 60c859c..1b35dbe 100644 --- a/scrapling/__init__.py +++ b/scrapling/__init__.py @@ -1,5 +1,5 @@ __author__ = "Karim Shoair (karim.shoair@pm.me)" -__version__ = "0.4" +__version__ = "0.4.1" __copyright__ = "Copyright (c) 2024 Karim Shoair" from typing import Any, TYPE_CHECKING diff --git a/setup.cfg b/setup.cfg index d8aedf7..bad13d4 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = scrapling -version = 0.4 +version = 0.4.1 author = Karim Shoair author_email = karim.shoair@pm.me description = Scrapling is an undetectable, powerful, flexible, high-performance Python library that makes Web Scraping easy and effortless as it should be! From efb320bb8c1b48bc22748ecb9ce4807b6e7dafea Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Fri, 27 Feb 2026 00:49:00 +0200 Subject: [PATCH 16/22] docs: style adjustments --- README.md | 3 +-- docs/README_AR.md | 1 + docs/README_CN.md | 1 + docs/README_DE.md | 1 + docs/README_ES.md | 1 + docs/README_JP.md | 1 + docs/README_RU.md | 1 + 7 files changed, 7 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index a39373a..56e852d 100644 --- a/README.md +++ b/README.md @@ -22,8 +22,6 @@ PyPI version PyPI Downloads - - MCP Integrated
Discord @@ -80,6 +78,7 @@ MySpider().start() # Platinum Sponsors +Do you want to be the first company to show up here? Click [here](https://github.com/sponsors/D4Vinci/sponsorships?tier_id=586646) # Sponsors diff --git a/docs/README_AR.md b/docs/README_AR.md index 35a47aa..e8486fc 100644 --- a/docs/README_AR.md +++ b/docs/README_AR.md @@ -75,6 +75,7 @@ MySpider().start() # الرعاة البلاتينيون +هل تريد أن تكون أول شركة تظهر هنا؟ انقر [هنا](https://github.com/sponsors/D4Vinci/sponsorships?tier_id=586646) # الرعاة diff --git a/docs/README_CN.md b/docs/README_CN.md index 24112b3..c54af53 100644 --- a/docs/README_CN.md +++ b/docs/README_CN.md @@ -75,6 +75,7 @@ MySpider().start() # 铂金赞助商 +想成为第一个出现在这里的公司吗？点击[这里](https://github.com/sponsors/D4Vinci/sponsorships?tier_id=586646) # 赞助商 diff --git a/docs/README_DE.md b/docs/README_DE.md index 9b0e6a8..c6d52bc 100644 --- a/docs/README_DE.md +++ b/docs/README_DE.md @@ -75,6 +75,7 @@ MySpider().start() # Platin-Sponsoren +Möchten Sie das erste Unternehmen sein, das hier erscheint? Klicken Sie [hier](https://github.com/sponsors/D4Vinci/sponsorships?tier_id=586646) # Sponsoren diff --git a/docs/README_ES.md b/docs/README_ES.md index 5fe6834..adcb887 100644 --- a/docs/README_ES.md +++ b/docs/README_ES.md @@ -75,6 +75,7 @@ MySpider().start() # Patrocinadores Platino +¿Quieres ser la primera empresa en aparecer aquí? Haz clic [aquí](https://github.com/sponsors/D4Vinci/sponsorships?tier_id=586646) # Patrocinadores diff --git a/docs/README_JP.md b/docs/README_JP.md index d423d9b..47d3f66 100644 --- a/docs/README_JP.md +++ b/docs/README_JP.md @@ -75,6 +75,7 @@ MySpider().start() # プラチナスポンサー +ここに最初に表示される企業になりませんか？[こちら](https://github.com/sponsors/D4Vinci/sponsorships?tier_id=586646)をクリック # スポンサー diff --git a/docs/README_RU.md b/docs/README_RU.md index 0343cc0..bae6392 100644 --- a/docs/README_RU.md +++ b/docs/README_RU.md @@ -75,6 +75,7 @@ MySpider().start() # Платиновые спонсоры +Хотите стать первой компанией, которая появится здесь? Нажмите [здесь](https://github.com/sponsors/D4Vinci/sponsorships?tier_id=586646) # Спонсоры From 5f885ae5f595bc4b7f17903985d24e7447dae94c Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Fri, 27 Feb 2026 01:03:56 +0200 Subject: [PATCH 17/22] docs(mcp): update cursor url --- docs/ai/mcp-server.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ai/mcp-server.md b/docs/ai/mcp-server.md index 316f455..bf59d66 100644 --- a/docs/ai/mcp-server.md +++ b/docs/ai/mcp-server.md @@ -125,7 +125,7 @@ If you are using the Docker image, then it would be something like } ``` -The same logic applies to [Cursor](https://docs.cursor.com/en/context/mcp), [WindSurf](https://windsurf.com/university/tutorials/configuring-first-mcp-server), and others. +The same logic applies to [Cursor](https://cursor.com/docs/context/mcp), [WindSurf](https://windsurf.com/university/tutorials/configuring-first-mcp-server), and others. ### Claude Code Here it's much simpler to do. If you have [Claude Code](https://www.anthropic.com/claude-code) installed, open the terminal and execute the following command: From ef29568cd0993ee3f8c85dacea8e009d9de1416a Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Fri, 27 Feb 2026 01:17:54 +0200 Subject: [PATCH 18/22] docs: add a code snippet on how to do `scrapling install` from code --- README.md | 11 ++++++++++- docs/README_AR.md | 11 ++++++++++- docs/README_CN.md | 11 ++++++++++- docs/README_DE.md | 11 ++++++++++- docs/README_ES.md | 11 ++++++++++- docs/README_JP.md | 11 ++++++++++- docs/README_RU.md | 11 ++++++++++- docs/index.md | 11 ++++++++++- 8 files changed, 80 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 56e852d..5b4ca2c 100644 --- a/README.md +++ b/README.md @@ -367,11 +367,20 @@ This installation only includes the parser engine and its dependencies, without ```bash pip install "scrapling[fetchers]" - scrapling install + scrapling install # normal install + scrapling install --force # force reinstall ``` This downloads all browsers, along with their system dependencies and fingerprint manipulation dependencies. + Or you can install them from the code instead of running a command like this: + ```python + from scrapling.cli import install + + install([], standalone_mode=False) # normal install + install(["--force"], standalone_mode=False) # force reinstall + ``` + 2. Extra features: - Install the MCP server feature: ```bash diff --git a/docs/README_AR.md b/docs/README_AR.md index e8486fc..d5f525c 100644 --- a/docs/README_AR.md +++ b/docs/README_AR.md @@ -364,11 +364,20 @@ pip install scrapling ```bash pip install "scrapling[fetchers]" - scrapling install + scrapling install # normal install + scrapling install --force # force reinstall ``` يقوم هذا بتنزيل جميع المتصفحات، إلى جانب تبعيات النظام وتبعيات معالجة fingerprint الخاصة بها. + أو يمكنك تثبيتها من الكود بدلاً من تشغيل أمر كالتالي: + ```python + from scrapling.cli import install + + install([], standalone_mode=False) # normal install + install(["--force"], standalone_mode=False) # force reinstall + ``` + 2. ميزات إضافية: - تثبيت ميزة خادم MCP: ```bash diff --git a/docs/README_CN.md b/docs/README_CN.md index c54af53..b18bff3 100644 --- a/docs/README_CN.md +++ b/docs/README_CN.md @@ -364,11 +364,20 @@ pip install scrapling ```bash pip install "scrapling[fetchers]" - scrapling install + scrapling install # normal install + scrapling install --force # force reinstall ``` 这会下载所有浏览器，以及它们的系统依赖项和fingerprint操作依赖项。 + 或者你可以从代码中安装，而不是运行命令： + ```python + from scrapling.cli import install + + install([], standalone_mode=False) # normal install + install(["--force"], standalone_mode=False) # force reinstall + ``` + 2. 额外功能： - 安装MCP服务器功能： ```bash diff --git a/docs/README_DE.md b/docs/README_DE.md index c6d52bc..4772d7f 100644 --- a/docs/README_DE.md +++ b/docs/README_DE.md @@ -364,11 +364,20 @@ Diese Installation enthält nur die Parser-Engine und ihre Abhängigkeiten, ohne ```bash pip install "scrapling[fetchers]" - scrapling install + scrapling install # normal install + scrapling install --force # force reinstall ``` Dies lädt alle Browser zusammen mit ihren Systemabhängigkeiten und Fingerprint-Manipulationsabhängigkeiten herunter. + Oder Sie können sie aus dem Code heraus installieren, anstatt einen Befehl auszuführen: + ```python + from scrapling.cli import install + + install([], standalone_mode=False) # normal install + install(["--force"], standalone_mode=False) # force reinstall + ``` + 2. Zusätzliche Funktionen: - MCP-Server-Funktion installieren: ```bash diff --git a/docs/README_ES.md b/docs/README_ES.md index adcb887..7037dd5 100644 --- a/docs/README_ES.md +++ b/docs/README_ES.md @@ -364,11 +364,20 @@ Esta instalación solo incluye el motor de análisis y sus dependencias, sin nin ```bash pip install "scrapling[fetchers]" - scrapling install + scrapling install # normal install + scrapling install --force # force reinstall ``` Esto descarga todos los navegadores, junto con sus dependencias del sistema y dependencias de manipulación de fingerprint. + O puedes instalarlos desde el código en lugar de ejecutar un comando: + ```python + from scrapling.cli import install + + install([], standalone_mode=False) # normal install + install(["--force"], standalone_mode=False) # force reinstall + ``` + 2. Características adicionales: - Instalar la característica del servidor MCP: ```bash diff --git a/docs/README_JP.md b/docs/README_JP.md index 47d3f66..890e049 100644 --- a/docs/README_JP.md +++ b/docs/README_JP.md @@ -364,11 +364,20 @@ pip install scrapling ```bash pip install "scrapling[fetchers]" - scrapling install + scrapling install # normal install + scrapling install --force # force reinstall ``` これにより、すべてのブラウザ、およびそれらのシステム依存関係とfingerprint操作依存関係がダウンロードされます。 + または、コマンドを実行する代わりにコードからインストールすることもできます： + ```python + from scrapling.cli import install + + install([], standalone_mode=False) # normal install + install(["--force"], standalone_mode=False) # force reinstall + ``` + 2. 追加機能： - MCPサーバー機能をインストール： ```bash diff --git a/docs/README_RU.md b/docs/README_RU.md index bae6392..091244c 100644 --- a/docs/README_RU.md +++ b/docs/README_RU.md @@ -364,11 +364,20 @@ pip install scrapling ```bash pip install "scrapling[fetchers]" - scrapling install + scrapling install # normal install + scrapling install --force # force reinstall ``` Это загрузит все браузеры вместе с их системными зависимостями и зависимостями для манипуляции fingerprint'ами. + Или вы можете установить их из кода вместо выполнения команды: + ```python + from scrapling.cli import install + + install([], standalone_mode=False) # normal install + install(["--force"], standalone_mode=False) # force reinstall + ``` + 2. Дополнительные возможности: - Установить функцию MCP-сервера: ```bash diff --git a/docs/index.md b/docs/index.md index 15f7b5d..2bafd80 100644 --- a/docs/index.md +++ b/docs/index.md @@ -154,11 +154,20 @@ This installation only includes the parser engine and its dependencies, without ```bash pip install "scrapling[fetchers]" - scrapling install + scrapling install # normal install + scrapling install --force # force reinstall ``` This downloads all browsers, along with their system dependencies and fingerprint manipulation dependencies. + Or you can install them from the code instead of running a command like this: + ```python + from scrapling.cli import install + + install([], standalone_mode=False) # normal install + install(["--force"], standalone_mode=False) # force reinstall + ``` + 2. Extra features: From d750fec46803dbdc4a8677efd0f66dbebc97ddd6 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Fri, 27 Feb 2026 02:32:21 +0200 Subject: [PATCH 19/22] fix(solver): Solve CF faster and handle websites that show captcha twice --- .gitignore | 3 ++ scrapling/engines/_browsers/_stealth.py | 64 +++++++++++++++---------- 2 files changed, 41 insertions(+), 26 deletions(-) diff --git a/.gitignore b/.gitignore index f27bc08..77d1210 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,7 @@ +# local files site/* +local_tests/* +.mcpregistry_* # AI related files .claude/* diff --git a/scrapling/engines/_browsers/_stealth.py b/scrapling/engines/_browsers/_stealth.py index fe71865..b375198 100644 --- a/scrapling/engines/_browsers/_stealth.py +++ b/scrapling/engines/_browsers/_stealth.py @@ -127,7 +127,7 @@ class StealthySession(SyncSession, StealthySessionMixin): challenge_type = self._detect_cloudflare(ResponseFactory._get_page_content(page)) if not challenge_type: log.error("No Cloudflare challenge found.") - return + return None else: log.info(f'The turnstile version discovered is "{challenge_type}"') if challenge_type == "non-interactive": @@ -136,7 +136,7 @@ class StealthySession(SyncSession, StealthySessionMixin): page.wait_for_timeout(1000) page.wait_for_load_state() log.info("Cloudflare captcha is solved") - return + return None else: box_selector = "#cf_turnstile div, #cf-turnstile div, .turnstile>div>div" @@ -161,7 +161,7 @@ class StealthySession(SyncSession, StealthySessionMixin): if not iframe or not outer_box: if "Just a moment..." not in (ResponseFactory._get_page_content(page)): log.info("Cloudflare captcha is solved") - return + return None outer_box = page.locator(box_selector).last.bounding_box() @@ -171,22 +171,28 @@ class StealthySession(SyncSession, StealthySessionMixin): # Move the mouse to the center of the window, then press and hold the left mouse button page.mouse.click(captcha_x, captcha_y, delay=randint(100, 200), button="left") self._wait_for_networkidle(page) - if iframe is not None: - # Wait for the frame to be removed from the page (with 30s timeout = 300 iterations * 100 ms) + + if challenge_type != "embedded": attempts = 0 - while iframe in page.frames: - if attempts >= 300: - log.info("Cloudflare iframe didn't disappear after 30s, continuing...") + while "Just a moment..." in ResponseFactory._get_page_content(page): + # Wait for the page + if attempts >= 100: + log.info("Cloudflare page didn't disappear after 10s, continuing...") break page.wait_for_timeout(100) attempts += 1 - if challenge_type != "embedded": - page.locator(box_selector).last.wait_for(state="detached") - page.locator(".zone-name-title").wait_for(state="hidden") + + # page.locator(box_selector).last.wait_for(state="detached") + # page.locator(".zone-name-title").wait_for(state="hidden") + self._wait_for_page_stability(page, True, False) - log.info("Cloudflare captcha is solved") - return + if "Just a moment..." not in (ResponseFactory._get_page_content(page)): + log.info("Cloudflare captcha is solved") + return None + else: + log.info("Looks like Cloudflare captcha is still present, solving again") + return self._cloudflare_solver(page) def fetch(self, url: str, **kwargs: Unpack[StealthFetchParams]) -> Response: """Opens up the browser and do your request based on your chosen options. @@ -384,7 +390,7 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin): challenge_type = self._detect_cloudflare(await ResponseFactory._get_async_page_content(page)) if not challenge_type: log.error("No Cloudflare challenge found.") - return + return None else: log.info(f'The turnstile version discovered is "{challenge_type}"') if challenge_type == "non-interactive": @@ -393,7 +399,7 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin): await page.wait_for_timeout(1000) await page.wait_for_load_state() log.info("Cloudflare captcha is solved") - return + return None else: box_selector = "#cf_turnstile div, #cf-turnstile div, .turnstile>div>div" @@ -418,7 +424,7 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin): if not iframe or not outer_box: if "Just a moment..." not in (await ResponseFactory._get_async_page_content(page)): log.info("Cloudflare captcha is solved") - return + return None outer_box = await page.locator(box_selector).last.bounding_box() @@ -428,22 +434,28 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin): # Move the mouse to the center of the window, then press and hold the left mouse button await page.mouse.click(captcha_x, captcha_y, delay=randint(100, 200), button="left") await self._wait_for_networkidle(page) - if iframe is not None: - # Wait for the frame to be removed from the page (with 30s timeout = 300 iterations * 100 ms) + + if challenge_type != "embedded": attempts = 0 - while iframe in page.frames: - if attempts >= 300: - log.info("Cloudflare iframe didn't disappear after 30s, continuing...") + while "Just a moment..." in (await ResponseFactory._get_async_page_content(page)): + # Wait for the page + if attempts >= 100: + log.info("Cloudflare page didn't disappear after 10s, continuing...") break await page.wait_for_timeout(100) attempts += 1 - if challenge_type != "embedded": - await page.locator(box_selector).last.wait_for(state="detached") - await page.locator(".zone-name-title").wait_for(state="hidden") + + # await page.locator(box_selector).last.wait_for(state="detached") + # await page.locator(".zone-name-title").wait_for(state="hidden") + await self._wait_for_page_stability(page, True, False) - log.info("Cloudflare captcha is solved") - return + if "Just a moment..." not in (await ResponseFactory._get_async_page_content(page)): + log.info("Cloudflare captcha is solved") + return None + else: + log.info("Looks like Cloudflare captcha is still present, solving again") + return await self._cloudflare_solver(page) async def fetch(self, url: str, **kwargs: Unpack[StealthFetchParams]) -> Response: """Opens up the browser and do your request based on your chosen options. From b50e8f050fd949bd77d0c0f9054a4558a27f5c07 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Fri, 27 Feb 2026 03:39:48 +0200 Subject: [PATCH 20/22] fix(mcp): make mcp use less tokens by striping useless tags --- scrapling/core/shell.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/scrapling/core/shell.py b/scrapling/core/shell.py index 0bd6efc..80caa99 100644 --- a/scrapling/core/shell.py +++ b/scrapling/core/shell.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- from sys import stderr +from copy import deepcopy from functools import wraps from re import sub as re_sub from collections import namedtuple @@ -571,6 +572,14 @@ class Convertor: return markdownify(body) + @classmethod + def _strip_noise_tags(cls, page: Selector) -> Selector: + """Return a copy of the Selector with noise tags removed.""" + clean_root = deepcopy(page._root) + for element in clean_root.iter(*{"script", "style", "noscript", "svg"}): + element.drop_tree() + return Selector(root=clean_root, url=page.url) + @classmethod def _extract_content( cls, @@ -587,6 +596,7 @@ class Convertor: else: if main_content_only: page = cast(Selector, page.css("body").first) or page + page = cls._strip_noise_tags(page) pages = [page] if not css_selector else cast(Selectors, page.css(css_selector)) for page in pages: @@ -596,7 +606,9 @@ class Convertor: case "html": yield page.html_content case "text": - txt_content = page.get_all_text(strip=True) + txt_content = page.get_all_text( + strip=True, ignore_tags=("script", "style", "noscript", "svg", "iframe") + ) for s in ( "\n", "\r", From e34651d8873ed287b216fdd061268a5bcc5445fd Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Fri, 27 Feb 2026 03:50:14 +0200 Subject: [PATCH 21/22] fix(stealth): improve stealth mode by removing unnecessary scripts It doesn't add anything to the table as before (it might turn out I was wrong and add them again lol) --- MANIFEST.in | 1 - scrapling/engines/_browsers/_config_tools.py | 34 --- scrapling/engines/_browsers/_stealth.py | 17 -- .../toolbelt/bypasses/navigator_plugins.js | 40 ---- .../bypasses/notification_permission.js | 5 - .../bypasses/playwright_fingerprint.js | 3 - .../engines/toolbelt/bypasses/screen_props.js | 27 --- .../toolbelt/bypasses/webdriver_fully.js | 27 --- .../toolbelt/bypasses/window_chrome.js | 213 ------------------ scrapling/engines/toolbelt/navigation.py | 14 -- tests/fetchers/test_utils.py | 21 -- 11 files changed, 402 deletions(-) delete mode 100644 scrapling/engines/toolbelt/bypasses/navigator_plugins.js delete mode 100644 scrapling/engines/toolbelt/bypasses/notification_permission.js delete mode 100644 scrapling/engines/toolbelt/bypasses/playwright_fingerprint.js delete mode 100644 scrapling/engines/toolbelt/bypasses/screen_props.js delete mode 100644 scrapling/engines/toolbelt/bypasses/webdriver_fully.js delete mode 100644 scrapling/engines/toolbelt/bypasses/window_chrome.js diff --git a/MANIFEST.in b/MANIFEST.in index 8c168dd..aa9bf23 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,7 +1,6 @@ include LICENSE include *.db include *.js -include scrapling/engines/toolbelt/bypasses/*.js include scrapling/*.db include scrapling/*.db* include scrapling/*.db-* diff --git a/scrapling/engines/_browsers/_config_tools.py b/scrapling/engines/_browsers/_config_tools.py index 2d7691b..5b9ee2e 100644 --- a/scrapling/engines/_browsers/_config_tools.py +++ b/scrapling/engines/_browsers/_config_tools.py @@ -1,38 +1,4 @@ -from functools import lru_cache - -from scrapling.engines.toolbelt.navigation import js_bypass_path from scrapling.engines.toolbelt.fingerprints import generate_headers __default_useragent__ = generate_headers(browser_mode=True).get("User-Agent") __default_chrome_useragent__ = generate_headers(browser_mode="chrome").get("User-Agent") - - -@lru_cache(1) -def _compiled_stealth_scripts(): - """Pre-read and compile stealth scripts""" - # Basic bypasses nothing fancy as I'm still working on it - # But with adding these bypasses to the above config, it bypasses many online tests like - # https://bot.sannysoft.com/ - # https://kaliiiiiiiiii.github.io/brotector/ - # https://pixelscan.net/ - # https://iphey.com/ - # https://www.browserscan.net/bot-detection <== this one also checks for the CDP runtime fingerprint - # https://arh.antoinevastel.com/bots/areyouheadless/ - # https://prescience-data.github.io/execution-monitor.html - stealth_scripts_paths = tuple( - js_bypass_path(script) - for script in ( - # Order is important - "webdriver_fully.js", - "window_chrome.js", - "navigator_plugins.js", - "notification_permission.js", - "screen_props.js", - "playwright_fingerprint.js", - ) - ) - scripts = [] - for script_path in stealth_scripts_paths: - with open(script_path, "r") as f: - scripts.append(f.read()) - return tuple(scripts) diff --git a/scrapling/engines/_browsers/_stealth.py b/scrapling/engines/_browsers/_stealth.py index b375198..797a0c1 100644 --- a/scrapling/engines/_browsers/_stealth.py +++ b/scrapling/engines/_browsers/_stealth.py @@ -17,7 +17,6 @@ from scrapling.core._types import Any, Optional, ProxyType, Unpack from scrapling.engines.toolbelt.proxy_rotation import is_proxy_error from scrapling.engines.toolbelt.convertor import Response, ResponseFactory from scrapling.engines.toolbelt.fingerprints import generate_convincing_referer -from scrapling.engines._browsers._config_tools import _compiled_stealth_scripts from scrapling.engines._browsers._types import StealthSession, StealthFetchParams from scrapling.engines._browsers._base import SyncSession, AsyncSession, StealthySessionMixin from scrapling.engines._browsers._validators import validate_fetch as _validate, StealthConfig @@ -109,14 +108,6 @@ class StealthySession(SyncSession, StealthySessionMixin): else: raise RuntimeError("Session has been already started") - def _initialize_context(self, config, ctx: BrowserContext) -> BrowserContext: - """Initialize the browser context.""" - for script in _compiled_stealth_scripts(): - ctx.add_init_script(script=script) - - ctx = super()._initialize_context(config, ctx) - return ctx - def _cloudflare_solver(self, page: Page) -> None: # pragma: no cover """Solve the cloudflare challenge displayed on the playwright page passed @@ -372,14 +363,6 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin): else: raise RuntimeError("Session has been already started") - async def _initialize_context(self, config: Any, ctx: AsyncBrowserContext) -> AsyncBrowserContext: - """Initialize the browser context.""" - for script in _compiled_stealth_scripts(): - await ctx.add_init_script(script=script) - - ctx = await super()._initialize_context(config, ctx) - return ctx - async def _cloudflare_solver(self, page: async_Page) -> None: # pragma: no cover """Solve the cloudflare challenge displayed on the playwright page passed diff --git a/scrapling/engines/toolbelt/bypasses/navigator_plugins.js b/scrapling/engines/toolbelt/bypasses/navigator_plugins.js deleted file mode 100644 index 653fa5f..0000000 --- a/scrapling/engines/toolbelt/bypasses/navigator_plugins.js +++ /dev/null @@ -1,40 +0,0 @@ -if(navigator.plugins.length == 0){ - Object.defineProperty(navigator, 'plugins', { - get: () => { - const PDFViewerPlugin = Object.create(Plugin.prototype, { - description: { value: 'Portable Document Format', enumerable: false }, - filename: { value: 'internal-pdf-viewer', enumerable: false }, - name: { value: 'PDF Viewer', enumerable: false }, - }); - const ChromePDFViewer = Object.create(Plugin.prototype, { - description: { value: 'Portable Document Format', enumerable: false }, - filename: { value: 'internal-pdf-viewer', enumerable: false }, - name: { value: 'Chrome PDF Viewer', enumerable: false }, - }); - const ChromiumPDFViewer = Object.create(Plugin.prototype, { - description: { value: 'Portable Document Format', enumerable: false }, - filename: { value: 'internal-pdf-viewer', enumerable: false }, - name: { value: 'Chromium PDF Viewer', enumerable: false }, - }); - const EdgePDFViewer = Object.create(Plugin.prototype, { - description: { value: 'Portable Document Format', enumerable: false }, - filename: { value: 'internal-pdf-viewer', enumerable: false }, - name: { value: 'Microsoft Edge PDF Viewer', enumerable: false }, - }); - const WebKitPDFPlugin = Object.create(Plugin.prototype, { - description: { value: 'Portable Document Format', enumerable: false }, - filename: { value: 'internal-pdf-viewer', enumerable: false }, - name: { value: 'WebKit built-in PDF', enumerable: false }, - }); - - return Object.create(PluginArray.prototype, { - length: { value: 5 }, - 0: { value: PDFViewerPlugin }, - 1: { value: ChromePDFViewer }, - 2: { value: ChromiumPDFViewer }, - 3: { value: EdgePDFViewer }, - 4: { value: WebKitPDFPlugin }, - }); - }, - }); -} \ No newline at end of file diff --git a/scrapling/engines/toolbelt/bypasses/notification_permission.js b/scrapling/engines/toolbelt/bypasses/notification_permission.js deleted file mode 100644 index 0c9c676..0000000 --- a/scrapling/engines/toolbelt/bypasses/notification_permission.js +++ /dev/null @@ -1,5 +0,0 @@ -// Bypasses `notificationIsDenied` test in creepsjs's 'Like Headless' sections -const isSecure = document.location.protocol.startsWith('https') -if (isSecure){ - Object.defineProperty(Notification, 'permission', {get: () => 'default'}) -} \ No newline at end of file diff --git a/scrapling/engines/toolbelt/bypasses/playwright_fingerprint.js b/scrapling/engines/toolbelt/bypasses/playwright_fingerprint.js deleted file mode 100644 index e1b959d..0000000 --- a/scrapling/engines/toolbelt/bypasses/playwright_fingerprint.js +++ /dev/null @@ -1,3 +0,0 @@ -// Remove playwright fingerprint => https://github.com/microsoft/playwright/commit/c9e673c6dca746384338ab6bb0cf63c7e7caa9b2#diff-087773eea292da9db5a3f27de8f1a2940cdb895383ad750c3cd8e01772a35b40R915 -delete window.__pwInitScripts; -delete window.__playwright__binding__; \ No newline at end of file diff --git a/scrapling/engines/toolbelt/bypasses/screen_props.js b/scrapling/engines/toolbelt/bypasses/screen_props.js deleted file mode 100644 index 5056b4b..0000000 --- a/scrapling/engines/toolbelt/bypasses/screen_props.js +++ /dev/null @@ -1,27 +0,0 @@ -const windowScreenProps = { - // Dimensions - innerHeight: 0, - innerWidth: 0, - outerHeight: 754, - outerWidth: 1313, - - // Position - screenX: 19, - pageXOffset: 0, - pageYOffset: 0, - - // Display - devicePixelRatio: 2 -}; - -try { - for (const [prop, value] of Object.entries(windowScreenProps)) { - if (value > 0) { - // The 0 values are introduced by collecting in the hidden iframe. - // They are document sizes anyway so no need to test them or inject them. - window[prop] = value; - } - } -} catch (e) { - console.warn(e); -}; \ No newline at end of file diff --git a/scrapling/engines/toolbelt/bypasses/webdriver_fully.js b/scrapling/engines/toolbelt/bypasses/webdriver_fully.js deleted file mode 100644 index 4bda260..0000000 --- a/scrapling/engines/toolbelt/bypasses/webdriver_fully.js +++ /dev/null @@ -1,27 +0,0 @@ -// Create a function that looks like a native getter -const nativeGetter = function get webdriver() { - return false; -}; - -// Copy over native function properties -Object.defineProperties(nativeGetter, { - name: { value: 'get webdriver', configurable: true }, - length: { value: 0, configurable: true }, - toString: { - value: function() { - return `function get webdriver() { [native code] }`; - }, - configurable: true - } -}); - -// Make it look native -Object.setPrototypeOf(nativeGetter, Function.prototype); - -// Apply the modified descriptor -Object.defineProperty(Navigator.prototype, 'webdriver', { - get: nativeGetter, - set: undefined, - enumerable: true, - configurable: true -}); \ No newline at end of file diff --git a/scrapling/engines/toolbelt/bypasses/window_chrome.js b/scrapling/engines/toolbelt/bypasses/window_chrome.js deleted file mode 100644 index ba63a9c..0000000 --- a/scrapling/engines/toolbelt/bypasses/window_chrome.js +++ /dev/null @@ -1,213 +0,0 @@ -// To escape `HEADCHR_CHROME_OBJ` test in headless mode => https://github.com/antoinevastel/fp-collect/blob/master/src/fpCollect.js#L322 -// Faking window.chrome fully - -if (!window.chrome) { - // First, save all existing properties - const originalKeys = Object.getOwnPropertyNames(window); - const tempObj = {}; - - // Recreate all properties in original order - for (const key of originalKeys) { - const descriptor = Object.getOwnPropertyDescriptor(window, key); - const value = window[key]; - // delete window[key]; - Object.defineProperty(tempObj, key, descriptor); - } - - // Use the exact property descriptor found in headful Chrome - // fetch it via `Object.getOwnPropertyDescriptor(window, 'chrome')` - const mockChrome = { - loadTimes: {}, - csi: {}, - app: { - isInstalled: false - }, - // Add other Chrome-specific properties - }; - - Object.defineProperty(tempObj, 'chrome', { - writable: true, - enumerable: true, - configurable: false, - value: mockChrome - }); - for (const key of Object.getOwnPropertyNames(tempObj)) { - try { - Object.defineProperty(window, key, - Object.getOwnPropertyDescriptor(tempObj, key)); - } catch (e) {} - }; - // todo: solve this - // Using line below bypasses the hasHighChromeIndex test in creepjs ==> https://github.com/abrahamjuliot/creepjs/blob/master/src/headless/index.ts#L121 - // Chrome object have to be in the end of the window properties - // Object.assign(window, tempObj); - // But makes window.chrome unreadable on 'https://bot.sannysoft.com/' -} - -// That means we're running headful and don't need to mock anything -if ('app' in window.chrome) { - return; // Nothing to do here -} -const makeError = { - ErrorInInvocation: fn => { - const err = new TypeError(`Error in invocation of app.${fn}()`); - return utils.stripErrorWithAnchor( - err, - `at ${fn} (eval at `, - ); - }, -}; -// check with: `JSON.stringify(window.chrome['app'])` -const STATIC_DATA = JSON.parse( - ` -{ - "isInstalled": false, - "InstallState": { - "DISABLED": "disabled", - "INSTALLED": "installed", - "NOT_INSTALLED": "not_installed" - }, - "RunningState": { - "CANNOT_RUN": "cannot_run", - "READY_TO_RUN": "ready_to_run", - "RUNNING": "running" - } -} - `.trim(), - ); -window.chrome.app = { - ...STATIC_DATA, - - get isInstalled() { - return false; - }, - - getDetails: function getDetails() { - if (arguments.length) { - throw makeError.ErrorInInvocation(`getDetails`); - } - return null; - }, - getIsInstalled: function getDetails() { - if (arguments.length) { - throw makeError.ErrorInInvocation(`getIsInstalled`); - } - return false; - }, - runningState: function getDetails() { - if (arguments.length) { - throw makeError.ErrorInInvocation(`runningState`); - } - return 'cannot_run'; - }, -}; -// Check that the Navigation Timing API v1 is available, we need that -if (!window.performance || !window.performance.timing) { - return; -} -const {timing} = window.performance; -window.chrome.csi = function () { - return { - onloadT: timing.domContentLoadedEventEnd, - startE: timing.navigationStart, - pageT: Date.now() - timing.navigationStart, - tran: 15, // Transition type or something - }; -}; -if (!window.PerformancePaintTiming){ - return; -} -const {performance} = window; -// Some stuff is not available on about:blank as it requires a navigation to occur, -// let's harden the code to not fail then: -const ntEntryFallback = { - nextHopProtocol: 'h2', - type: 'other', -}; - -// The API exposes some funky info regarding the connection -const protocolInfo = { - get connectionInfo() { - const ntEntry = - performance.getEntriesByType('navigation')[0] || ntEntryFallback; - return ntEntry.nextHopProtocol; - }, - get npnNegotiatedProtocol() { - // NPN is deprecated in favor of ALPN, but this implementation returns the - // HTTP/2 or HTTP2+QUIC/39 requests negotiated via ALPN. - const ntEntry = - performance.getEntriesByType('navigation')[0] || ntEntryFallback; - return ['h2', 'hq'].includes(ntEntry.nextHopProtocol) - ? ntEntry.nextHopProtocol - : 'unknown'; - }, - get navigationType() { - const ntEntry = - performance.getEntriesByType('navigation')[0] || ntEntryFallback; - return ntEntry.type; - }, - get wasAlternateProtocolAvailable() { - // The Alternate-Protocol header is deprecated in favor of Alt-Svc - // (https://www.mnot.net/blog/2016/03/09/alt-svc), so technically this - // should always return false. - return false; - }, - get wasFetchedViaSpdy() { - // SPDY is deprecated in favor of HTTP/2, but this implementation returns - // true for HTTP/2 or HTTP2+QUIC/39 as well. - const ntEntry = - performance.getEntriesByType('navigation')[0] || ntEntryFallback; - return ['h2', 'hq'].includes(ntEntry.nextHopProtocol); - }, - get wasNpnNegotiated() { - // NPN is deprecated in favor of ALPN, but this implementation returns true - // for HTTP/2 or HTTP2+QUIC/39 requests negotiated via ALPN. - const ntEntry = - performance.getEntriesByType('navigation')[0] || ntEntryFallback; - return ['h2', 'hq'].includes(ntEntry.nextHopProtocol); - }, -}; - -// Truncate number to specific number of decimals, most of the `loadTimes` stuff has 3 -function toFixed(num, fixed) { - var re = new RegExp('^-?\\d+(?:.\\d{0,' + (fixed || -1) + '})?'); - return num.toString().match(re)[0]; -} - -const timingInfo = { - get firstPaintAfterLoadTime() { - // This was never actually implemented and always returns 0. - return 0; - }, - get requestTime() { - return timing.navigationStart / 1000; - }, - get startLoadTime() { - return timing.navigationStart / 1000; - }, - get commitLoadTime() { - return timing.responseStart / 1000; - }, - get finishDocumentLoadTime() { - return timing.domContentLoadedEventEnd / 1000; - }, - get finishLoadTime() { - return timing.loadEventEnd / 1000; - }, - get firstPaintTime() { - const fpEntry = performance.getEntriesByType('paint')[0] || { - startTime: timing.loadEventEnd / 1000, // Fallback if no navigation occured (`about:blank`) - }; - return toFixed( - (fpEntry.startTime + performance.timeOrigin) / 1000, - 3, - ); - }, -}; - -window.chrome.loadTimes = function () { - return { - ...protocolInfo, - ...timingInfo, - }; -}; \ No newline at end of file diff --git a/scrapling/engines/toolbelt/navigation.py b/scrapling/engines/toolbelt/navigation.py index 5c19427..9b233d8 100644 --- a/scrapling/engines/toolbelt/navigation.py +++ b/scrapling/engines/toolbelt/navigation.py @@ -2,8 +2,6 @@ Functions related to files and URLs """ -from pathlib import Path -from functools import lru_cache from urllib.parse import urlparse from playwright.async_api import Route as async_Route @@ -14,8 +12,6 @@ from scrapling.core.utils import log from scrapling.core._types import Dict, Set, Tuple, Optional, Callable from scrapling.engines.constants import EXTRA_RESOURCES -__BYPASSES_DIR__ = Path(__file__).parent / "bypasses" - class ProxyDict(Struct): server: str @@ -111,13 +107,3 @@ def construct_proxy_dict(proxy_string: str | Dict[str, str] | Tuple) -> Dict: raise TypeError(f"Invalid proxy dictionary: {e}") raise TypeError(f"Invalid proxy string: {proxy_string}") - - -@lru_cache(10, typed=True) -def js_bypass_path(filename: str) -> str: - """Takes the base filename of a JS file inside the `bypasses` folder, then return the full path of it - - :param filename: The base filename of the JS file. - :return: The full path of the JS file. - """ - return str(__BYPASSES_DIR__ / filename) diff --git a/tests/fetchers/test_utils.py b/tests/fetchers/test_utils.py index 4637a63..0a4be9f 100644 --- a/tests/fetchers/test_utils.py +++ b/tests/fetchers/test_utils.py @@ -1,12 +1,10 @@ import pytest -from pathlib import Path from scrapling.engines.toolbelt.custom import StatusText, Response from scrapling.engines.toolbelt.navigation import ( construct_proxy_dict, create_intercept_handler, create_async_intercept_handler, - js_bypass_path, ) from scrapling.engines.toolbelt.fingerprints import ( generate_convincing_referer, @@ -203,25 +201,6 @@ class TestConstructProxyDict: construct_proxy_dict({"invalid": "structure"}) -class TestJsBypassPath: - """Test JavaScript bypass path utility""" - - def test_js_bypass_path(self): - """Test getting JavaScript bypass file path""" - result = js_bypass_path("webdriver_fully.js") - - assert isinstance(result, str) - assert result.endswith("webdriver_fully.js") - assert Path(result).exists() - - def test_js_bypass_path_caching(self): - """Test that js_bypass_path is cached""" - result1 = js_bypass_path("webdriver_fully.js") - result2 = js_bypass_path("webdriver_fully.js") - - assert result1 == result2 - - class TestFingerprintFunctions: """Test fingerprint generation functions""" From d0cc6e5a3d8f627dc573eac2b358db492e27ecb5 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Fri, 27 Feb 2026 04:52:16 +0200 Subject: [PATCH 22/22] ops: pump mcp server version up --- server.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server.json b/server.json index c88d9d1..fa9a08e 100644 --- a/server.json +++ b/server.json @@ -14,12 +14,12 @@ "mimeType": "image/png" } ], - "version": "0.4.0", + "version": "0.4.1", "packages": [ { "registryType": "pypi", "identifier": "scrapling", - "version": "0.4.0", + "version": "0.4.1", "runtimeHint": "uvx", "packageArguments": [ {