v0.4.1 (#154)

.gitignore

@@ -1,4 +1,7 @@
+# local files
 site/*
+local_tests/*
+.mcpregistry_*
 
 # AI related files
 .claude/*

MANIFEST.in

@@ -1,7 +1,6 @@
 include LICENSE
 include *.db
 include *.js
-include scrapling/engines/toolbelt/bypasses/*.js
 include scrapling/*.db
 include scrapling/*.db*
 include scrapling/*.db-*

README.md

@@ -78,6 +78,7 @@ MySpider().start()
 
 # Platinum Sponsors
 
+<i><sub>Do you want to be the first company to show up here? Click [here](https://github.com/sponsors/D4Vinci/sponsorships?tier_id=586646)</sub></i>
 # Sponsors 
 
 <!-- sponsors -->
@@ -366,11 +367,20 @@ This installation only includes the parser engine and its dependencies, without
     ```bash
     pip install "scrapling[fetchers]"
     
-    scrapling install
+    scrapling install           # normal install
+    scrapling install  --force  # force reinstall
     ```
 
     This downloads all browsers, along with their system dependencies and fingerprint manipulation dependencies.
 
+    Or you can install them from the code instead of running a command like this:
+    ```python
+    from scrapling.cli import install
+    
+    install([], standalone_mode=False)          # normal install
+    install(["--force"], standalone_mode=False) # force reinstall
+    ```
+
 2. Extra features:
    - Install the MCP server feature:
        ```bash

docs/README_AR.md

@@ -1,3 +1,5 @@
+<!-- mcp-name: io.github.D4Vinci/Scrapling -->
+
 <h1 align="center">
     <a href="https://scrapling.readthedocs.io">
         <picture>
@@ -73,6 +75,7 @@ MySpider().start()
 
 # الرعاة البلاتينيون
 
+<i><sub>هل تريد أن تكون أول شركة تظهر هنا؟ انقر [هنا](https://github.com/sponsors/D4Vinci/sponsorships?tier_id=586646)</sub></i>
 # الرعاة
 
 <!-- sponsors -->
@@ -361,11 +364,20 @@ pip install scrapling
     ```bash
     pip install "scrapling[fetchers]"
 
-    scrapling install
+    scrapling install           # normal install
+    scrapling install  --force  # force reinstall
     ```
 
     يقوم هذا بتنزيل جميع المتصفحات، إلى جانب تبعيات النظام وتبعيات معالجة fingerprint الخاصة بها.
 
+    أو يمكنك تثبيتها من الكود بدلاً من تشغيل أمر كالتالي:
+    ```python
+    from scrapling.cli import install
+
+    install([], standalone_mode=False)          # normal install
+    install(["--force"], standalone_mode=False) # force reinstall
+    ```
+
 2. ميزات إضافية:
    - تثبيت ميزة خادم MCP:
        ```bash

docs/README_CN.md

@@ -1,3 +1,5 @@
+<!-- mcp-name: io.github.D4Vinci/Scrapling -->
+
 <h1 align="center">
     <a href="https://scrapling.readthedocs.io">
         <picture>
@@ -73,6 +75,7 @@ MySpider().start()
 
 # 铂金赞助商
 
+<i><sub>想成为第一个出现在这里的公司吗？点击[这里](https://github.com/sponsors/D4Vinci/sponsorships?tier_id=586646)</sub></i>
 # 赞助商
 
 <!-- sponsors -->
@@ -361,11 +364,20 @@ pip install scrapling
     ```bash
     pip install "scrapling[fetchers]"
 
-    scrapling install
+    scrapling install           # normal install
+    scrapling install  --force  # force reinstall
     ```
 
     这会下载所有浏览器，以及它们的系统依赖项和fingerprint操作依赖项。
 
+    或者你可以从代码中安装，而不是运行命令：
+    ```python
+    from scrapling.cli import install
+
+    install([], standalone_mode=False)          # normal install
+    install(["--force"], standalone_mode=False) # force reinstall
+    ```
+
 2. 额外功能：
    - 安装MCP服务器功能：
        ```bash

docs/README_DE.md

@@ -1,3 +1,5 @@
+<!-- mcp-name: io.github.D4Vinci/Scrapling -->
+
 <h1 align="center">
     <a href="https://scrapling.readthedocs.io">
         <picture>
@@ -73,6 +75,7 @@ MySpider().start()
 
 # Platin-Sponsoren
 
+<i><sub>Möchten Sie das erste Unternehmen sein, das hier erscheint? Klicken Sie [hier](https://github.com/sponsors/D4Vinci/sponsorships?tier_id=586646)</sub></i>
 # Sponsoren
 
 <!-- sponsors -->
@@ -361,11 +364,20 @@ Diese Installation enthält nur die Parser-Engine und ihre Abhängigkeiten, ohne
     ```bash
     pip install "scrapling[fetchers]"
 
-    scrapling install
+    scrapling install           # normal install
+    scrapling install  --force  # force reinstall
     ```
 
     Dies lädt alle Browser zusammen mit ihren Systemabhängigkeiten und Fingerprint-Manipulationsabhängigkeiten herunter.
 
+    Oder Sie können sie aus dem Code heraus installieren, anstatt einen Befehl auszuführen:
+    ```python
+    from scrapling.cli import install
+
+    install([], standalone_mode=False)          # normal install
+    install(["--force"], standalone_mode=False) # force reinstall
+    ```
+
 2. Zusätzliche Funktionen:
    - MCP-Server-Funktion installieren:
        ```bash

docs/README_ES.md

@@ -1,3 +1,5 @@
+<!-- mcp-name: io.github.D4Vinci/Scrapling -->
+
 <h1 align="center">
     <a href="https://scrapling.readthedocs.io">
         <picture>
@@ -73,6 +75,7 @@ MySpider().start()
 
 # Patrocinadores Platino
 
+<i><sub>¿Quieres ser la primera empresa en aparecer aquí? Haz clic [aquí](https://github.com/sponsors/D4Vinci/sponsorships?tier_id=586646)</sub></i>
 # Patrocinadores
 
 <!-- sponsors -->
@@ -361,11 +364,20 @@ Esta instalación solo incluye el motor de análisis y sus dependencias, sin nin
     ```bash
     pip install "scrapling[fetchers]"
 
-    scrapling install
+    scrapling install           # normal install
+    scrapling install  --force  # force reinstall
     ```
 
     Esto descarga todos los navegadores, junto con sus dependencias del sistema y dependencias de manipulación de fingerprint.
 
+    O puedes instalarlos desde el código en lugar de ejecutar un comando:
+    ```python
+    from scrapling.cli import install
+
+    install([], standalone_mode=False)          # normal install
+    install(["--force"], standalone_mode=False) # force reinstall
+    ```
+
 2. Características adicionales:
    - Instalar la característica del servidor MCP:
        ```bash

docs/README_JP.md

@@ -1,3 +1,5 @@
+<!-- mcp-name: io.github.D4Vinci/Scrapling -->
+
 <h1 align="center">
     <a href="https://scrapling.readthedocs.io">
         <picture>
@@ -73,6 +75,7 @@ MySpider().start()
 
 # プラチナスポンサー
 
+<i><sub>ここに最初に表示される企業になりませんか？[こちら](https://github.com/sponsors/D4Vinci/sponsorships?tier_id=586646)をクリック</sub></i>
 # スポンサー
 
 <!-- sponsors -->
@@ -361,11 +364,20 @@ pip install scrapling
     ```bash
     pip install "scrapling[fetchers]"
 
-    scrapling install
+    scrapling install           # normal install
+    scrapling install  --force  # force reinstall
     ```
 
     これにより、すべてのブラウザ、およびそれらのシステム依存関係とfingerprint操作依存関係がダウンロードされます。
 
+    または、コマンドを実行する代わりにコードからインストールすることもできます：
+    ```python
+    from scrapling.cli import install
+
+    install([], standalone_mode=False)          # normal install
+    install(["--force"], standalone_mode=False) # force reinstall
+    ```
+
 2. 追加機能：
    - MCPサーバー機能をインストール：
        ```bash

docs/README_RU.md

@@ -1,3 +1,5 @@
+<!-- mcp-name: io.github.D4Vinci/Scrapling -->
+
 <h1 align="center">
     <a href="https://scrapling.readthedocs.io">
         <picture>
@@ -73,6 +75,7 @@ MySpider().start()
 
 # Платиновые спонсоры
 
+<i><sub>Хотите стать первой компанией, которая появится здесь? Нажмите [здесь](https://github.com/sponsors/D4Vinci/sponsorships?tier_id=586646)</sub></i>
 # Спонсоры
 
 <!-- sponsors -->
@@ -361,11 +364,20 @@ pip install scrapling
     ```bash
     pip install "scrapling[fetchers]"
 
-    scrapling install
+    scrapling install           # normal install
+    scrapling install  --force  # force reinstall
     ```
 
     Это загрузит все браузеры вместе с их системными зависимостями и зависимостями для манипуляции fingerprint'ами.
 
+    Или вы можете установить их из кода вместо выполнения команды:
+    ```python
+    from scrapling.cli import install
+
+    install([], standalone_mode=False)          # normal install
+    install(["--force"], standalone_mode=False) # force reinstall
+    ```
+
 2. Дополнительные возможности:
    - Установить функцию MCP-сервера:
        ```bash

docs/ai/mcp-server.md

@@ -125,7 +125,7 @@ If you are using the Docker image, then it would be something like
 }
 ```
 
-The same logic applies to [Cursor](https://docs.cursor.com/en/context/mcp), [WindSurf](https://windsurf.com/university/tutorials/configuring-first-mcp-server), and others.
+The same logic applies to [Cursor](https://cursor.com/docs/context/mcp), [WindSurf](https://windsurf.com/university/tutorials/configuring-first-mcp-server), and others.
 
 ### Claude Code
 Here it's much simpler to do. If you have [Claude Code](https://www.anthropic.com/claude-code) installed, open the terminal and execute the following command:

docs/index.md

@@ -154,11 +154,20 @@ This installation only includes the parser engine and its dependencies, without
     ```bash
     pip install "scrapling[fetchers]"
     
-    scrapling install
+    scrapling install           # normal install
+    scrapling install  --force  # force reinstall
     ```
 
     This downloads all browsers, along with their system dependencies and fingerprint manipulation dependencies.
 
+    Or you can install them from the code instead of running a command like this:
+    ```python
+    from scrapling.cli import install
+    
+    install([], standalone_mode=False)          # normal install
+    install(["--force"], standalone_mode=False) # force reinstall
+    ```
+
 2. Extra features:
 
 

docs/requirements.txt

@@ -1,8 +1,8 @@
-zensical>=0.0.23
+zensical>=0.0.24
 mkdocstrings>=1.0.3
-mkdocstrings-python>=2.0.2
+mkdocstrings-python>=2.0.3
-griffe-inherited-docstrings
+griffe-inherited-docstrings>=1.1.3
-griffe-runtime-objects
+griffe-runtime-objects>=0.3.1
-griffe-sphinx
+griffe-sphinx>=0.2.1
 black>=26.1.0
 pngquant

pyproject.toml

@@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta"
 [project]
 name = "scrapling"
 # Static version instead of a dynamic version so we can get better layer caching while building docker, check the docker file to understand
-version = "0.4"
+version = "0.4.1"
 description = "Scrapling is an undetectable, powerful, flexible, high-performance Python library that makes Web Scraping easy and effortless as it should be!"
 readme = {file = "README.md", content-type = "text/markdown"}
 license = {file = "LICENSE"}
@@ -80,7 +80,7 @@ fetchers = [
     "anyio>=4.12.1"
 ]
 ai = [
-    "mcp>=1.24.0",
+    "mcp>=1.26.0",
     "markdownify>=1.2.0",
     "scrapling[fetchers]",
 ]

scrapling/__init__.py

@@ -1,5 +1,5 @@
 __author__ = "Karim Shoair (karim.shoair@pm.me)"
-__version__ = "0.4"
+__version__ = "0.4.1"
 __copyright__ = "Copyright (c) 2024 Karim Shoair"
 
 from typing import Any, TYPE_CHECKING

scrapling/core/ai.py

@@ -37,11 +37,25 @@ class ResponseModel(BaseModel):
     url: str = Field(description="The URL given by the user that resulted in this response.")
 
 
-def _ContentTranslator(content: Generator[str, None, None], page: _ScraplingResponse) -> ResponseModel:
+def _content_translator(content: Generator[str, None, None], page: _ScraplingResponse) -> ResponseModel:
     """Convert a content generator to a list of ResponseModel objects."""
     return ResponseModel(status=page.status, content=[result for result in content], url=page.url)
 
 
+def _normalize_credentials(credentials: Optional[Dict[str, str]]) -> Optional[Tuple[str, str]]:
+    """Convert a credentials dictionary to a tuple accepted by fetchers."""
+    if not credentials:
+        return None
+
+    username = credentials.get("username")
+    password = credentials.get("password")
+
+    if username is None or password is None:
+        raise ValueError("Credentials dictionary must contain both 'username' and 'password' keys")
+
+    return username, password
+
+
 class ScraplingMCPServer:
     @staticmethod
     def get(
@@ -50,17 +64,17 @@ class ScraplingMCPServer:
         extraction_type: extraction_types = "markdown",
         css_selector: Optional[str] = None,
         main_content_only: bool = True,
-        params: Optional[Dict | List | Tuple] = None,
+        params: Optional[Dict] = None,
         headers: Optional[Mapping[str, Optional[str]]] = None,
-        cookies: Optional[Dict[str, str] | list[tuple[str, str]]] = None,
+        cookies: Optional[Dict[str, str]] = None,
         timeout: Optional[int | float] = 30,
         follow_redirects: bool = True,
         max_redirects: int = 30,
         retries: Optional[int] = 3,
         retry_delay: Optional[int] = 1,
         proxy: Optional[str] = None,
-        proxy_auth: Optional[Tuple[str, str]] = None,
+        proxy_auth: Optional[Dict[str, str]] = None,
-        auth: Optional[Tuple[str, str]] = None,
+        auth: Optional[Dict[str, str]] = None,
         verify: Optional[bool] = True,
         http3: Optional[bool] = False,
         stealthy_headers: Optional[bool] = True,
@@ -87,20 +101,23 @@ class ScraplingMCPServer:
         :param retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second.
         :param proxy: Proxy URL to use. Format: "http://username:password@localhost:8030".
                      Cannot be used together with the `proxies` parameter.
-        :param proxy_auth: HTTP basic auth for proxy, tuple of (username, password).
+        :param proxy_auth: HTTP basic auth for proxy in dictionary format with `username` and `password` keys.
-        :param auth: HTTP basic auth tuple of (username, password). Only basic auth is supported.
+        :param auth: HTTP basic auth in dictionary format with `username` and `password` keys.
         :param verify: Whether to verify HTTPS certificates.
         :param http3: Whether to use HTTP3. Defaults to False. It might be problematic if used it with `impersonate`.
         :param stealthy_headers: If enabled (default), it creates and adds real browser headers. It also sets the referer header as if this request came from a Google search of URL's domain.
         """
+        normalized_proxy_auth = _normalize_credentials(proxy_auth)
+        normalized_auth = _normalize_credentials(auth)
+
         page = Fetcher.get(
             url,
-            auth=auth,
+            auth=normalized_auth,
             proxy=proxy,
             http3=http3,
             verify=verify,
             params=params,
-            proxy_auth=proxy_auth,
+            proxy_auth=normalized_proxy_auth,
             retry_delay=retry_delay,
             stealthy_headers=stealthy_headers,
             impersonate=impersonate,
@@ -111,7 +128,7 @@ class ScraplingMCPServer:
             max_redirects=max_redirects,
             follow_redirects=follow_redirects,
         )
-        return _ContentTranslator(
+        return _content_translator(
             Convertor._extract_content(
                 page,
                 css_selector=css_selector,
@@ -123,22 +140,22 @@ class ScraplingMCPServer:
 
     @staticmethod
     async def bulk_get(
-        urls: Tuple[str, ...],
+        urls: List[str],
         impersonate: ImpersonateType = "chrome",
         extraction_type: extraction_types = "markdown",
         css_selector: Optional[str] = None,
         main_content_only: bool = True,
-        params: Optional[Dict | List | Tuple] = None,
+        params: Optional[Dict] = None,
         headers: Optional[Mapping[str, Optional[str]]] = None,
-        cookies: Optional[Dict[str, str] | list[tuple[str, str]]] = None,
+        cookies: Optional[Dict[str, str]] = None,
         timeout: Optional[int | float] = 30,
         follow_redirects: bool = True,
         max_redirects: int = 30,
         retries: Optional[int] = 3,
         retry_delay: Optional[int] = 1,
         proxy: Optional[str] = None,
-        proxy_auth: Optional[Tuple[str, str]] = None,
+        proxy_auth: Optional[Dict[str, str]] = None,
-        auth: Optional[Tuple[str, str]] = None,
+        auth: Optional[Dict[str, str]] = None,
         verify: Optional[bool] = True,
         http3: Optional[bool] = False,
         stealthy_headers: Optional[bool] = True,
@@ -147,7 +164,7 @@ class ScraplingMCPServer:
         Note: This is only suitable for low-mid protection levels. For high-protection levels or websites that require JS loading, use the other tools directly.
         Note: If the `css_selector` resolves to more than one element, all the elements will be returned.
 
-        :param urls: A tuple of the URLs to request.
+        :param urls: A list of the URLs to request.
         :param impersonate: Browser version to impersonate its fingerprint. It's using the latest chrome version by default.
         :param extraction_type: The type of content to extract from the page. Defaults to "markdown". Options are:
             - Markdown will convert the page content to Markdown format.
@@ -165,17 +182,20 @@ class ScraplingMCPServer:
         :param retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second.
         :param proxy: Proxy URL to use. Format: "http://username:password@localhost:8030".
                      Cannot be used together with the `proxies` parameter.
-        :param proxy_auth: HTTP basic auth for proxy, tuple of (username, password).
+        :param proxy_auth: HTTP basic auth for proxy in dictionary format with `username` and `password` keys.
-        :param auth: HTTP basic auth tuple of (username, password). Only basic auth is supported.
+        :param auth: HTTP basic auth in dictionary format with `username` and `password` keys.
         :param verify: Whether to verify HTTPS certificates.
         :param http3: Whether to use HTTP3. Defaults to False. It might be problematic if used it with `impersonate`.
         :param stealthy_headers: If enabled (default), it creates and adds real browser headers. It also sets the referer header as if this request came from a Google search of URL's domain.
         """
+        normalized_proxy_auth = _normalize_credentials(proxy_auth)
+        normalized_auth = _normalize_credentials(auth)
+
         async with FetcherSession() as session:
             tasks: List[Any] = [
                 session.get(
                     url,
-                    auth=auth,
+                    auth=normalized_auth,
                     proxy=proxy,
                     http3=http3,
                     verify=verify,
@@ -184,7 +204,7 @@ class ScraplingMCPServer:
                     cookies=cookies,
                     timeout=timeout,
                     retries=retries,
-                    proxy_auth=proxy_auth,
+                    proxy_auth=normalized_proxy_auth,
                     retry_delay=retry_delay,
                     impersonate=impersonate,
                     max_redirects=max_redirects,
@@ -195,7 +215,7 @@ class ScraplingMCPServer:
             ]
             responses = await gather(*tasks)
             return [
-                _ContentTranslator(
+                _content_translator(
                     Convertor._extract_content(
                         page,
                         css_selector=css_selector,
@@ -279,7 +299,7 @@ class ScraplingMCPServer:
             disable_resources=disable_resources,
             wait_selector_state=wait_selector_state,
         )
-        return _ContentTranslator(
+        return _content_translator(
             Convertor._extract_content(
                 page,
                 css_selector=css_selector,
@@ -291,7 +311,7 @@ class ScraplingMCPServer:
 
     @staticmethod
     async def bulk_fetch(
-        urls: Tuple[str, ...],
+        urls: List[str],
         extraction_type: extraction_types = "markdown",
         css_selector: Optional[str] = None,
         main_content_only: bool = True,
@@ -316,7 +336,7 @@ class ScraplingMCPServer:
         Note: This is only suitable for low-mid protection levels.
         Note: If the `css_selector` resolves to more than one element, all the elements will be returned.
 
-        :param urls: A tuple of the URLs to request.
+        :param urls: A list of the URLs to request.
         :param extraction_type: The type of content to extract from the page. Defaults to "markdown". Options are:
             - Markdown will convert the page content to Markdown format.
             - HTML will return the raw HTML content of the page.
@@ -364,7 +384,7 @@ class ScraplingMCPServer:
             tasks = [session.fetch(url) for url in urls]
             responses = await gather(*tasks)
             return [
-                _ContentTranslator(
+                _content_translator(
                     Convertor._extract_content(
                         page,
                         css_selector=css_selector,
@@ -463,7 +483,7 @@ class ScraplingMCPServer:
             disable_resources=disable_resources,
             wait_selector_state=wait_selector_state,
         )
-        return _ContentTranslator(
+        return _content_translator(
             Convertor._extract_content(
                 page,
                 css_selector=css_selector,
@@ -475,7 +495,7 @@ class ScraplingMCPServer:
 
     @staticmethod
     async def bulk_stealthy_fetch(
-        urls: Tuple[str, ...],
+        urls: List[str],
         extraction_type: extraction_types = "markdown",
         css_selector: Optional[str] = None,
         main_content_only: bool = True,
@@ -505,7 +525,7 @@ class ScraplingMCPServer:
         Note: This is the only suitable fetcher for high protection levels.
         Note: If the `css_selector` resolves to more than one element, all the elements will be returned.
 
-        :param urls: A tuple of the URLs to request.
+        :param urls: A list of the URLs to request.
         :param extraction_type: The type of content to extract from the page. Defaults to "markdown". Options are:
             - Markdown will convert the page content to Markdown format.
             - HTML will return the raw HTML content of the page.
@@ -562,7 +582,7 @@ class ScraplingMCPServer:
             tasks = [session.fetch(url) for url in urls]
             responses = await gather(*tasks)
             return [
-                _ContentTranslator(
+                _content_translator(
                     Convertor._extract_content(
                         page,
                         css_selector=css_selector,

scrapling/core/shell.py

@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 from sys import stderr
+from copy import deepcopy
 from functools import wraps
 from re import sub as re_sub
 from collections import namedtuple
@@ -571,6 +572,14 @@ class Convertor:
 
         return markdownify(body)
 
+    @classmethod
+    def _strip_noise_tags(cls, page: Selector) -> Selector:
+        """Return a copy of the Selector with noise tags removed."""
+        clean_root = deepcopy(page._root)
+        for element in clean_root.iter(*{"script", "style", "noscript", "svg"}):
+            element.drop_tree()
+        return Selector(root=clean_root, url=page.url)
+
     @classmethod
     def _extract_content(
         cls,
@@ -587,6 +596,7 @@ class Convertor:
         else:
             if main_content_only:
                 page = cast(Selector, page.css("body").first) or page
+                page = cls._strip_noise_tags(page)
 
             pages = [page] if not css_selector else cast(Selectors, page.css(css_selector))
             for page in pages:
@@ -596,7 +606,9 @@ class Convertor:
                     case "html":
                         yield page.html_content
                     case "text":
-                        txt_content = page.get_all_text(strip=True)
+                        txt_content = page.get_all_text(
+                            strip=True, ignore_tags=("script", "style", "noscript", "svg", "iframe")
+                        )
                         for s in (
                             "\n",
                             "\r",

scrapling/engines/_browsers/_config_tools.py

@@ -1,38 +1,4 @@
-from functools import lru_cache
-
-from scrapling.engines.toolbelt.navigation import js_bypass_path
 from scrapling.engines.toolbelt.fingerprints import generate_headers
 
 __default_useragent__ = generate_headers(browser_mode=True).get("User-Agent")
 __default_chrome_useragent__ = generate_headers(browser_mode="chrome").get("User-Agent")
-
-
-@lru_cache(1)
-def _compiled_stealth_scripts():
-    """Pre-read and compile stealth scripts"""
-    # Basic bypasses nothing fancy as I'm still working on it
-    # But with adding these bypasses to the above config, it bypasses many online tests like
-    # https://bot.sannysoft.com/
-    # https://kaliiiiiiiiii.github.io/brotector/
-    # https://pixelscan.net/
-    # https://iphey.com/
-    # https://www.browserscan.net/bot-detection <== this one also checks for the CDP runtime fingerprint
-    # https://arh.antoinevastel.com/bots/areyouheadless/
-    # https://prescience-data.github.io/execution-monitor.html
-    stealth_scripts_paths = tuple(
-        js_bypass_path(script)
-        for script in (
-            # Order is important
-            "webdriver_fully.js",
-            "window_chrome.js",
-            "navigator_plugins.js",
-            "notification_permission.js",
-            "screen_props.js",
-            "playwright_fingerprint.js",
-        )
-    )
-    scripts = []
-    for script_path in stealth_scripts_paths:
-        with open(script_path, "r") as f:
-            scripts.append(f.read())
-    return tuple(scripts)

scrapling/engines/_browsers/_stealth.py

@@ -17,12 +17,11 @@ from scrapling.core._types import Any, Optional, ProxyType, Unpack
 from scrapling.engines.toolbelt.proxy_rotation import is_proxy_error
 from scrapling.engines.toolbelt.convertor import Response, ResponseFactory
 from scrapling.engines.toolbelt.fingerprints import generate_convincing_referer
-from scrapling.engines._browsers._config_tools import _compiled_stealth_scripts
 from scrapling.engines._browsers._types import StealthSession, StealthFetchParams
 from scrapling.engines._browsers._base import SyncSession, AsyncSession, StealthySessionMixin
 from scrapling.engines._browsers._validators import validate_fetch as _validate, StealthConfig
 
-__CF_PATTERN__ = re_compile("challenges.cloudflare.com/cdn-cgi/challenge-platform/.*")
+__CF_PATTERN__ = re_compile(r"^https?://challenges\.cloudflare\.com/cdn-cgi/challenge-platform/.*")
 
 
 class StealthySession(SyncSession, StealthySessionMixin):
@@ -109,14 +108,6 @@ class StealthySession(SyncSession, StealthySessionMixin):
         else:
             raise RuntimeError("Session has been already started")
 
-    def _initialize_context(self, config, ctx: BrowserContext) -> BrowserContext:
-        """Initialize the browser context."""
-        for script in _compiled_stealth_scripts():
-            ctx.add_init_script(script=script)
-
-        ctx = super()._initialize_context(config, ctx)
-        return ctx
-
     def _cloudflare_solver(self, page: Page) -> None:  # pragma: no cover
         """Solve the cloudflare challenge displayed on the playwright page passed
 
@@ -127,7 +118,7 @@ class StealthySession(SyncSession, StealthySessionMixin):
         challenge_type = self._detect_cloudflare(ResponseFactory._get_page_content(page))
         if not challenge_type:
             log.error("No Cloudflare challenge found.")
-            return
+            return None
         else:
             log.info(f'The turnstile version discovered is "{challenge_type}"')
             if challenge_type == "non-interactive":
@@ -136,7 +127,7 @@ class StealthySession(SyncSession, StealthySessionMixin):
                     page.wait_for_timeout(1000)
                     page.wait_for_load_state()
                 log.info("Cloudflare captcha is solved")
-                return
+                return None
 
             else:
                 box_selector = "#cf_turnstile div, #cf-turnstile div, .turnstile>div>div"
@@ -161,7 +152,7 @@ class StealthySession(SyncSession, StealthySessionMixin):
                 if not iframe or not outer_box:
                     if "<title>Just a moment...</title>" not in (ResponseFactory._get_page_content(page)):
                         log.info("Cloudflare captcha is solved")
-                        return
+                        return None
 
                     outer_box = page.locator(box_selector).last.bounding_box()
 
@@ -171,22 +162,28 @@ class StealthySession(SyncSession, StealthySessionMixin):
                 # Move the mouse to the center of the window, then press and hold the left mouse button
                 page.mouse.click(captcha_x, captcha_y, delay=randint(100, 200), button="left")
                 self._wait_for_networkidle(page)
-                if iframe is not None:
+
-                    # Wait for the frame to be removed from the page (with 30s timeout = 300 iterations * 100 ms)
+                if challenge_type != "embedded":
                     attempts = 0
-                    while iframe in page.frames:
+                    while "<title>Just a moment...</title>" in ResponseFactory._get_page_content(page):
-                        if attempts >= 300:
+                        # Wait for the page
-                            log.info("Cloudflare iframe didn't disappear after 30s, continuing...")
+                        if attempts >= 100:
+                            log.info("Cloudflare page didn't disappear after 10s, continuing...")
                             break
                         page.wait_for_timeout(100)
                         attempts += 1
-                if challenge_type != "embedded":
+
-                    page.locator(box_selector).last.wait_for(state="detached")
+                    # page.locator(box_selector).last.wait_for(state="detached")
-                    page.locator(".zone-name-title").wait_for(state="hidden")
+                    # page.locator(".zone-name-title").wait_for(state="hidden")
+
                 self._wait_for_page_stability(page, True, False)
 
-                log.info("Cloudflare captcha is solved")
+                if "<title>Just a moment...</title>" not in (ResponseFactory._get_page_content(page)):
-                return
+                    log.info("Cloudflare captcha is solved")
+                    return None
+                else:
+                    log.info("Looks like Cloudflare captcha is still present, solving again")
+                    return self._cloudflare_solver(page)
 
     def fetch(self, url: str, **kwargs: Unpack[StealthFetchParams]) -> Response:
         """Opens up the browser and do your request based on your chosen options.
@@ -366,14 +363,6 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin):
         else:
             raise RuntimeError("Session has been already started")
 
-    async def _initialize_context(self, config: Any, ctx: AsyncBrowserContext) -> AsyncBrowserContext:
-        """Initialize the browser context."""
-        for script in _compiled_stealth_scripts():
-            await ctx.add_init_script(script=script)
-
-        ctx = await super()._initialize_context(config, ctx)
-        return ctx
-
     async def _cloudflare_solver(self, page: async_Page) -> None:  # pragma: no cover
         """Solve the cloudflare challenge displayed on the playwright page passed
 
@@ -384,7 +373,7 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin):
         challenge_type = self._detect_cloudflare(await ResponseFactory._get_async_page_content(page))
         if not challenge_type:
             log.error("No Cloudflare challenge found.")
-            return
+            return None
         else:
             log.info(f'The turnstile version discovered is "{challenge_type}"')
             if challenge_type == "non-interactive":
@@ -393,7 +382,7 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin):
                     await page.wait_for_timeout(1000)
                     await page.wait_for_load_state()
                 log.info("Cloudflare captcha is solved")
-                return
+                return None
 
             else:
                 box_selector = "#cf_turnstile div, #cf-turnstile div, .turnstile>div>div"
@@ -418,7 +407,7 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin):
                 if not iframe or not outer_box:
                     if "<title>Just a moment...</title>" not in (await ResponseFactory._get_async_page_content(page)):
                         log.info("Cloudflare captcha is solved")
-                        return
+                        return None
 
                     outer_box = await page.locator(box_selector).last.bounding_box()
 
@@ -428,22 +417,28 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin):
                 # Move the mouse to the center of the window, then press and hold the left mouse button
                 await page.mouse.click(captcha_x, captcha_y, delay=randint(100, 200), button="left")
                 await self._wait_for_networkidle(page)
-                if iframe is not None:
+
-                    # Wait for the frame to be removed from the page (with 30s timeout = 300 iterations * 100 ms)
+                if challenge_type != "embedded":
                     attempts = 0
-                    while iframe in page.frames:
+                    while "<title>Just a moment...</title>" in (await ResponseFactory._get_async_page_content(page)):
-                        if attempts >= 300:
+                        # Wait for the page
-                            log.info("Cloudflare iframe didn't disappear after 30s, continuing...")
+                        if attempts >= 100:
+                            log.info("Cloudflare page didn't disappear after 10s, continuing...")
                             break
                         await page.wait_for_timeout(100)
                         attempts += 1
-                if challenge_type != "embedded":
+
-                    await page.locator(box_selector).last.wait_for(state="detached")
+                    # await page.locator(box_selector).last.wait_for(state="detached")
-                    await page.locator(".zone-name-title").wait_for(state="hidden")
+                    # await page.locator(".zone-name-title").wait_for(state="hidden")
+
                 await self._wait_for_page_stability(page, True, False)
 
-                log.info("Cloudflare captcha is solved")
+                if "<title>Just a moment...</title>" not in (await ResponseFactory._get_async_page_content(page)):
-                return
+                    log.info("Cloudflare captcha is solved")
+                    return None
+                else:
+                    log.info("Looks like Cloudflare captcha is still present, solving again")
+                    return await self._cloudflare_solver(page)
 
     async def fetch(self, url: str, **kwargs: Unpack[StealthFetchParams]) -> Response:
         """Opens up the browser and do your request based on your chosen options.

scrapling/engines/toolbelt/bypasses/navigator_plugins.js

@@ -1,40 +0,0 @@
-if(navigator.plugins.length == 0){
-    Object.defineProperty(navigator, 'plugins', {
-        get: () => {
-            const PDFViewerPlugin = Object.create(Plugin.prototype, {
-                description: { value: 'Portable Document Format', enumerable: false },
-                filename: { value: 'internal-pdf-viewer', enumerable: false },
-                name: { value: 'PDF Viewer', enumerable: false },
-            });
-            const ChromePDFViewer = Object.create(Plugin.prototype, {
-                description: { value: 'Portable Document Format', enumerable: false },
-                filename: { value: 'internal-pdf-viewer', enumerable: false },
-                name: { value: 'Chrome PDF Viewer', enumerable: false },
-            });
-            const ChromiumPDFViewer = Object.create(Plugin.prototype, {
-                description: { value: 'Portable Document Format', enumerable: false },
-                filename: { value: 'internal-pdf-viewer', enumerable: false },
-                name: { value: 'Chromium PDF Viewer', enumerable: false },
-            });
-            const EdgePDFViewer = Object.create(Plugin.prototype, {
-                description: { value: 'Portable Document Format', enumerable: false },
-                filename: { value: 'internal-pdf-viewer', enumerable: false },
-                name: { value: 'Microsoft Edge PDF Viewer', enumerable: false },
-            });
-            const WebKitPDFPlugin = Object.create(Plugin.prototype, {
-                description: { value: 'Portable Document Format', enumerable: false },
-                filename: { value: 'internal-pdf-viewer', enumerable: false },
-                name: { value: 'WebKit built-in PDF', enumerable: false },
-            });
-
-            return Object.create(PluginArray.prototype, {
-                length: { value: 5 },
-                0: { value: PDFViewerPlugin },
-                1: { value: ChromePDFViewer },
-                2: { value: ChromiumPDFViewer },
-                3: { value: EdgePDFViewer },
-                4: { value: WebKitPDFPlugin },
-            });
-        },
-    });
-}

scrapling/engines/toolbelt/bypasses/notification_permission.js

@@ -1,5 +0,0 @@
-// Bypasses `notificationIsDenied` test in creepsjs's 'Like Headless' sections
-const isSecure = document.location.protocol.startsWith('https')
-if (isSecure){
-    Object.defineProperty(Notification, 'permission', {get: () => 'default'})
-}

scrapling/engines/toolbelt/bypasses/playwright_fingerprint.js

@@ -1,3 +0,0 @@
-// Remove playwright fingerprint => https://github.com/microsoft/playwright/commit/c9e673c6dca746384338ab6bb0cf63c7e7caa9b2#diff-087773eea292da9db5a3f27de8f1a2940cdb895383ad750c3cd8e01772a35b40R915
-delete window.__pwInitScripts;
-delete window.__playwright__binding__;

scrapling/engines/toolbelt/bypasses/screen_props.js

@@ -1,27 +0,0 @@
-const windowScreenProps = {
-    // Dimensions
-    innerHeight: 0,
-    innerWidth: 0,
-    outerHeight: 754,
-    outerWidth: 1313,
-
-    // Position
-    screenX: 19,
-    pageXOffset: 0,
-    pageYOffset: 0,
-
-    // Display
-    devicePixelRatio: 2
-};
-
-try {
-    for (const [prop, value] of Object.entries(windowScreenProps)) {
-        if (value > 0) {
-            // The 0 values are introduced by collecting in the hidden iframe.
-            // They are document sizes anyway so no need to test them or inject them.
-            window[prop] = value;
-        }
-    }
-} catch (e) {
-    console.warn(e);
-};

scrapling/engines/toolbelt/bypasses/webdriver_fully.js

@@ -1,27 +0,0 @@
-// Create a function that looks like a native getter
-const nativeGetter = function get webdriver() {
-    return false;
-};
-
-// Copy over native function properties
-Object.defineProperties(nativeGetter, {
-    name: { value: 'get webdriver', configurable: true },
-    length: { value: 0, configurable: true },
-    toString: {
-        value: function() {
-            return `function get webdriver() { [native code] }`;
-        },
-        configurable: true
-    }
-});
-
-// Make it look native
-Object.setPrototypeOf(nativeGetter, Function.prototype);
-
-// Apply the modified descriptor
-Object.defineProperty(Navigator.prototype, 'webdriver', {
-    get: nativeGetter,
-    set: undefined,
-    enumerable: true,
-    configurable: true
-});

scrapling/engines/toolbelt/bypasses/window_chrome.js

@@ -1,213 +0,0 @@
-// To escape `HEADCHR_CHROME_OBJ` test in headless mode => https://github.com/antoinevastel/fp-collect/blob/master/src/fpCollect.js#L322
-// Faking window.chrome fully
-
-if (!window.chrome) {
-    // First, save all existing properties
-    const originalKeys = Object.getOwnPropertyNames(window);
-    const tempObj = {};
-
-    // Recreate all properties in original order
-    for (const key of originalKeys) {
-        const descriptor = Object.getOwnPropertyDescriptor(window, key);
-        const value = window[key];
-        // delete window[key];
-        Object.defineProperty(tempObj, key, descriptor);
-    }
-
-    // Use the exact property descriptor found in headful Chrome
-    // fetch it via `Object.getOwnPropertyDescriptor(window, 'chrome')`
-    const mockChrome = {
-        loadTimes: {},
-        csi: {},
-        app: {
-            isInstalled: false
-        },
-        // Add other Chrome-specific properties
-    };
-
-    Object.defineProperty(tempObj, 'chrome', {
-        writable: true,
-        enumerable: true,
-        configurable: false,
-        value: mockChrome
-    });
-    for (const key of Object.getOwnPropertyNames(tempObj)) {
-        try {
-            Object.defineProperty(window, key,
-                Object.getOwnPropertyDescriptor(tempObj, key));
-        } catch (e) {}
-    };
-    // todo: solve this
-    // Using line below bypasses the hasHighChromeIndex test in creepjs ==> https://github.com/abrahamjuliot/creepjs/blob/master/src/headless/index.ts#L121
-    // Chrome object have to be in the end of the window properties
-    // Object.assign(window, tempObj);
-    // But makes window.chrome unreadable on 'https://bot.sannysoft.com/'
-}
-
-// That means we're running headful and don't need to mock anything
-if ('app' in window.chrome) {
-    return; // Nothing to do here
-}
-const makeError = {
-    ErrorInInvocation: fn => {
-        const err = new TypeError(`Error in invocation of app.${fn}()`);
-        return utils.stripErrorWithAnchor(
-            err,
-            `at ${fn} (eval at <anonymous>`,
-        );
-    },
-};
-// check with: `JSON.stringify(window.chrome['app'])`
-const STATIC_DATA = JSON.parse(
-    `
-{
-  "isInstalled": false,
-  "InstallState": {
-    "DISABLED": "disabled",
-    "INSTALLED": "installed",
-    "NOT_INSTALLED": "not_installed"
-  },
-  "RunningState": {
-    "CANNOT_RUN": "cannot_run",
-    "READY_TO_RUN": "ready_to_run",
-    "RUNNING": "running"
-  }
-}
-    `.trim(),
-    );
-window.chrome.app = {
-    ...STATIC_DATA,
-
-    get isInstalled() {
-        return false;
-    },
-
-    getDetails: function getDetails() {
-        if (arguments.length) {
-            throw makeError.ErrorInInvocation(`getDetails`);
-        }
-        return null;
-    },
-    getIsInstalled: function getDetails() {
-        if (arguments.length) {
-            throw makeError.ErrorInInvocation(`getIsInstalled`);
-        }
-        return false;
-    },
-    runningState: function getDetails() {
-        if (arguments.length) {
-            throw makeError.ErrorInInvocation(`runningState`);
-        }
-        return 'cannot_run';
-    },
-};
-// Check that the Navigation Timing API v1 is available, we need that
-if (!window.performance || !window.performance.timing) {
-    return;
-}
-const {timing} = window.performance;
-window.chrome.csi = function () {
-    return {
-        onloadT: timing.domContentLoadedEventEnd,
-        startE: timing.navigationStart,
-        pageT: Date.now() - timing.navigationStart,
-        tran: 15, // Transition type or something
-    };
-};
-if (!window.PerformancePaintTiming){
-    return;
-}
-const {performance} = window;
-// Some stuff is not available on about:blank as it requires a navigation to occur,
-// let's harden the code to not fail then:
-const ntEntryFallback = {
-    nextHopProtocol: 'h2',
-    type: 'other',
-};
-
-// The API exposes some funky info regarding the connection
-const protocolInfo = {
-    get connectionInfo() {
-        const ntEntry =
-            performance.getEntriesByType('navigation')[0] || ntEntryFallback;
-        return ntEntry.nextHopProtocol;
-    },
-    get npnNegotiatedProtocol() {
-        // NPN is deprecated in favor of ALPN, but this implementation returns the
-        // HTTP/2 or HTTP2+QUIC/39 requests negotiated via ALPN.
-        const ntEntry =
-            performance.getEntriesByType('navigation')[0] || ntEntryFallback;
-        return ['h2', 'hq'].includes(ntEntry.nextHopProtocol)
-            ? ntEntry.nextHopProtocol
-            : 'unknown';
-    },
-    get navigationType() {
-        const ntEntry =
-            performance.getEntriesByType('navigation')[0] || ntEntryFallback;
-        return ntEntry.type;
-    },
-    get wasAlternateProtocolAvailable() {
-        // The Alternate-Protocol header is deprecated in favor of Alt-Svc
-        // (https://www.mnot.net/blog/2016/03/09/alt-svc), so technically this
-        // should always return false.
-        return false;
-    },
-    get wasFetchedViaSpdy() {
-        // SPDY is deprecated in favor of HTTP/2, but this implementation returns
-        // true for HTTP/2 or HTTP2+QUIC/39 as well.
-        const ntEntry =
-            performance.getEntriesByType('navigation')[0] || ntEntryFallback;
-        return ['h2', 'hq'].includes(ntEntry.nextHopProtocol);
-    },
-    get wasNpnNegotiated() {
-        // NPN is deprecated in favor of ALPN, but this implementation returns true
-        // for HTTP/2 or HTTP2+QUIC/39 requests negotiated via ALPN.
-        const ntEntry =
-            performance.getEntriesByType('navigation')[0] || ntEntryFallback;
-        return ['h2', 'hq'].includes(ntEntry.nextHopProtocol);
-    },
-};
-
-// Truncate number to specific number of decimals, most of the `loadTimes` stuff has 3
-function toFixed(num, fixed) {
-    var re = new RegExp('^-?\\d+(?:.\\d{0,' + (fixed || -1) + '})?');
-    return num.toString().match(re)[0];
-}
-
-const timingInfo = {
-    get firstPaintAfterLoadTime() {
-        // This was never actually implemented and always returns 0.
-        return 0;
-    },
-    get requestTime() {
-        return timing.navigationStart / 1000;
-    },
-    get startLoadTime() {
-        return timing.navigationStart / 1000;
-    },
-    get commitLoadTime() {
-        return timing.responseStart / 1000;
-    },
-    get finishDocumentLoadTime() {
-        return timing.domContentLoadedEventEnd / 1000;
-    },
-    get finishLoadTime() {
-        return timing.loadEventEnd / 1000;
-    },
-    get firstPaintTime() {
-        const fpEntry = performance.getEntriesByType('paint')[0] || {
-            startTime: timing.loadEventEnd / 1000, // Fallback if no navigation occured (`about:blank`)
-        };
-        return toFixed(
-            (fpEntry.startTime + performance.timeOrigin) / 1000,
-            3,
-        );
-    },
-};
-
-window.chrome.loadTimes = function () {
-    return {
-        ...protocolInfo,
-        ...timingInfo,
-    };
-};

scrapling/engines/toolbelt/navigation.py

@@ -2,8 +2,6 @@
 Functions related to files and URLs
 """
 
-from pathlib import Path
-from functools import lru_cache
 from urllib.parse import urlparse
 
 from playwright.async_api import Route as async_Route
@@ -14,8 +12,6 @@ from scrapling.core.utils import log
 from scrapling.core._types import Dict, Set, Tuple, Optional, Callable
 from scrapling.engines.constants import EXTRA_RESOURCES
 
-__BYPASSES_DIR__ = Path(__file__).parent / "bypasses"
-
 
 class ProxyDict(Struct):
     server: str
@@ -111,13 +107,3 @@ def construct_proxy_dict(proxy_string: str | Dict[str, str] | Tuple) -> Dict:
             raise TypeError(f"Invalid proxy dictionary: {e}")
 
     raise TypeError(f"Invalid proxy string: {proxy_string}")
-
-
-@lru_cache(10, typed=True)
-def js_bypass_path(filename: str) -> str:
-    """Takes the base filename of a JS file inside the `bypasses` folder, then return the full path of it
-
-    :param filename: The base filename of the JS file.
-    :return: The full path of the JS file.
-    """
-    return str(__BYPASSES_DIR__ / filename)

server.json

@@ -0,0 +1,50 @@
+{
+  "$schema": "https://static.modelcontextprotocol.io/schemas/2025-12-11/server.schema.json",
+  "name": "io.github.D4Vinci/Scrapling",
+  "title": "Scrapling MCP Server",
+  "description": "Web scraping with stealth HTTP, real browsers, and Cloudflare bypass. CSS selectors supported.",
+  "websiteUrl": "https://scrapling.readthedocs.io/en/latest/ai/mcp-server.html",
+  "repository": {
+    "url": "https://github.com/D4Vinci/Scrapling",
+    "source": "github"
+  },
+  "icons": [
+    {
+      "src": "https://raw.githubusercontent.com/D4Vinci/Scrapling/main/docs/assets/logo.png",
+      "mimeType": "image/png"
+    }
+  ],
+  "version": "0.4.1",
+  "packages": [
+    {
+      "registryType": "pypi",
+      "identifier": "scrapling",
+      "version": "0.4.1",
+      "runtimeHint": "uvx",
+      "packageArguments": [
+        {
+          "type": "positional",
+          "valueHint": "mcp",
+          "isFixed": true
+        }
+      ],
+      "transport": {
+        "type": "stdio"
+      }
+    },
+    {
+      "registryType": "oci",
+      "identifier": "ghcr.io/d4vinci/scrapling",
+      "packageArguments": [
+        {
+          "type": "positional",
+          "valueHint": "mcp",
+          "isFixed": true
+        }
+      ],
+      "transport": {
+        "type": "stdio"
+      }
+    }
+  ]
+}

setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = scrapling
-version = 0.4
+version = 0.4.1
 author = Karim Shoair
 author_email = karim.shoair@pm.me
 description = Scrapling is an undetectable, powerful, flexible, high-performance Python library that makes Web Scraping easy and effortless as it should be!

tests/fetchers/test_utils.py

@@ -1,12 +1,10 @@
 import pytest
-from pathlib import Path
 
 from scrapling.engines.toolbelt.custom import StatusText, Response
 from scrapling.engines.toolbelt.navigation import (
     construct_proxy_dict,
     create_intercept_handler,
     create_async_intercept_handler,
-    js_bypass_path,
 )
 from scrapling.engines.toolbelt.fingerprints import (
     generate_convincing_referer,
@@ -203,25 +201,6 @@ class TestConstructProxyDict:
             construct_proxy_dict({"invalid": "structure"})
 
 
-class TestJsBypassPath:
-    """Test JavaScript bypass path utility"""
-
-    def test_js_bypass_path(self):
-        """Test getting JavaScript bypass file path"""
-        result = js_bypass_path("webdriver_fully.js")
-
-        assert isinstance(result, str)
-        assert result.endswith("webdriver_fully.js")
-        assert Path(result).exists()
-
-    def test_js_bypass_path_caching(self):
-        """Test that js_bypass_path is cached"""
-        result1 = js_bypass_path("webdriver_fully.js")
-        result2 = js_bypass_path("webdriver_fully.js")
-
-        assert result1 == result2
-
-
 class TestFingerprintFunctions:
     """Test fingerprint generation functions"""