v0.4.7 (#251)

2026-04-17 23:09:43 +02:00
parent 11e038768b 2a1e9e22f2
commit 54a080aae1
16 changed files with 892 additions and 35 deletions
@@ -14,7 +14,7 @@
 <p align="center">
    <a href="https://trendshift.io/repositories/14244" target="_blank"><img src="https://trendshift.io/api/badge/repositories/14244" alt="D4Vinci%2FScrapling | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
    <br/>
-    <a href="https://github.com/D4Vinci/Scrapling/blob/main/docs/README_AR.md">العربيه</a> | <a href="https://github.com/D4Vinci/Scrapling/blob/main/docs/README_ES.md">Español</a> | <a href="https://github.com/D4Vinci/Scrapling/blob/main/docs/README_FR.md">Français</a> | <a href="https://github.com/D4Vinci/Scrapling/blob/main/docs/README_DE.md">Deutsch</a> | <a href="https://github.com/D4Vinci/Scrapling/blob/main/docs/README_CN.md">简体中文</a> | <a href="https://github.com/D4Vinci/Scrapling/blob/main/docs/README_JP.md">日本語</a> |  <a href="https://github.com/D4Vinci/Scrapling/blob/main/docs/README_RU.md">Русский</a> | <a href="https://github.com/D4Vinci/Scrapling/blob/main/docs/README_KR.md">한국어</a>
+    <a href="https://github.com/D4Vinci/Scrapling/blob/main/docs/README_AR.md">العربيه</a> | <a href="https://github.com/D4Vinci/Scrapling/blob/main/docs/README_ES.md">Español</a> | <a href="https://github.com/D4Vinci/Scrapling/blob/main/docs/README_PT_BR.md">Português (Brasil)</a> | <a href="https://github.com/D4Vinci/Scrapling/blob/main/docs/README_FR.md">Français</a> | <a href="https://github.com/D4Vinci/Scrapling/blob/main/docs/README_DE.md">Deutsch</a> | <a href="https://github.com/D4Vinci/Scrapling/blob/main/docs/README_CN.md">简体中文</a> | <a href="https://github.com/D4Vinci/Scrapling/blob/main/docs/README_JP.md">日本語</a> |  <a href="https://github.com/D4Vinci/Scrapling/blob/main/docs/README_RU.md">Русский</a> | <a href="https://github.com/D4Vinci/Scrapling/blob/main/docs/README_KR.md">한국어</a>
    <br/>
    <a href="https://github.com/D4Vinci/Scrapling/actions/workflows/tests.yml" alt="Tests">
        <img alt="Tests" src="https://github.com/D4Vinci/Scrapling/actions/workflows/tests.yml/badge.svg"></a>
@@ -1,7 +1,7 @@
 ---
 name: scrapling-official
 description: Scrape web pages using Scrapling with anti-bot bypass (like Cloudflare Turnstile), stealth headless browsing, spiders framework, adaptive scraping, and JavaScript rendering. Use when asked to scrape, crawl, or extract data from websites; web_fetch fails; the site has anti-bot protections; write Python code to scrape/crawl; or write spiders.
-version: "0.4.6"
+version: "0.4.7"
 license: Complete terms in LICENSE.txt
 metadata:
  homepage: "https://scrapling.readthedocs.io/en/latest/index.html"
@@ -40,7 +40,7 @@ Blazing fast crawls with real-time stats and streaming. Built by Web Scrapers fo
 Create a virtual Python environment through any way available, like `venv`, then inside the environment do:
-`pip install "scrapling[all]>=0.4.6"`
+`pip install "scrapling[all]>=0.4.7"`
 Then do this to download all the browsers' dependencies:
@@ -9,7 +9,7 @@ All examples collect **all 100 quotes across 10 pages**.
 Make sure Scrapling is installed:
 ```bash
-pip install "scrapling[all]>=0.4.6"
+pip install "scrapling[all]>=0.4.7"
 scrapling install --force
 ```
@@ -1,8 +1,8 @@
 # Scrapling MCP Server
-The Scrapling MCP server exposes nine web scraping tools over the MCP protocol. It supports CSS-selector-based content narrowing (reducing tokens by extracting only relevant elements before returning results), three levels of scraping capability (plain HTTP, browser-rendered, and stealth/anti-bot bypass), and persistent browser session management.
+The Scrapling MCP server exposes ten tools over the MCP protocol. It supports CSS-selector-based content narrowing (reducing tokens by extracting only relevant elements before returning results), three levels of scraping capability (plain HTTP, browser-rendered, and stealth/anti-bot bypass), persistent browser session management, and page screenshots returned as real image content blocks.
-All scraping tools return a `ResponseModel` with fields: `status` (int), `content` (list of strings), `url` (str).
+All scraping tools return a `ResponseModel` with fields: `status` (int), `content` (list of strings), `url` (str). The `screenshot` tool returns a list of MCP content blocks: an `ImageContent` (the screenshot bytes) followed by a `TextContent` (the post-redirect URL).
 ## Tools
@@ -99,17 +99,18 @@ Opens a browser session that stays alive across multiple fetch calls, avoiding t
 **Key parameters:**
-| Parameter          | Type                        | Default      | Description                                                         |
+| Parameter          | Type                        | Default      | Description                                                                                           |
-|--------------------|-----------------------------|--------------|---------------------------------------------------------------------|
+|--------------------|-----------------------------|--------------|-------------------------------------------------------------------------------------------------------|
-| `session_type`     | `"dynamic"` / `"stealthy"`  | required     | Type of browser session to create                                   |
+| `session_type`     | `"dynamic"` / `"stealthy"`  | required     | Type of browser session to create                                                                     |
-| `headless`         | bool                        | true         | Run browser hidden or visible                                       |
+| `session_id`       | str or null                 | null         | Custom ID for the session. If omitted, a random 12-char hex ID is generated. Raises if already in use |
-| `max_pages`        | int                         | 5            | Max concurrent browser tabs (1-50)                                  |
+| `headless`         | bool                        | true         | Run browser hidden or visible                                                                         |
-| `proxy`            | str or dict or null         | null         | Proxy for all requests in this session                              |
+| `max_pages`        | int                         | 5            | Max concurrent browser tabs (1-50)                                                                    |
-| `timeout`          | number                      | 30000        | Default timeout in ms                                               |
+| `proxy`            | str or dict or null         | null         | Proxy for all requests in this session                                                                |
-| `solve_cloudflare` | bool                        | false        | (Stealthy only) Auto-solve Cloudflare challenges                    |
+| `timeout`          | number                      | 30000        | Default timeout in ms                                                                                 |
-| `hide_canvas`      | bool                        | false        | (Stealthy only) Canvas fingerprint noise                            |
+| `solve_cloudflare` | bool                        | false        | (Stealthy only) Auto-solve Cloudflare challenges                                                      |
-| `block_webrtc`     | bool                        | false        | (Stealthy only) Block WebRTC IP leak                                |
+| `hide_canvas`      | bool                        | false        | (Stealthy only) Canvas fingerprint noise                                                              |
-| `allow_webgl`      | bool                        | true         | (Stealthy only) Keep WebGL enabled                                  |
+| `block_webrtc`     | bool                        | false        | (Stealthy only) Block WebRTC IP leak                                                                  |
 | `allow_webgl`      | bool                        | true         | (Stealthy only) Keep WebGL enabled                                                                    |
 Plus all other browser session parameters (`google_search`, `real_chrome`, `cdp_url`, `locale`, `timezone_id`, `useragent`, `extra_headers`, `cookies`, `disable_resources`, `network_idle`, `wait_selector`, `wait_selector_state`).
@@ -131,6 +132,25 @@ Returns a list of `SessionInfo` objects, each with `session_id`, `session_type`,
 No parameters.
 ### `screenshot` -- Capture a page screenshot
 Navigates to a URL inside an existing browser session and returns the screenshot as an MCP `ImageContent` block (the bytes the model can see directly, not a base64 string in JSON) followed by a `TextContent` block carrying the post-redirect URL.
 Requires an open browser session. Call `open_session` first, then pass the `session_id` here. Both `dynamic` and `stealthy` sessions are accepted.
 | Parameter             | Type                  | Default      | Description                                                                          |
 |-----------------------|-----------------------|--------------|--------------------------------------------------------------------------------------|
 | `url`                 | str                   | required     | URL to navigate to and capture                                                       |
 | `session_id`          | str                   | required     | ID of an open browser session created with `open_session`                            |
 | `image_type`          | `"png"` / `"jpeg"`    | `"png"`      | Image format. Use `"jpeg"` for smaller payloads                                      |
 | `full_page`           | bool                  | false        | Capture the full scrollable page instead of just the viewport                        |
 | `quality`             | int or null           | null         | JPEG quality 0-100. Raises if passed with `image_type="png"`                         |
 | `wait`                | number                | 0            | Extra wait (ms) after page load before capture                                       |
 | `wait_selector`       | str or null           | null         | CSS selector to wait for before capture                                              |
 | `wait_selector_state` | str                   | `"attached"` | State for `wait_selector`: `"attached"` / `"visible"` / `"hidden"` / `"detached"`    |
 | `network_idle`        | bool                  | false        | Wait until no network activity for 500ms                                             |
 | `timeout`             | number                | 30000        | Timeout in milliseconds                                                              |
 ## Tool selection guide
 | Scenario                                 | Tool                                                          |
@@ -142,6 +162,7 @@ No parameters.
 | Cloudflare or strong anti-bot protection | `stealthy_fetch` (with `solve_cloudflare=true` for Turnstile) |
 | Multiple protected pages                 | `bulk_stealthy_fetch`                                         |
 | Multiple pages from the same site        | `open_session` + `fetch`/`stealthy_fetch` with `session_id`  |
 | Need a screenshot of a page              | `open_session` + `screenshot` with `session_id`              |
 Start with `get` (fastest, lowest resource cost). Escalate to `fetch` if content requires JS rendering. Escalate to `stealthy_fetch` only if blocked. For multiple pages from the same site, use a persistent session to avoid browser launch overhead.
@@ -0,0 +1,554 @@
 <!-- mcp-name: io.github.D4Vinci/Scrapling -->
 <h1 align="center">
    <a href="https://scrapling.readthedocs.io">
        <picture>
          <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/docs/assets/cover_dark.svg?sanitize=true">
          <img alt="Scrapling Poster" src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/docs/assets/cover_light.svg?sanitize=true">
        </picture>
    </a>
    <br>
    <small>Web Scraping sem esforço para a web moderna</small>
 </h1>
 <p align="center">
    <a href="https://trendshift.io/repositories/14244" target="_blank"><img src="https://trendshift.io/api/badge/repositories/14244" alt="D4Vinci%2FScrapling | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
    <br/>
    <a href="https://github.com/D4Vinci/Scrapling/actions/workflows/tests.yml" alt="Tests">
        <img alt="Tests" src="https://github.com/D4Vinci/Scrapling/actions/workflows/tests.yml/badge.svg"></a>
    <a href="https://badge.fury.io/py/Scrapling" alt="PyPI version">
        <img alt="PyPI version" src="https://badge.fury.io/py/Scrapling.svg"></a>
    <a href="https://clickpy.clickhouse.com/dashboard/scrapling" rel="nofollow"><img src="https://img.shields.io/pypi/dm/scrapling" alt="PyPI package downloads"></a>
    <a href="https://github.com/D4Vinci/Scrapling/tree/main/agent-skill" alt="AI Agent Skill directory">
        <img alt="Static Badge" src="https://img.shields.io/badge/Skill-black?style=flat&label=Agent&link=https%3A%2F%2Fgithub.com%2FD4Vinci%2FScrapling%2Ftree%2Fmain%2Fagent-skill"></a>
    <a href="https://clawhub.ai/D4Vinci/scrapling-official" alt="OpenClaw Skill">
        <img alt="OpenClaw Skill" src="https://img.shields.io/badge/Clawhub-darkred?style=flat&label=OpenClaw&link=https%3A%2F%2Fclawhub.ai%2FD4Vinci%2Fscrapling-official"></a>
    <br/>
    <a href="https://discord.gg/EMgGbDceNQ" alt="Discord" target="_blank">
      <img alt="Discord" src="https://img.shields.io/discord/1360786381042880532?style=social&logo=discord&link=https%3A%2F%2Fdiscord.gg%2FEMgGbDceNQ">
    </a>
    <a href="https://x.com/Scrapling_dev" alt="X (formerly Twitter)">
      <img alt="X (formerly Twitter) Follow" src="https://img.shields.io/twitter/follow/Scrapling_dev?style=social&logo=x&link=https%3A%2F%2Fx.com%2FScrapling_dev">
    </a>
    <br/>
    <a href="https://pypi.org/project/scrapling/" alt="Supported Python versions">
        <img alt="Supported Python versions" src="https://img.shields.io/pypi/pyversions/scrapling.svg"></a>
 </p>
 <p align="center">
    <a href="https://scrapling.readthedocs.io/en/latest/parsing/selection.html"><strong>Métodos de seleção</strong></a>
    &middot;
    <a href="https://scrapling.readthedocs.io/en/latest/fetching/choosing.html"><strong>Fetchers</strong></a>
    &middot;
    <a href="https://scrapling.readthedocs.io/en/latest/spiders/architecture.html"><strong>Spiders</strong></a>
    &middot;
    <a href="https://scrapling.readthedocs.io/en/latest/spiders/proxy-blocking.html"><strong>Rotação de proxy</strong></a>
    &middot;
    <a href="https://scrapling.readthedocs.io/en/latest/cli/overview.html"><strong>CLI</strong></a>
    &middot;
    <a href="https://scrapling.readthedocs.io/en/latest/ai/mcp-server.html"><strong>MCP</strong></a>
 </p>
 Scrapling é um framework adaptativo de Web Scraping que lida com tudo, desde uma única requisição até um crawl em larga escala.
 Seu parser aprende com as mudanças nos sites e relocaliza automaticamente seus elementos quando as páginas são atualizadas. Seus fetchers contornam sistemas anti-bot como o Cloudflare Turnstile de forma nativa. E seu framework de spiders permite escalar para crawls concorrentes com múltiplas sessões, pausa/retomada e rotação automática de proxies, tudo em poucas linhas de Python. Uma biblioteca, zero concessões.
 Crawls extremamente rápidos com estatísticas em tempo real e streaming. Feito por Web Scrapers para Web Scrapers e usuários comuns, há algo para todo mundo.
 ```python
 from scrapling.fetchers import Fetcher, AsyncFetcher, StealthyFetcher, DynamicFetcher
 StealthyFetcher.adaptive = True
 p = StealthyFetcher.fetch('https://example.com', headless=True, network_idle=True)  # Busque o site sem chamar atenção!
 products = p.css('.product', auto_save=True)                                        # Extraia dados que sobrevivem a mudanças no design do site!
 products = p.css('.product', adaptive=True)                                         # Depois, se a estrutura do site mudar, passe `adaptive=True` para encontrá-los!
 ```
 Ou escale para crawls completos
 ```python
 from scrapling.spiders import Spider, Response
 class MySpider(Spider):
  name = "demo"
  start_urls = ["https://example.com/"]
  async def parse(self, response: Response):
      for item in response.css('.product'):
          yield {"title": item.css('h2::text').get()}
 MySpider().start()
 ```
 <p align="center">
    <a href="https://dataimpulse.com/?utm_source=scrapling&utm_medium=banner&utm_campaign=scrapling" target="_blank" style="display:flex; justify-content:center; padding:4px 0;">
        <img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/DataImpulse.png" alt="At DataImpulse, we specialize in developing custom proxy services for your business. Make requests from anywhere, collect data, and enjoy fast connections with our premium proxies." style="max-height:60px;">
    </a>
 </p>
 # Patrocinadores Platina
 <table>
  <tr>
    <td width="200">
      <a href="https://hypersolutions.co/?utm_source=github&utm_medium=readme&utm_campaign=scrapling" target="_blank" title="Bot Protection Bypass API for Akamai, DataDome, Incapsula & Kasada">
        <img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/HyperSolutions.png">
      </a>
    </td>
    <td> Scrapling lida com o Cloudflare Turnstile. Para proteção de nível empresarial, <a href="https://hypersolutions.co?utm_source=github&utm_medium=readme&utm_campaign=scrapling">
        <b>Hyper Solutions</b>
      </a> oferece endpoints de API que geram tokens antibot válidos para <b>Akamai</b>, <b>DataDome</b>, <b>Kasada</b> e <b>Incapsula</b>. Chamadas simples de API, sem necessidade de automação de navegador. </td>
  </tr>
  <tr>
    <td width="200">
      <a href="https://birdproxies.com/t/scrapling" target="_blank" title="At Bird Proxies, we eliminate your pains such as banned IPs, geo restriction, and high costs so you can focus on your work.">
        <img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/BirdProxies.jpg">
      </a>
    </td>
    <td>Nós criamos a <a href="https://birdproxies.com/t/scrapling">
        <b>BirdProxies</b>
      </a> porque proxies não deveriam ser complicados nem caros. Proxies residenciais e ISP rápidos em mais de 195 localidades, preços justos e suporte de verdade. <br />
      <b>Experimente nosso jogo FlappyBird na landing page para ganhar dados grátis!</b>
    </td>
  </tr>
  <tr>
    <td width="200">
      <a href="https://evomi.com?utm_source=github&utm_medium=banner&utm_campaign=d4vinci-scrapling" target="_blank" title="Evomi is your Swiss Quality Proxy Provider, starting at $0.49/GB">
        <img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/evomi.png">
      </a>
    </td>
    <td>
      <a href="https://evomi.com?utm_source=github&utm_medium=banner&utm_campaign=d4vinci-scrapling">
        <b>Evomi</b>
      </a>: proxies residenciais a partir de US$0.49/GB. Navegador de scraping com Chromium totalmente spoofado, IPs residenciais, resolução automática de CAPTCHA e bypass anti-bot. </br>
      <b>Scraper API para resultados sem complicação. Integrações com MCP e N8N estão disponíveis.</b>
    </td>
  </tr>
  <tr>
    <td width="200">
      <a href="https://tikhub.io/?utm_source=github.com/D4Vinci/Scrapling&utm_medium=marketing_social&utm_campaign=retargeting&utm_content=carousel_ad" target="_blank" title="Unlock the Power of Social Media Data & AI">
        <img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/TikHub.jpg">
      </a>
    </td>
    <td>
      <a href="https://tikhub.io/?utm_source=github.com/D4Vinci/Scrapling&utm_medium=marketing_social&utm_campaign=retargeting&utm_content=carousel_ad" target="_blank">TikHub.io</a> oferece mais de 900 APIs estáveis em mais de 16 plataformas, incluindo TikTok, X, YouTube e Instagram, com mais de 40M de datasets. <br /> Também oferece <a href="https://ai.tikhub.io/?ref=KarimShoair" target="_blank">modelos de IA com desconto</a> - Claude, GPT, GEMINI e mais com até 71% de desconto.
    </td>
  </tr>
  <tr>
    <td width="200">
      <a href="https://www.nsocks.com/?keyword=2p67aivg" target="_blank" title="Scalable Web Data Access for AI Applications">
        <img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/nsocks.png">
      </a>
    </td>
    <td>
    <a href="https://www.nsocks.com/?keyword=2p67aivg" target="_blank">Nsocks</a> fornece proxies residenciais e ISP rápidos para desenvolvedores e scrapers. Cobertura global de IPs, alto anonimato, rotação inteligente e desempenho confiável para automação e extração de dados. Use o <a href="https://www.xcrawl.com/?keyword=2p67aivg" target="_blank">Xcrawl</a> para simplificar o crawling web em larga escala.
    </td>
  </tr>
  <tr>
    <td width="200">
      <a href="https://petrosky.io/d4vinci" target="_blank" title="PetroSky delivers cutting-edge VPS hosting.">
        <img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/petrosky.png">
      </a>
    </td>
    <td>
    Feche o notebook. Seus scrapers continuam rodando. <br />
    <a href="https://petrosky.io/d4vinci" target="_blank">PetroSky VPS</a> - servidores em nuvem feitos para automação ininterrupta. Máquinas Windows e Linux com controle total. A partir de €6.99/mês.
    </td>
  </tr>
  <tr>
    <td width="200">
      <a href="https://substack.thewebscraping.club/p/scrapling-hands-on-guide?utm_source=github&utm_medium=repo&utm_campaign=scrapling" target="_blank" title="The #1 newsletter dedicated to Web Scraping">
        <img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/TWSC.png">
      </a>
    </td>
    <td>
    Leia uma análise completa do <a href="https://substack.thewebscraping.club/p/scrapling-hands-on-guide?utm_source=github&utm_medium=repo&utm_campaign=scrapling" target="_blank">Scrapling no The Web Scraping Club</a> (nov. 2025), a newsletter número 1 dedicada a Web Scraping.
    </td>
  </tr>
  <tr>
    <td width="200">
      <a href="https://proxy-seller.com/?partner=CU9CAA5TBYFFT2" target="_blank" title="Proxy-Seller provides reliable proxy infrastructure for Web Scraping">
        <img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/ProxySeller.png">
      </a>
    </td>
    <td>
    <a href="https://proxy-seller.com/?partner=CU9CAA5TBYFFT2" target="_blank">Proxy-Seller</a> fornece infraestrutura de proxy confiável para web scraping, oferecendo proxies IPv4, IPv6, ISP, residenciais e móveis com desempenho estável, ampla cobertura geográfica e planos flexíveis para coleta de dados em escala empresarial.
    </td>
  </tr>
  <tr>
    <td width="200">
      <a href="http://mangoproxy.com/?utm_source=D4Vinci&utm_medium=GitHub&utm_campaign=D4Vinci" target="_blank" title="Proxies You Can Rely On: Residential, Server, and Mobile">
        <img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/MangoProxy.png">
      </a>
    </td>
    <td>
    <a href="http://mangoproxy.com/?utm_source=D4Vinci&utm_medium=GitHub&utm_campaign=D4Vinci" target="_blank">Proxies estáveis</a> para scraping, automação e multi-accounting. IPs limpos, resposta rápida e desempenho confiável sob carga. Feito para fluxos de trabalho escaláveis.
    </td>
  </tr>
  <tr>
    <td width="200">
      <a href="https://www.swiftproxy.net/?ref=D4Vinci" target="_blank" title="Scalable Solutions for Web Data Access">
        <img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/SwiftProxy.png">
      </a>
    </td>
    <td>
    <a href="https://www.swiftproxy.net/?ref=D4Vinci" target="_blank">Swiftproxy</a> fornece proxies residenciais escaláveis com mais de 80M de IPs em mais de 195 países, entregando conexões rápidas e confiáveis, rotação automática e forte desempenho anti-bloqueio. Teste grátis disponível.
    </td>
  </tr>
 </table>
 <i><sub>Quer mostrar seu anúncio aqui? Clique [aqui](https://github.com/sponsors/D4Vinci/sponsorships?tier_id=586646)</sub></i>
 # Patrocinadores
 <!-- sponsors -->
 <a href="https://serpapi.com/?utm_source=scrapling" target="_blank" title="Scrape Google and other search engines with SerpApi"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/SerpApi.png"></a>
 <a href="https://visit.decodo.com/Dy6W0b" target="_blank" title="Try the Most Efficient Residential Proxies for Free"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/decodo.png"></a>
 <a href="https://hasdata.com/?utm_source=github&utm_medium=banner&utm_campaign=D4Vinci" target="_blank" title="The web scraping service that actually beats anti-bot systems!"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/hasdata.png"></a>
 <a href="https://proxyempire.io/?ref=scrapling&utm_source=scrapling" target="_blank" title="Collect The Data Your Project Needs with the Best Residential Proxies"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/ProxyEmpire.png"></a>
 <a href="https://www.webshare.io/?referral_code=48r2m2cd5uz1" target="_blank" title="The Most Reliable Proxy with Unparalleled Performance"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/webshare.png"></a>
 <a href="https://www.crawleo.dev/?utm_source=github&utm_medium=sponsor&utm_campaign=scrapling" target="_blank" title="Supercharge your AI with Real-Time Web Intelligence"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/crawleo.png"></a>
 <a href="https://www.rapidproxy.io/?ref=d4v" target="_blank" title="Affordable Access to the Proxy World – bypass CAPTCHAs blocks, and avoid additional costs."><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/rapidproxy.jpg"></a>
 <!-- /sponsors -->
 <i><sub>Quer mostrar seu anúncio aqui? Clique [aqui](https://github.com/sponsors/D4Vinci) e escolha o plano que fizer mais sentido para você!</sub></i>
 ---
 ## Principais Recursos
 ### Spiders - Um Framework Completo de Crawling
 - 🕷️ **API de Spider estilo Scrapy**: Defina spiders com `start_urls`, callbacks assíncronos `parse` e objetos `Request`/`Response`.
 - ⚡ **Crawling Concorrente**: Limites de concorrência configuráveis, throttling por domínio e delays de download.
 - 🔄 **Suporte Multi-Sessão**: Interface unificada para requisições HTTP e navegadores headless furtivos em uma única spider - direcione requisições para diferentes sessões por ID.
 - 💾 **Pausa e Retomada**: Persistência de crawl baseada em checkpoints. Pressione Ctrl+C para um encerramento gracioso; reinicie para continuar de onde parou.
 - 📡 **Modo Streaming**: Faça streaming dos itens extraídos conforme chegam com `async for item in spider.stream()` e estatísticas em tempo real - ideal para UI, pipelines e crawls de longa duração.
 - 🛡️ **Detecção de Requisições Bloqueadas**: Detecção automática e retry de requisições bloqueadas com lógica personalizável.
 - 🤖 **Conformidade com robots.txt**: Flag opcional `robots_txt_obey` que respeita as diretivas `Disallow`, `Crawl-delay` e `Request-rate` com cache por domínio.
 - 🧪 **Modo de Desenvolvimento**: Armazene respostas em disco na primeira execução e reproduza-as nas seguintes - itere sobre sua lógica de `parse()` sem reenviar requisições aos servidores-alvo.
 - 📦 **Exportação Nativa**: Exporte resultados via hooks, seu próprio pipeline ou JSON/JSONL nativos com `result.items.to_json()` / `result.items.to_jsonl()` respectivamente.
 ### Busca Avançada de Sites com Suporte a Sessões
 - **Requisições HTTP**: Requisições HTTP rápidas e furtivas com a classe `Fetcher`. Pode imitar fingerprint TLS de navegadores, cabeçalhos e usar HTTP/3.
 - **Carregamento Dinâmico**: Busque sites dinâmicos com automação completa de navegador através da classe `DynamicFetcher`, compatível com o Chromium do Playwright e o Google Chrome.
 - **Bypass Anti-Bot**: Capacidades avançadas de stealth com `StealthyFetcher` e spoofing de fingerprint. Pode contornar facilmente todos os tipos de Turnstile/Interstitial do Cloudflare com automação.
 - **Gerenciamento de Sessão**: Suporte a sessões persistentes com as classes `FetcherSession`, `StealthySession` e `DynamicSession` para gerenciar cookies e estado entre requisições.
 - **Rotação de Proxy**: `ProxyRotator` nativo com estratégias cíclicas ou personalizadas em todos os tipos de sessão, além de sobrescritas de proxy por requisição.
 - **Bloqueio de Domínios e Anúncios**: Bloqueie requisições para domínios específicos (e seus subdomínios) ou habilite o bloqueio nativo de anúncios (~3.500 domínios conhecidos de anúncios/rastreadores) nos fetchers baseados em navegador.
 - **Prevenção de Vazamento de DNS**: Suporte opcional a DNS-over-HTTPS para rotear consultas DNS através do DoH da Cloudflare, evitando vazamentos de DNS ao usar proxies.
 - **Suporte Async**: Suporte assíncrono completo em todos os fetchers e classes dedicadas de sessão async.
 ### Scraping Adaptativo e Integração com IA
 - 🔄 **Rastreamento Inteligente de Elementos**: Relocalize elementos após mudanças no site usando algoritmos inteligentes de similaridade.
 - 🎯 **Seleção Flexível Inteligente**: Seletores CSS, seletores XPath, busca baseada em filtros, busca por texto, busca por regex e muito mais.
 - 🔍 **Encontrar Elementos Semelhantes**: Localize automaticamente elementos parecidos com os elementos encontrados.
 - 🤖 **Servidor MCP para uso com IA**: Servidor MCP nativo para Web Scraping assistido por IA e extração de dados. O servidor MCP oferece capacidades poderosas e personalizadas que usam o Scrapling para extrair conteúdo direcionado antes de passá-lo à IA (Claude/Cursor/etc), acelerando as operações e reduzindo custos ao minimizar o uso de tokens. ([vídeo demo](https://www.youtube.com/watch?v=qyFk3ZNwOxE))
 ### Arquitetura de Alto Desempenho e Testada em Batalha
 - 🚀 **Muito Rápido**: Desempenho otimizado que supera a maioria das bibliotecas Python de scraping.
 - 🔋 **Eficiente em Memória**: Estruturas de dados otimizadas e lazy loading para um uso mínimo de memória.
 - ⚡ **Serialização JSON Rápida**: 10x mais rápido que a biblioteca padrão.
 - 🏗️ **Testado em batalha**: O Scrapling não apenas tem 92% de cobertura de testes e cobertura completa de type hints, como também vem sendo usado diariamente por centenas de Web Scrapers ao longo do último ano.
 ### Experiência Amigável para Desenvolvedores/Web Scrapers
 - 🎯 **Shell Interativo de Web Scraping**: Shell opcional embutido em IPython com integração ao Scrapling, atalhos e novas ferramentas para acelerar o desenvolvimento de scripts de Web Scraping, como converter requisições curl em requisições Scrapling e visualizar resultados no navegador.
 - 🚀 **Use diretamente no Terminal**: Opcionalmente, você pode usar o Scrapling para extrair uma URL sem escrever uma única linha de código!
 - 🛠️ **API Rica de Navegação**: Travessia avançada do DOM com métodos de navegação por pais, irmãos e filhos.
 - 🧬 **Processamento de Texto Aprimorado**: Métodos nativos de regex, limpeza e operações de string otimizadas.
 - 📝 **Geração Automática de Seletores**: Gere seletores CSS/XPath robustos para qualquer elemento.
 - 🔌 **API Familiar**: Semelhante a Scrapy/BeautifulSoup, com os mesmos pseudo-elementos usados em Scrapy/Parsel.
 - 📘 **Cobertura Completa de Tipos**: Type hints completos para excelente suporte em IDEs e autocompletar de código. Todo o codebase é escaneado automaticamente com **PyRight** e **MyPy** a cada alteração.
 - 🔋 **Imagem Docker Pronta**: A cada release, uma imagem Docker contendo todos os navegadores é construída e publicada automaticamente.
 ## Primeiros Passos
 Vamos dar uma visão rápida do que o Scrapling pode fazer sem entrar em muitos detalhes.
 ### Uso Básico
 Requisições HTTP com suporte a sessões
 ```python
 from scrapling.fetchers import Fetcher, FetcherSession
 with FetcherSession(impersonate='chrome') as session:  # Use a versão mais recente da fingerprint TLS do Chrome
    page = session.get('https://quotes.toscrape.com/', stealthy_headers=True)
    quotes = page.css('.quote .text::text').getall()
 # Ou use requisições avulsas
 page = Fetcher.get('https://quotes.toscrape.com/')
 quotes = page.css('.quote .text::text').getall()
 ```
 Modo stealth avançado
 ```python
 from scrapling.fetchers import StealthyFetcher, StealthySession
 with StealthySession(headless=True, solve_cloudflare=True) as session:  # Mantenha o navegador aberto até terminar
    page = session.fetch('https://nopecha.com/demo/cloudflare', google_search=False)
    data = page.css('#padded_content a').getall()
 # Ou use o estilo de requisição avulsa, ele abre o navegador para esta requisição e o fecha ao finalizar
 page = StealthyFetcher.fetch('https://nopecha.com/demo/cloudflare')
 data = page.css('#padded_content a').getall()
 ```
 Automação completa de navegador
 ```python
 from scrapling.fetchers import DynamicFetcher, DynamicSession
 with DynamicSession(headless=True, disable_resources=False, network_idle=True) as session:  # Mantenha o navegador aberto até terminar
    page = session.fetch('https://quotes.toscrape.com/', load_dom=False)
    data = page.xpath('//span[@class="text"]/text()').getall()  # Se preferir, use seletor XPath
 # Ou use o estilo de requisição avulsa, ele abre o navegador para esta requisição e o fecha ao finalizar
 page = DynamicFetcher.fetch('https://quotes.toscrape.com/')
 data = page.css('.quote .text::text').getall()
 ```
 ### Spiders
 Construa crawlers completos com requisições concorrentes, múltiplos tipos de sessão e pausa/retomada:
 ```python
 from scrapling.spiders import Spider, Request, Response
 class QuotesSpider(Spider):
    name = "quotes"
    start_urls = ["https://quotes.toscrape.com/"]
    concurrent_requests = 10
    async def parse(self, response: Response):
        for quote in response.css('.quote'):
            yield {
                "text": quote.css('.text::text').get(),
                "author": quote.css('.author::text').get(),
            }
        next_page = response.css('.next a')
        if next_page:
            yield response.follow(next_page[0].attrib['href'])
 result = QuotesSpider().start()
 print(f"Extraídas {len(result.items)} citações")
 result.items.to_json("quotes.json")
 ```
 Use múltiplos tipos de sessão em uma única spider:
 ```python
 from scrapling.spiders import Spider, Request, Response
 from scrapling.fetchers import FetcherSession, AsyncStealthySession
 class MultiSessionSpider(Spider):
    name = "multi"
    start_urls = ["https://example.com/"]
    def configure_sessions(self, manager):
        manager.add("fast", FetcherSession(impersonate="chrome"))
        manager.add("stealth", AsyncStealthySession(headless=True), lazy=True)
    async def parse(self, response: Response):
        for link in response.css('a::attr(href)').getall():
            # Direcione páginas protegidas através da sessão stealth
            if "protected" in link:
                yield Request(link, sid="stealth")
            else:
                yield Request(link, sid="fast", callback=self.parse)  # callback explícito
 ```
 Pause e retome crawls longos com checkpoints executando a spider assim:
 ```python
 QuotesSpider(crawldir="./crawl_data").start()
 ```
 Pressione Ctrl+C para pausar de forma graciosa - o progresso é salvo automaticamente. Depois, quando você iniciar a spider novamente, passe o mesmo `crawldir` e ela continuará de onde parou.
 ### Parsing Avançado e Navegação
 ```python
 from scrapling.fetchers import Fetcher
 # Seleção rica de elementos e navegação
 page = Fetcher.get('https://quotes.toscrape.com/')
 # Obtenha citações com múltiplos métodos de seleção
 quotes = page.css('.quote')  # Seletor CSS
 quotes = page.xpath('//div[@class="quote"]')  # XPath
 quotes = page.find_all('div', {'class': 'quote'})  # Estilo BeautifulSoup
 # O mesmo que
 quotes = page.find_all('div', class_='quote')
 quotes = page.find_all(['div'], class_='quote')
 quotes = page.find_all(class_='quote')  # e assim por diante...
 # Encontre elementos por conteúdo de texto
 quotes = page.find_by_text('quote', tag='div')
 # Navegação avançada
 quote_text = page.css('.quote')[0].css('.text::text').get()
 quote_text = page.css('.quote').css('.text::text').getall()  # Seletores encadeados
 first_quote = page.css('.quote')[0]
 author = first_quote.next_sibling.css('.author::text')
 parent_container = first_quote.parent
 # Relações e similaridade entre elementos
 similar_elements = first_quote.find_similar()
 below_elements = first_quote.below_elements()
 ```
 Você pode usar o parser imediatamente se não quiser buscar sites, como abaixo:
 ```python
 from scrapling.parser import Selector
 page = Selector("<html>...</html>")
 ```
 E ele funciona exatamente da mesma maneira!
 ### Exemplos de Gerenciamento de Sessão Assíncrona
 ```python
 import asyncio
 from scrapling.fetchers import FetcherSession, AsyncStealthySession, AsyncDynamicSession
 async with FetcherSession(http3=True) as session:  # `FetcherSession` entende o contexto e funciona tanto em padrões sync quanto async
    page1 = session.get('https://quotes.toscrape.com/')
    page2 = session.get('https://quotes.toscrape.com/', impersonate='firefox135')
 # Uso de sessão assíncrona
 async with AsyncStealthySession(max_pages=2) as session:
    tasks = []
    urls = ['https://example.com/page1', 'https://example.com/page2']
    for url in urls:
        task = session.fetch(url)
        tasks.append(task)
    print(session.get_pool_stats())  # Opcional - O estado do pool de abas do navegador (ocupada/livre/erro)
    results = await asyncio.gather(*tasks)
    print(session.get_pool_stats())
 ```
 ## CLI e Shell Interativo
 O Scrapling inclui uma poderosa interface de linha de comando:
 [![asciicast](https://asciinema.org/a/736339.svg)](https://asciinema.org/a/736339)
 Inicie o shell interativo de Web Scraping
 ```bash
 scrapling shell
 ```
 Extraia páginas diretamente para um arquivo sem programar (por padrão, extrai o conteúdo dentro da tag `body`). Se o arquivo de saída terminar com `.txt`, então o conteúdo em texto do alvo será extraído. Se terminar com `.md`, será uma representação em Markdown do conteúdo HTML; se terminar com `.html`, será o próprio conteúdo HTML.
 ```bash
 scrapling extract get 'https://example.com' content.md
 scrapling extract get 'https://example.com' content.txt --css-selector '#fromSkipToProducts' --impersonate 'chrome'  # Todos os elementos que correspondem ao seletor CSS '#fromSkipToProducts'
 scrapling extract fetch 'https://example.com' content.md --css-selector '#fromSkipToProducts' --no-headless
 scrapling extract stealthy-fetch 'https://nopecha.com/demo/cloudflare' captchas.html --css-selector '#padded_content a' --solve-cloudflare
 ```
 > [!NOTE]
 > Existem muitos recursos adicionais, mas queremos manter esta página concisa, incluindo o servidor MCP e o Shell Interativo de Web Scraping. Confira a documentação completa [aqui](https://scrapling.readthedocs.io/en/latest/)
 ## Benchmarks de Desempenho
 O Scrapling não é apenas poderoso - ele também é extremamente rápido. Os benchmarks abaixo comparam o parser do Scrapling com as versões mais recentes de outras bibliotecas populares.
 ### Teste de Velocidade de Extração de Texto (5000 elementos aninhados)
 | # |    Biblioteca     | Tempo (ms) | vs Scrapling | 
 |---|:-----------------:|:----------:|:------------:|
 | 1 |     Scrapling     |    2.02    |     1.0x     |
 | 2 |   Parsel/Scrapy   |    2.04    |     1.01     |
 | 3 |     Raw Lxml      |    2.54    |    1.257     |
 | 4 |      PyQuery      |   24.17    |     ~12x     |
 | 5 |    Selectolax     |   82.63    |     ~41x     |
 | 6 |  MechanicalSoup   |  1549.71   |   ~767.1x    |
 | 7 |   BS4 with Lxml   |  1584.31   |   ~784.3x    |
 | 8 | BS4 with html5lib |  3391.91   |   ~1679.1x   |
 ### Desempenho de Similaridade de Elementos e Busca por Texto
 Os recursos de localização adaptativa de elementos do Scrapling superam significativamente as alternativas:
 | Biblioteca  | Tempo (ms) | vs Scrapling |
 |-------------|:----------:|:------------:|
 | Scrapling   |    2.39    |     1.0x     |
 | AutoScraper |   12.45    |    5.209x    |
 > Todos os benchmarks representam médias de 100+ execuções. Veja [benchmarks.py](https://github.com/D4Vinci/Scrapling/blob/main/benchmarks.py) para a metodologia.
 ## Instalação
 O Scrapling requer Python 3.10 ou superior:
 ```bash
 pip install scrapling
 ```
 Esta instalação inclui apenas o motor de parsing e suas dependências, sem fetchers nem dependências de linha de comando.
 ### Dependências Opcionais
 1. Se você vai usar qualquer um dos recursos extras abaixo, os fetchers ou suas classes, precisará instalar as dependências dos fetchers e as dependências de navegador deles da seguinte forma:
    ```bash
    pip install "scrapling[fetchers]"
    scrapling install           # instalação normal
    scrapling install  --force  # forçar reinstalação
    ```
    Isso baixa todos os navegadores, juntamente com suas dependências de sistema e dependências de manipulação de fingerprint.
    Ou você pode instalá-los a partir do código em vez de executar um comando como este:
    ```python
    from scrapling.cli import install
    install([], standalone_mode=False)          # instalação normal
    install(["--force"], standalone_mode=False) # forçar reinstalação
    ```
 2. Recursos extras:
   - Instale o recurso do servidor MCP:
       ```bash
       pip install "scrapling[ai]"
       ```
   - Instale os recursos do shell (Shell de Web Scraping e o comando `extract`):
       ```bash
       pip install "scrapling[shell]"
       ```
   - Instale tudo:
       ```bash
       pip install "scrapling[all]"
       ```
   Lembre-se de que você precisa instalar as dependências de navegador com `scrapling install` depois de qualquer um desses extras (caso ainda não tenha feito isso)
 ### Docker
 Você também pode baixar uma imagem Docker com todos os extras e navegadores com o seguinte comando a partir do DockerHub:
 ```bash
 docker pull pyd4vinci/scrapling
 ```
 Ou baixá-la do registro do GitHub:
 ```bash
 docker pull ghcr.io/d4vinci/scrapling:latest
 ```
 Essa imagem é construída e publicada automaticamente usando GitHub Actions e o branch principal do repositório.
 ## Contribuindo
 Contribuições são bem-vindas! Leia nossas [diretrizes de contribuição](https://github.com/D4Vinci/Scrapling/blob/main/CONTRIBUTING.md) antes de começar.
 ## Aviso Legal
 > [!CAUTION]
 > Esta biblioteca é fornecida apenas para fins educacionais e de pesquisa. Ao usar esta biblioteca, você concorda em cumprir as leis locais e internacionais de scraping de dados e privacidade. Os autores e contribuidores não se responsabilizam por qualquer uso indevido deste software. Sempre respeite os termos de serviço dos sites e os arquivos robots.txt.
 ## 🎓 Citações
 Se você usou nossa biblioteca para fins de pesquisa, cite-nos com a seguinte referência:
 ```text
  @misc{scrapling,
    author = {Karim Shoair},
    title = {Scrapling},
    year = {2024},
    url = {https://github.com/D4Vinci/Scrapling},
    note = {An adaptive Web Scraping framework that handles everything from a single request to a full-scale crawl!}
  }
 ```
 ## Licença
 Este trabalho está licenciado sob a licença BSD-3-Clause.
 ## Agradecimentos
 Este projeto inclui código adaptado de:
 - Parsel (Licença BSD) - usado para o submódulo [translator](https://github.com/D4Vinci/Scrapling/blob/main/scrapling/core/translator.py)
 ---
 <div align="center"><small>Projetado e desenvolvido com ❤️ por Karim Shoair.</small></div><br>
@@ -6,7 +6,7 @@ The **Scrapling MCP Server** is a new feature that brings Scrapling's powerful W
 ## Features
-The Scrapling MCP Server provides nine powerful tools for web scraping:
+The Scrapling MCP Server provides ten powerful tools for web scraping:
 ### 🚀 Basic HTTP Scraping
 - **`get`**: Fast HTTP requests with browser fingerprint impersonation, generating real browser headers matching the TLS version, HTTP/3, and more!
@@ -20,6 +20,9 @@ The Scrapling MCP Server provides nine powerful tools for web scraping:
 - **`stealthy_fetch`**: Uses our Stealthy browser to bypass Cloudflare Turnstile/Interstitial and other anti-bot systems with complete control over the request/browser!
 - **`bulk_stealthy_fetch`**: An async version of the above tool that allows stealth scraping of multiple URLs in different browser tabs at the same time!
 ### 📸 Screenshots
 - **`screenshot`**: Capture a PNG or JPEG screenshot of a page using an open browser session, returned as an image content block the model can actually see (not a base64 string blob). Supports full-page captures, JPEG quality, and the usual readiness controls (`wait`, `wait_selector`, `network_idle`).
 ### 🔌 Session Management
 - **`open_session`**: Create a persistent browser session (dynamic or stealthy) that stays open across multiple fetch calls, avoiding the overhead of launching a new browser each time.
 - **`close_session`**: Close a persistent browser session and free its resources.
@@ -331,6 +334,14 @@ This protection runs automatically on all MCP tool responses. Keep `main_content
 - Always close sessions with `close_session` when done to free resources
 - Use `list_sessions` to check which sessions are still active
 - A `session_id` from a dynamic session can only be used with `fetch`/`bulk_fetch`, and a stealthy session can only be used with `stealthy_fetch`/`bulk_stealthy_fetch`
 - Pass a custom `session_id` to `open_session` to give sessions meaningful names (e.g. `"search"`, `"checkout"`) instead of the random hex default. `open_session` raises if the chosen ID is already in use, so you can detect collisions up front
 ### 7. Capturing Screenshots
 - `screenshot` only works through an existing browser session, so call `open_session` first (either `dynamic` or `stealthy` works)
 - The image is returned as a real `ImageContent` block, not a base64 string in JSON, so the model sees the page directly
 - Use `full_page=True` when you need everything below the fold; the default captures only the visible viewport
 - Pick `image_type="jpeg"` with a `quality` value (0-100) for smaller payloads when pixel-perfect color isn't needed
 - The same `wait`, `wait_selector`, `network_idle`, and `timeout` controls used by `fetch` are available here too
 ## Legal and Ethical Considerations
@@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta"
 [project]
 name = "scrapling"
 # Static version instead of a dynamic version so we can get better layer caching while building docker, check the docker file to understand
-version = "0.4.6"
+version = "0.4.7"
 description = "Scrapling is an undetectable, powerful, flexible, high-performance Python library that makes Web Scraping easy and effortless as it should be!"
 readme = {file = "README.md", content-type = "text/markdown"}
 license = {file = "LICENSE"}
@@ -77,8 +77,8 @@ fetchers = [
    "patchright==1.58.2",
    "browserforge>=1.2.4",
    "apify-fingerprint-datapoints>=0.12.0",
-    "msgspec>=0.21.0",
+    "msgspec>=0.21.1",
-    "anyio>=4.12.1",
+    "anyio>=4.13.0",
    "protego>=0.6.0",
 ]
 ai = [
@@ -1,5 +1,5 @@
 __author__ = "Karim Shoair (karim.shoair@pm.me)"
-__version__ = "0.4.6"
+__version__ = "0.4.7"
 __copyright__ = "Copyright (c) 2024 Karim Shoair"
 from typing import Any, TYPE_CHECKING
@@ -3,7 +3,8 @@ from asyncio import gather
 from datetime import datetime, timezone
 from dataclasses import dataclass, field
-from mcp.server.fastmcp import FastMCP
+from mcp.server.fastmcp import FastMCP, Image
 from mcp.types import ImageContent, TextContent
 from pydantic import BaseModel, Field
 from scrapling.core.shell import Convertor
@@ -31,6 +32,7 @@ from scrapling.core._types import (
 )
 SessionType = Literal["dynamic", "stealthy"]
 ScreenshotType = Literal["png", "jpeg"]
 class ResponseModel(BaseModel):
@@ -106,14 +108,14 @@ class ScraplingMCPServer:
    def __init__(self):
        self._sessions: Dict[str, _SessionEntry] = {}
-    def _get_session(self, session_id: str, expected_type: SessionType) -> _SessionEntry:
+    def _get_session(self, session_id: str, expected_type: Optional[SessionType]) -> _SessionEntry:
-        """Look up a session by ID and validate its type."""
+        """Look up a session by ID, optionally validating its type. Pass `None` to skip the type check."""
        entry = self._sessions.get(session_id)
        if entry is None:
            raise ValueError(f"Session '{session_id}' not found. Use list_sessions to see active sessions.")
        if not entry.session._is_alive:
            raise ValueError(f"Session '{session_id}' is no longer alive. Open a new session.")
-        if entry.session_type != expected_type:
+        if expected_type is not None and entry.session_type != expected_type:
            raise ValueError(
                f"Session '{session_id}' is a '{entry.session_type}' session, but this tool requires a "
                f"'{expected_type}' session. Use the matching fetch tool for your session type."
@@ -123,6 +125,7 @@ class ScraplingMCPServer:
    async def open_session(
        self,
        session_type: SessionType,
        session_id: Optional[str] = None,
        headless: bool = True,
        google_search: bool = True,
        real_chrome: bool = False,
@@ -152,6 +155,7 @@ class ScraplingMCPServer:
        Use close_session to close the session when done, and list_sessions to see all active sessions.
        :param session_type: The type of session to open. Use "dynamic" for standard Playwright browser, or "stealthy" for anti-bot bypass with fingerprint spoofing.
        :param session_id: Optional custom session ID. If not provided, a random 12-character hex ID will be generated. Useful for naming sessions for easier management.
        :param headless: Run the browser in headless/hidden (default), or headful/visible mode.
        :param google_search: Enabled by default, Scrapling will set a Google referer header.
        :param real_chrome: If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch an instance of your browser and use it.
@@ -175,6 +179,12 @@ class ScraplingMCPServer:
        :param solve_cloudflare: (Stealthy only) Solves all types of the Cloudflare's Turnstile/Interstitial challenges.
        :param additional_args: (Stealthy only) Additional arguments to be passed to Playwright's context as additional settings.
        """
        session_id = session_id or uuid4().hex[:12]
        if session_id in self._sessions:
            raise ValueError(
                f"Session '{session_id}' already exists. Use a different ID or close the existing session first."
            )
        common_kwargs: Dict[str, Any] = dict(
            wait=wait,
            proxy=proxy,
@@ -211,7 +221,6 @@ class ScraplingMCPServer:
        await session.start()
        session_id = uuid4().hex[:12]
        entry = _SessionEntry(session=session, session_type=session_type)
        self._sessions[session_id] = entry
@@ -253,6 +262,69 @@ class ScraplingMCPServer:
            for sid, entry in self._sessions.items()
        ]
    async def screenshot(
        self,
        url: str,
        session_id: str,
        image_type: ScreenshotType = "png",
        full_page: bool = False,
        quality: Optional[int] = None,
        wait: int | float = 0,
        wait_selector: Optional[str] = None,
        wait_selector_state: SelectorWaitStates = "attached",
        network_idle: bool = False,
        timeout: int | float = 30000,
    ) -> List[ImageContent | TextContent]:
        """Capture a screenshot of a web page using an existing browser session and return it as an image.
        A browser session must be opened first with `open_session` (either `dynamic` or `stealthy`); the session ID is then passed here.
        :param url: The URL to navigate to and capture.
        :param session_id: ID of an open browser session created with `open_session`.
        :param image_type: Image format. Defaults to "png". Use "jpeg" for smaller file sizes.
        :param full_page: When True, captures the full scrollable page instead of just the viewport. Defaults to False.
        :param quality: Image quality (0-100) for JPEG only. Raises if passed with `image_type="png"`.
        :param wait: Time in milliseconds to wait after page load before capturing. Defaults to 0.
        :param wait_selector: Optional CSS selector to wait for before capturing.
        :param wait_selector_state: State to wait for the selector. Defaults to "attached".
        :param network_idle: Wait for the page until there are no network connections for at least 500 ms.
        :param timeout: Timeout in milliseconds for page operations. Defaults to 30,000.
        """
        if quality is not None and image_type != "jpeg":
            raise ValueError("'quality' is only valid when 'image_type' is 'jpeg'.")
        entry = self._get_session(session_id, expected_type=None)
        screenshot_kwargs: Dict[str, Any] = {"type": image_type, "full_page": full_page}
        if quality is not None:
            screenshot_kwargs["quality"] = quality
        captured: Dict[str, Any] = {}
        async def _capture(page: Any) -> None:
            try:
                captured["bytes"] = await page.screenshot(**screenshot_kwargs)
                captured["url"] = page.url
            except Exception as exc:
                captured["error"] = exc
        await entry.session.fetch(
            url,
            wait=wait,
            timeout=timeout,
            network_idle=network_idle,
            wait_selector=wait_selector,
            wait_selector_state=wait_selector_state,
            page_action=_capture,
        )
        if "error" in captured:
            raise captured["error"]
        if "bytes" not in captured:
            raise RuntimeError(f"Failed to capture screenshot for {url}")
        image = Image(data=captured["bytes"], format=image_type).to_image_content()
        return [image, TextContent(type="text", text=captured["url"])]
    @staticmethod
    async def get(
        url: str,
@@ -291,7 +363,8 @@ class ScraplingMCPServer:
        :param headers: Headers to include in the request.
        :param cookies: Cookies to use in the request.
        :param timeout: Number of seconds to wait before timing out.
-        :param follow_redirects: Whether to follow redirects. Defaults to "safe", which follows redirects but rejects those targeting internal/private IPs (SSRF protection). Pass True to follow all redirects without restriction.
+        :param follow_redirects: Whether to follow redirects. Defaults to "safe", which follows redirects but rejects those targeting internal/private IPs (SSRF protection).
            Pass True to follow all redirects without restriction.
        :param max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited.
        :param retries: Number of retry attempts. Defaults to 3.
        :param retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second.
@@ -364,7 +437,8 @@ class ScraplingMCPServer:
        :param headers: Headers to include in the request.
        :param cookies: Cookies to use in the request.
        :param timeout: Number of seconds to wait before timing out.
-        :param follow_redirects: Whether to follow redirects. Defaults to "safe", which follows redirects but rejects those targeting internal/private IPs (SSRF protection). Pass True to follow all redirects without restriction.
+        :param follow_redirects: Whether to follow redirects. Defaults to "safe", which follows redirects but rejects those targeting internal/private IPs (SSRF protection).
            Pass True to follow all redirects without restriction.
        :param max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited.
        :param retries: Number of retry attempts. Defaults to 3.
        :param retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second.
@@ -828,4 +902,6 @@ class ScraplingMCPServer:
            description=self.bulk_stealthy_fetch.__doc__,
            structured_output=True,
        )
        # Screenshot tool (returns image + url content blocks, not structured JSON)
        server.add_tool(self.screenshot, title="screenshot", description=self.screenshot.__doc__)
        server.run(transport="stdio" if not http else "streamable-http")
@@ -149,6 +149,7 @@ class _ConfigurationLogic(ABC):
            # Browser session params (ignored by HTTP sessions)
            "extra_headers",
            "google_search",
            "block_ads",
        }
        for k, v in method_kwargs.items():
            if k not in skip_keys and v is not None:
@@ -718,8 +719,13 @@ class FetcherSession:
            config["selector_config"] = self.selector_config
            config["proxy_rotator"] = self._proxy_rotator
            self._client = _SyncSessionLogic(**config)
            try:
                result = self._client.__enter__()
            except Exception:
                self._client = None
                raise
            self._is_alive = True
-            return self._client.__enter__()
+            return result
        raise RuntimeError("This FetcherSession instance already has an active synchronous session.")
    def __exit__(self, exc_type, exc_val, exc_tb):
@@ -739,8 +745,13 @@ class FetcherSession:
            config["selector_config"] = self.selector_config
            config["proxy_rotator"] = self._proxy_rotator
            self._client = _ASyncSessionLogic(**config)
            try:
                result = await self._client.__aenter__()
            except Exception:
                self._client = None
                raise
            self._is_alive = True
-            return await self._client.__aenter__()
+            return result
        raise RuntimeError("This FetcherSession instance already has an active asynchronous session.")
    async def __aexit__(self, exc_type, exc_val, exc_tb):
@@ -93,7 +93,9 @@ class SessionManager:
    async def close(self) -> None:
        """Close all registered sessions."""
-        for session in self._sessions.values():
+        for sid, session in self._sessions.items():
            if sid in self._lazy_sessions and not session._is_alive:
                continue
            _ = await session.__aexit__(None, None, None)
        self._started = False
@@ -14,12 +14,12 @@
      "mimeType": "image/png"
    }
  ],
-  "version": "0.4.6",
+  "version": "0.4.7",
  "packages": [
    {
      "registryType": "pypi",
      "identifier": "scrapling",
-      "version": "0.4.6",
+      "version": "0.4.7",
      "runtimeHint": "uvx",
      "packageArguments": [
        {
@@ -1,6 +1,6 @@
 [metadata]
 name = scrapling
-version = 0.4.6
+version = 0.4.7
 author = Karim Shoair
 author_email = karim.shoair@pm.me
 description = Scrapling is an undetectable, powerful, flexible, high-performance Python library that makes Web Scraping easy and effortless as it should be!
@@ -1,5 +1,12 @@
 import base64
 import struct
 from contextlib import contextmanager
 from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
 from threading import Thread
 import pytest
 import pytest_httpbin
 from mcp.types import ImageContent, TextContent
 from scrapling.core.ai import (
    ScraplingMCPServer,
@@ -177,6 +184,137 @@ class TestSessionManagement:
        with pytest.raises(ValueError, match="not found"):
            await server.fetch(url=test_url, session_id=session_id)
    @pytest.mark.asyncio
    async def test_open_session_with_custom_id(self, server):
        """Test opening a session with a custom session_id"""
        result = await server.open_session(session_type="dynamic", session_id="my-session", headless=True)
        assert isinstance(result, SessionCreatedModel)
        assert result.session_id == "my-session"
        await server.close_session("my-session")
    @pytest.mark.asyncio
    async def test_open_session_duplicate_id_raises(self, server):
        """Test that opening a session with a duplicate session_id raises an error"""
        await server.open_session(session_type="dynamic", session_id="dupe", headless=True)
        with pytest.raises(ValueError, match="already exists"):
            await server.open_session(session_type="dynamic", session_id="dupe", headless=True)
        await server.close_session("dupe")
 def _png_height(data: bytes) -> int:
    """Read the height field from a PNG IHDR chunk."""
    return struct.unpack(">I", data[20:24])[0]
@contextmanager
 def _serve_html(body: bytes):
    """Serve a fixed HTML body on localhost, yielding its URL."""
    class _Handler(BaseHTTPRequestHandler):
        def do_GET(self):
            self.send_response(200)
            self.send_header("Content-Type", "text/html; charset=utf-8")
            self.send_header("Content-Length", str(len(body)))
            self.end_headers()
            self.wfile.write(body)
        def log_message(self, *args, **kwargs):
            pass
    server = ThreadingHTTPServer(("127.0.0.1", 0), _Handler)
    thread = Thread(target=server.serve_forever, daemon=True)
    thread.start()
    try:
        yield f"http://127.0.0.1:{server.server_address[1]}/"
    finally:
        server.shutdown()
        server.server_close()
@pytest_httpbin.use_class_based_httpbin
 class TestScreenshot:
    """Test the screenshot tool"""
    @pytest.fixture(scope="class")
    def test_url(self, httpbin):
        return f"{httpbin.url}/html"
    @pytest.fixture
    def server(self):
        return ScraplingMCPServer()
    @pytest.mark.asyncio
    async def test_screenshot_png_with_dynamic_session(self, server, test_url):
        """PNG screenshot via a dynamic session returns image and url content blocks"""
        opened = await server.open_session(session_type="dynamic", headless=True)
        try:
            result = await server.screenshot(url=test_url, session_id=opened.session_id)
            assert isinstance(result, list) and len(result) == 2
            assert isinstance(result[0], ImageContent)
            assert result[0].mimeType == "image/png"
            assert isinstance(result[1], TextContent)
            assert result[1].text == test_url
        finally:
            await server.close_session(opened.session_id)
    @pytest.mark.asyncio
    async def test_screenshot_jpeg_with_quality(self, server, test_url):
        """JPEG screenshot with quality parameter via a dynamic session"""
        opened = await server.open_session(session_type="dynamic", headless=True)
        try:
            result = await server.screenshot(url=test_url, session_id=opened.session_id, image_type="jpeg", quality=80)
            assert isinstance(result[0], ImageContent)
            assert result[0].mimeType == "image/jpeg"
        finally:
            await server.close_session(opened.session_id)
    @pytest.mark.asyncio
    async def test_screenshot_with_stealthy_session(self, server, test_url):
        """PNG screenshot via a stealthy session"""
        opened = await server.open_session(session_type="stealthy", headless=True)
        try:
            result = await server.screenshot(url=test_url, session_id=opened.session_id)
            assert isinstance(result[0], ImageContent)
            assert result[0].mimeType == "image/png"
        finally:
            await server.close_session(opened.session_id)
    @pytest.mark.asyncio
    async def test_screenshot_full_page_taller_than_viewport(self, server):
        """full_page=True produces an image taller than the viewport-only capture"""
        tall_html = b"<html><body><div style='height:5000px;background:#abc'></div></body></html>"
        with _serve_html(tall_html) as tall_url:
            opened = await server.open_session(session_type="dynamic", headless=True)
            try:
                viewport_result = await server.screenshot(url=tall_url, session_id=opened.session_id, full_page=False)
                full_result = await server.screenshot(url=tall_url, session_id=opened.session_id, full_page=True)
                viewport_png = base64.b64decode(viewport_result[0].data)
                full_png = base64.b64decode(full_result[0].data)
                assert _png_height(full_png) > _png_height(viewport_png)
            finally:
                await server.close_session(opened.session_id)
    @pytest.mark.asyncio
    async def test_screenshot_invalid_session_id_raises(self, server, test_url):
        """Unknown session_id raises ValueError"""
        with pytest.raises(ValueError, match="not found"):
            await server.screenshot(url=test_url, session_id="does-not-exist")
    @pytest.mark.asyncio
    async def test_screenshot_quality_with_png_raises(self, server, test_url):
        """quality is rejected when image_type is png"""
        opened = await server.open_session(session_type="dynamic", headless=True)
        try:
            with pytest.raises(ValueError, match="quality"):
                await server.screenshot(url=test_url, session_id=opened.session_id, image_type="png", quality=90)
        finally:
            await server.close_session(opened.session_id)
 class TestNormalizeCredentials:
    """Test the _normalize_credentials helper"""
@@ -0,0 +1,44 @@
 """Tests for _merge_request_args to ensure browser-only kwargs are excluded.
 Regression tests for https://github.com/D4Vinci/Scrapling/issues/247
 """
 import pytest
 from scrapling.engines.static import FetcherClient
 class TestMergeRequestArgsSkipsBrowserParams:
    """Verify that browser-only keyword arguments are stripped before
    the request dict is forwarded to curl_cffi's Session.request()."""
    def _build_args(self, **extra_kwargs):
        """Helper: instantiate a FetcherClient and call _merge_request_args."""
        client = FetcherClient()
        return client._merge_request_args(url="https://example.com", **extra_kwargs)
    def test_block_ads_excluded(self):
        """block_ads is a browser-engine param and must not leak into the
        HTTP request dict (fixes #247)."""
        args = self._build_args(block_ads=True)
        assert "block_ads" not in args
    def test_google_search_excluded(self):
        """google_search is a browser-engine param and should be stripped."""
        args = self._build_args(google_search=True)
        assert "google_search" not in args
    def test_extra_headers_excluded(self):
        """extra_headers is a browser-engine param and should be stripped."""
        args = self._build_args(extra_headers={"X-Custom": "val"})
        assert "extra_headers" not in args
    def test_url_present(self):
        """The url must always be present in the output dict."""
        args = self._build_args()
        assert args["url"] == "https://example.com"
    def test_valid_kwargs_passed_through(self):
        """Arbitrary curl_cffi-compatible kwargs should survive."""
        args = self._build_args(cookies={"session": "abc"})
        assert args.get("cookies") == {"session": "abc"}