From 00897dad2f60dc632e4f33972e7ae0c0889b1fff Mon Sep 17 00:00:00 2001 From: Jules Omlor Date: Tue, 14 Apr 2026 01:17:52 -0400 Subject: [PATCH 1/3] feat(mcp): add optional session_id parameter to open_session Allow users to specify a custom session_id when opening a browser session, rather than always generating a random UUID. Useful for naming sessions for easier management across multiple tool calls. - Add session_id: Optional[str] = None parameter - Validate session_id doesn't already exist before starting browser - Fall back to uuid4().hex[:12] if not provided - Add tests for custom session_id and duplicate detection Co-Authored-By: Claude Opus 4.5 --- scrapling/core/ai.py | 7 ++++++- tests/ai/test_ai_mcp.py | 19 +++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/scrapling/core/ai.py b/scrapling/core/ai.py index 060cb20..7292a83 100644 --- a/scrapling/core/ai.py +++ b/scrapling/core/ai.py @@ -123,6 +123,7 @@ class ScraplingMCPServer: async def open_session( self, session_type: SessionType, + session_id: Optional[str] = None, headless: bool = True, google_search: bool = True, real_chrome: bool = False, @@ -152,6 +153,7 @@ class ScraplingMCPServer: Use close_session to close the session when done, and list_sessions to see all active sessions. :param session_type: The type of session to open. Use "dynamic" for standard Playwright browser, or "stealthy" for anti-bot bypass with fingerprint spoofing. + :param session_id: Optional custom session ID. If not provided, a random 12-character hex ID will be generated. Useful for naming sessions for easier management. :param headless: Run the browser in headless/hidden (default), or headful/visible mode. :param google_search: Enabled by default, Scrapling will set a Google referer header. :param real_chrome: If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch an instance of your browser and use it. @@ -175,6 +177,10 @@ class ScraplingMCPServer: :param solve_cloudflare: (Stealthy only) Solves all types of the Cloudflare's Turnstile/Interstitial challenges. :param additional_args: (Stealthy only) Additional arguments to be passed to Playwright's context as additional settings. """ + session_id = session_id or uuid4().hex[:12] + if session_id in self._sessions: + raise ValueError(f"Session '{session_id}' already exists. Use a different ID or close the existing session first.") + common_kwargs: Dict[str, Any] = dict( wait=wait, proxy=proxy, @@ -211,7 +217,6 @@ class ScraplingMCPServer: await session.start() - session_id = uuid4().hex[:12] entry = _SessionEntry(session=session, session_type=session_type) self._sessions[session_id] = entry diff --git a/tests/ai/test_ai_mcp.py b/tests/ai/test_ai_mcp.py index d897bb5..4806088 100644 --- a/tests/ai/test_ai_mcp.py +++ b/tests/ai/test_ai_mcp.py @@ -177,6 +177,25 @@ class TestSessionManagement: with pytest.raises(ValueError, match="not found"): await server.fetch(url=test_url, session_id=session_id) + @pytest.mark.asyncio + async def test_open_session_with_custom_id(self, server): + """Test opening a session with a custom session_id""" + result = await server.open_session(session_type="dynamic", session_id="my-session", headless=True) + assert isinstance(result, SessionCreatedModel) + assert result.session_id == "my-session" + + await server.close_session("my-session") + + @pytest.mark.asyncio + async def test_open_session_duplicate_id_raises(self, server): + """Test that opening a session with a duplicate session_id raises an error""" + await server.open_session(session_type="dynamic", session_id="dupe", headless=True) + + with pytest.raises(ValueError, match="already exists"): + await server.open_session(session_type="dynamic", session_id="dupe", headless=True) + + await server.close_session("dupe") + class TestNormalizeCredentials: """Test the _normalize_credentials helper""" From 614d136f8cb9f0c31e2720189f25b26472c42d68 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Wed, 15 Apr 2026 20:44:38 +0200 Subject: [PATCH 2/3] build: pump up version and deps --- agent-skill/Scrapling-Skill/SKILL.md | 4 ++-- agent-skill/Scrapling-Skill/examples/README.md | 2 +- pyproject.toml | 6 +++--- scrapling/__init__.py | 2 +- scrapling/core/ai.py | 4 +++- server.json | 4 ++-- setup.cfg | 2 +- 7 files changed, 13 insertions(+), 11 deletions(-) diff --git a/agent-skill/Scrapling-Skill/SKILL.md b/agent-skill/Scrapling-Skill/SKILL.md index e1fc735..2cd1f84 100644 --- a/agent-skill/Scrapling-Skill/SKILL.md +++ b/agent-skill/Scrapling-Skill/SKILL.md @@ -1,7 +1,7 @@ --- name: scrapling-official description: Scrape web pages using Scrapling with anti-bot bypass (like Cloudflare Turnstile), stealth headless browsing, spiders framework, adaptive scraping, and JavaScript rendering. Use when asked to scrape, crawl, or extract data from websites; web_fetch fails; the site has anti-bot protections; write Python code to scrape/crawl; or write spiders. -version: "0.4.6" +version: "0.4.7" license: Complete terms in LICENSE.txt metadata: homepage: "https://scrapling.readthedocs.io/en/latest/index.html" @@ -40,7 +40,7 @@ Blazing fast crawls with real-time stats and streaming. Built by Web Scrapers fo Create a virtual Python environment through any way available, like `venv`, then inside the environment do: -`pip install "scrapling[all]>=0.4.6"` +`pip install "scrapling[all]>=0.4.7"` Then do this to download all the browsers' dependencies: diff --git a/agent-skill/Scrapling-Skill/examples/README.md b/agent-skill/Scrapling-Skill/examples/README.md index 4f645cd..388a594 100644 --- a/agent-skill/Scrapling-Skill/examples/README.md +++ b/agent-skill/Scrapling-Skill/examples/README.md @@ -9,7 +9,7 @@ All examples collect **all 100 quotes across 10 pages**. Make sure Scrapling is installed: ```bash -pip install "scrapling[all]>=0.4.6" +pip install "scrapling[all]>=0.4.7" scrapling install --force ``` diff --git a/pyproject.toml b/pyproject.toml index e8a5500..ada11a4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta" [project] name = "scrapling" # Static version instead of a dynamic version so we can get better layer caching while building docker, check the docker file to understand -version = "0.4.6" +version = "0.4.7" description = "Scrapling is an undetectable, powerful, flexible, high-performance Python library that makes Web Scraping easy and effortless as it should be!" readme = {file = "README.md", content-type = "text/markdown"} license = {file = "LICENSE"} @@ -77,8 +77,8 @@ fetchers = [ "patchright==1.58.2", "browserforge>=1.2.4", "apify-fingerprint-datapoints>=0.12.0", - "msgspec>=0.21.0", - "anyio>=4.12.1", + "msgspec>=0.21.1", + "anyio>=4.13.0", "protego>=0.6.0", ] ai = [ diff --git a/scrapling/__init__.py b/scrapling/__init__.py index c0c66ff..97af0c5 100644 --- a/scrapling/__init__.py +++ b/scrapling/__init__.py @@ -1,5 +1,5 @@ __author__ = "Karim Shoair (karim.shoair@pm.me)" -__version__ = "0.4.6" +__version__ = "0.4.7" __copyright__ = "Copyright (c) 2024 Karim Shoair" from typing import Any, TYPE_CHECKING diff --git a/scrapling/core/ai.py b/scrapling/core/ai.py index 7292a83..315733f 100644 --- a/scrapling/core/ai.py +++ b/scrapling/core/ai.py @@ -179,7 +179,9 @@ class ScraplingMCPServer: """ session_id = session_id or uuid4().hex[:12] if session_id in self._sessions: - raise ValueError(f"Session '{session_id}' already exists. Use a different ID or close the existing session first.") + raise ValueError( + f"Session '{session_id}' already exists. Use a different ID or close the existing session first." + ) common_kwargs: Dict[str, Any] = dict( wait=wait, diff --git a/server.json b/server.json index 36f60e7..5415056 100644 --- a/server.json +++ b/server.json @@ -14,12 +14,12 @@ "mimeType": "image/png" } ], - "version": "0.4.6", + "version": "0.4.7", "packages": [ { "registryType": "pypi", "identifier": "scrapling", - "version": "0.4.6", + "version": "0.4.7", "runtimeHint": "uvx", "packageArguments": [ { diff --git a/setup.cfg b/setup.cfg index 0794d59..72d64fc 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = scrapling -version = 0.4.6 +version = 0.4.7 author = Karim Shoair author_email = karim.shoair@pm.me description = Scrapling is an undetectable, powerful, flexible, high-performance Python library that makes Web Scraping easy and effortless as it should be! From f4186ab9987fcc54e978dafd76492b09a3e15eea Mon Sep 17 00:00:00 2001 From: yetval Date: Wed, 15 Apr 2026 21:48:24 -0400 Subject: [PATCH 3/3] fix: prevent FetcherSession state corruption and lazy session close crash --- scrapling/engines/static.py | 14 ++++++++++++-- scrapling/spiders/session.py | 4 +++- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/scrapling/engines/static.py b/scrapling/engines/static.py index 1f4b09b..b74c730 100644 --- a/scrapling/engines/static.py +++ b/scrapling/engines/static.py @@ -716,8 +716,13 @@ class FetcherSession: config["selector_config"] = self.selector_config config["proxy_rotator"] = self._proxy_rotator self._client = _SyncSessionLogic(**config) + try: + result = self._client.__enter__() + except Exception: + self._client = None + raise self._is_alive = True - return self._client.__enter__() + return result raise RuntimeError("This FetcherSession instance already has an active synchronous session.") def __exit__(self, exc_type, exc_val, exc_tb): @@ -737,8 +742,13 @@ class FetcherSession: config["selector_config"] = self.selector_config config["proxy_rotator"] = self._proxy_rotator self._client = _ASyncSessionLogic(**config) + try: + result = await self._client.__aenter__() + except Exception: + self._client = None + raise self._is_alive = True - return await self._client.__aenter__() + return result raise RuntimeError("This FetcherSession instance already has an active asynchronous session.") async def __aexit__(self, exc_type, exc_val, exc_tb): diff --git a/scrapling/spiders/session.py b/scrapling/spiders/session.py index 536be6d..5799e8c 100644 --- a/scrapling/spiders/session.py +++ b/scrapling/spiders/session.py @@ -93,7 +93,9 @@ class SessionManager: async def close(self) -> None: """Close all registered sessions.""" - for session in self._sessions.values(): + for sid, session in self._sessions.items(): + if sid in self._lazy_sessions and not session._is_alive: + continue _ = await session.__aexit__(None, None, None) self._started = False