From ac2d4d3833d95aa1ce444f5fcaec3b4e3abc1f1e Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Tue, 2 Dec 2025 02:15:42 +0200 Subject: [PATCH 1/4] fix: use the correct signature for the `stealthy_fetch` shortcut --- scrapling/core/shell.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scrapling/core/shell.py b/scrapling/core/shell.py index 143d8be..0499e81 100644 --- a/scrapling/core/shell.py +++ b/scrapling/core/shell.py @@ -314,7 +314,7 @@ class CurlParser: return None -def _unpack_signature(func): +def _unpack_signature(func, signature_name=None): """ Unpack TypedDict from Unpack[TypedDict] annotations in **kwargs and reconstruct the signature. @@ -322,7 +322,7 @@ def _unpack_signature(func): """ try: sig = signature(func) - func_name = getattr(func, "__name__", None) + func_name = signature_name or getattr(func, "__name__", None) # Check if this function has known parameters if func_name not in Signatures_map: @@ -467,7 +467,7 @@ Type 'exit' or press Ctrl+D to exit. return result - def create_wrapper(self, func, get_signature=True): + def create_wrapper(self, func, get_signature=True, signature_name=None): """Create a wrapper that preserves function signature but updates page""" @wraps(func) @@ -477,7 +477,7 @@ Type 'exit' or press Ctrl+D to exit. if get_signature: # Explicitly preserve and unpack signature for IPython introspection and autocompletion - wrapper.__signature__ = _unpack_signature(func) # pyright: ignore + wrapper.__signature__ = _unpack_signature(func, signature_name) # pyright: ignore else: wrapper.__signature__ = signature(func) # pyright: ignore @@ -492,7 +492,7 @@ Type 'exit' or press Ctrl+D to exit. put = self.create_wrapper(self.__Fetcher.put) delete = self.create_wrapper(self.__Fetcher.delete) dynamic_fetch = self.create_wrapper(self.__DynamicFetcher.fetch) - stealthy_fetch = self.create_wrapper(self.__StealthyFetcher.fetch) + stealthy_fetch = self.create_wrapper(self.__StealthyFetcher.fetch, signature_name="stealthy_fetch") curl2fetcher = self.create_wrapper(self._curl_parser.convert2fetcher, get_signature=False) # Create the namespace dictionary From 16ff07a28765d39e87fc199c5191fcfbcbd1d3a2 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Wed, 3 Dec 2025 02:20:18 +0200 Subject: [PATCH 2/4] build: Pump version up and mcp --- pyproject.toml | 4 ++-- scrapling/__init__.py | 2 +- setup.cfg | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index cd2145b..c9e04cc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta" [project] name = "scrapling" # Static version instead of a dynamic version so we can get better layer caching while building docker, check the docker file to understand -version = "0.3.10" +version = "0.3.11" description = "Scrapling is an undetectable, powerful, flexible, high-performance Python library that makes Web Scraping easy and effortless as it should be!" readme = {file = "docs/README.md", content-type = "text/markdown"} license = {file = "LICENSE"} @@ -74,7 +74,7 @@ fetchers = [ "msgspec>=0.20.0", ] ai = [ - "mcp>=1.19.0", + "mcp>=1.23.0", "markdownify>=1.2.0", "scrapling[fetchers]", ] diff --git a/scrapling/__init__.py b/scrapling/__init__.py index 214a0f9..7cd7812 100644 --- a/scrapling/__init__.py +++ b/scrapling/__init__.py @@ -1,5 +1,5 @@ __author__ = "Karim Shoair (karim.shoair@pm.me)" -__version__ = "0.3.10" +__version__ = "0.3.11" __copyright__ = "Copyright (c) 2024 Karim Shoair" from typing import Any, TYPE_CHECKING diff --git a/setup.cfg b/setup.cfg index 8a55f35..a37bb14 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = scrapling -version = 0.3.10 +version = 0.3.11 author = Karim Shoair author_email = karim.shoair@pm.me description = Scrapling is an undetectable, powerful, flexible, high-performance Python library that makes Web Scraping easy and effortless as it should be! From aed34e43b5eb0d9b6df337f8f7bf71c75ce717df Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Wed, 3 Dec 2025 03:11:30 +0200 Subject: [PATCH 3/4] fix(fetchers): better handling for infinite `network_idle` --- scrapling/engines/_browsers/_base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapling/engines/_browsers/_base.py b/scrapling/engines/_browsers/_base.py index f329fa4..6e5734e 100644 --- a/scrapling/engines/_browsers/_base.py +++ b/scrapling/engines/_browsers/_base.py @@ -105,7 +105,7 @@ class SyncSession: """Wait for the page to become idle (no network activity) even if there are never-ending requests.""" try: page.wait_for_load_state("networkidle", timeout=timeout) - except PlaywrightError: + except (PlaywrightError, Exception): pass def _wait_for_page_stability(self, page: Page | Frame, load_dom: bool, network_idle: bool): @@ -221,7 +221,7 @@ class AsyncSession: """Wait for the page to become idle (no network activity) even if there are never-ending requests.""" try: await page.wait_for_load_state("networkidle", timeout=timeout) - except PlaywrightError: + except (PlaywrightError, Exception): pass async def _wait_for_page_stability(self, page: AsyncPage | AsyncFrame, load_dom: bool, network_idle: bool): From 51d347270f1ecf68fbaff0beb276fc1e96467ecd Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Wed, 3 Dec 2025 03:43:17 +0200 Subject: [PATCH 4/4] build: remove unnecessary reinstallation of deps in tox --- tox.ini | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tox.ini b/tox.ini index 0109863..758939f 100644 --- a/tox.ini +++ b/tox.ini @@ -15,11 +15,6 @@ deps = camoufox -r{toxinidir}/tests/requirements.txt extras = ai,shell -commands_pre = - # Install browsers in the tox virtual environment - python -m playwright install chromium - python -m playwright install-deps chromium firefox - python -m camoufox fetch --browserforge commands = # Run browser tests without parallelization (avoid browser conflicts) pytest --cov=scrapling --cov-report=xml -k "DynamicFetcher or StealthyFetcher" --verbose