diff --git a/pyproject.toml b/pyproject.toml index cd2145b..c9e04cc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta" [project] name = "scrapling" # Static version instead of a dynamic version so we can get better layer caching while building docker, check the docker file to understand -version = "0.3.10" +version = "0.3.11" description = "Scrapling is an undetectable, powerful, flexible, high-performance Python library that makes Web Scraping easy and effortless as it should be!" readme = {file = "docs/README.md", content-type = "text/markdown"} license = {file = "LICENSE"} @@ -74,7 +74,7 @@ fetchers = [ "msgspec>=0.20.0", ] ai = [ - "mcp>=1.19.0", + "mcp>=1.23.0", "markdownify>=1.2.0", "scrapling[fetchers]", ] diff --git a/scrapling/__init__.py b/scrapling/__init__.py index 214a0f9..7cd7812 100644 --- a/scrapling/__init__.py +++ b/scrapling/__init__.py @@ -1,5 +1,5 @@ __author__ = "Karim Shoair (karim.shoair@pm.me)" -__version__ = "0.3.10" +__version__ = "0.3.11" __copyright__ = "Copyright (c) 2024 Karim Shoair" from typing import Any, TYPE_CHECKING diff --git a/scrapling/core/shell.py b/scrapling/core/shell.py index 143d8be..0499e81 100644 --- a/scrapling/core/shell.py +++ b/scrapling/core/shell.py @@ -314,7 +314,7 @@ class CurlParser: return None -def _unpack_signature(func): +def _unpack_signature(func, signature_name=None): """ Unpack TypedDict from Unpack[TypedDict] annotations in **kwargs and reconstruct the signature. @@ -322,7 +322,7 @@ def _unpack_signature(func): """ try: sig = signature(func) - func_name = getattr(func, "__name__", None) + func_name = signature_name or getattr(func, "__name__", None) # Check if this function has known parameters if func_name not in Signatures_map: @@ -467,7 +467,7 @@ Type 'exit' or press Ctrl+D to exit. return result - def create_wrapper(self, func, get_signature=True): + def create_wrapper(self, func, get_signature=True, signature_name=None): """Create a wrapper that preserves function signature but updates page""" @wraps(func) @@ -477,7 +477,7 @@ Type 'exit' or press Ctrl+D to exit. if get_signature: # Explicitly preserve and unpack signature for IPython introspection and autocompletion - wrapper.__signature__ = _unpack_signature(func) # pyright: ignore + wrapper.__signature__ = _unpack_signature(func, signature_name) # pyright: ignore else: wrapper.__signature__ = signature(func) # pyright: ignore @@ -492,7 +492,7 @@ Type 'exit' or press Ctrl+D to exit. put = self.create_wrapper(self.__Fetcher.put) delete = self.create_wrapper(self.__Fetcher.delete) dynamic_fetch = self.create_wrapper(self.__DynamicFetcher.fetch) - stealthy_fetch = self.create_wrapper(self.__StealthyFetcher.fetch) + stealthy_fetch = self.create_wrapper(self.__StealthyFetcher.fetch, signature_name="stealthy_fetch") curl2fetcher = self.create_wrapper(self._curl_parser.convert2fetcher, get_signature=False) # Create the namespace dictionary diff --git a/scrapling/engines/_browsers/_base.py b/scrapling/engines/_browsers/_base.py index f329fa4..6e5734e 100644 --- a/scrapling/engines/_browsers/_base.py +++ b/scrapling/engines/_browsers/_base.py @@ -105,7 +105,7 @@ class SyncSession: """Wait for the page to become idle (no network activity) even if there are never-ending requests.""" try: page.wait_for_load_state("networkidle", timeout=timeout) - except PlaywrightError: + except (PlaywrightError, Exception): pass def _wait_for_page_stability(self, page: Page | Frame, load_dom: bool, network_idle: bool): @@ -221,7 +221,7 @@ class AsyncSession: """Wait for the page to become idle (no network activity) even if there are never-ending requests.""" try: await page.wait_for_load_state("networkidle", timeout=timeout) - except PlaywrightError: + except (PlaywrightError, Exception): pass async def _wait_for_page_stability(self, page: AsyncPage | AsyncFrame, load_dom: bool, network_idle: bool): diff --git a/setup.cfg b/setup.cfg index 8a55f35..a37bb14 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = scrapling -version = 0.3.10 +version = 0.3.11 author = Karim Shoair author_email = karim.shoair@pm.me description = Scrapling is an undetectable, powerful, flexible, high-performance Python library that makes Web Scraping easy and effortless as it should be! diff --git a/tox.ini b/tox.ini index 0109863..758939f 100644 --- a/tox.ini +++ b/tox.ini @@ -15,11 +15,6 @@ deps = camoufox -r{toxinidir}/tests/requirements.txt extras = ai,shell -commands_pre = - # Install browsers in the tox virtual environment - python -m playwright install chromium - python -m playwright install-deps chromium firefox - python -m camoufox fetch --browserforge commands = # Run browser tests without parallelization (avoid browser conflicts) pytest --cov=scrapling --cov-report=xml -k "DynamicFetcher or StealthyFetcher" --verbose