From e9344a738409830adfe97eec21e6ea34504d39c3 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Wed, 5 Mar 2025 03:11:46 +0200 Subject: [PATCH 1/5] fix(TextHandler): Fixing an issue with slicing Slicing was returning `TextHandlers` instead of `TextHandler`. This solves #41 and corrects the type hint --- scrapling/core/custom_types.py | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/scrapling/core/custom_types.py b/scrapling/core/custom_types.py index 30f61e1..67e4e00 100644 --- a/scrapling/core/custom_types.py +++ b/scrapling/core/custom_types.py @@ -23,19 +23,8 @@ class TextHandler(str): return super().__new__(cls, string) return super().__new__(cls, '') - @typing.overload - def __getitem__(self, key: SupportsIndex) -> 'TextHandler': - pass - - @typing.overload - def __getitem__(self, key: slice) -> "TextHandlers": - pass - - def __getitem__(self, key: Union[SupportsIndex, slice]) -> Union["TextHandler", "TextHandlers"]: + def __getitem__(self, key: Union[SupportsIndex, slice]) -> "TextHandler": lst = super().__getitem__(key) - if isinstance(key, slice): - lst = [TextHandler(s) for s in lst] - return TextHandlers(typing.cast(List[_TextHandlerType], lst)) return typing.cast(_TextHandlerType, TextHandler(lst)) def split(self, sep: str = None, maxsplit: SupportsIndex = -1) -> 'TextHandlers': From 5f06a34a2ff3b3bc0913e6f0138121525d0e5b18 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Wed, 5 Mar 2025 03:13:47 +0200 Subject: [PATCH 2/5] build: Pumping version up --- scrapling/__init__.py | 2 +- setup.cfg | 2 +- setup.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/scrapling/__init__.py b/scrapling/__init__.py index c6908db..13b01fe 100644 --- a/scrapling/__init__.py +++ b/scrapling/__init__.py @@ -5,7 +5,7 @@ from scrapling.fetchers import (AsyncFetcher, CustomFetcher, Fetcher, from scrapling.parser import Adaptor, Adaptors __author__ = "Karim Shoair (karim.shoair@pm.me)" -__version__ = "0.2.95" +__version__ = "0.2.96" __copyright__ = "Copyright (c) 2024 Karim Shoair" diff --git a/setup.cfg b/setup.cfg index ad4de36..cf8a52f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = scrapling -version = 0.2.95 +version = 0.2.96 author = Karim Shoair author_email = karim.shoair@pm.me description = Scrapling is an undetectable, powerful, flexible, high-performance Python library that makes Web Scraping easy again! diff --git a/setup.py b/setup.py index 81afabd..cb2ff3d 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ with open("README.md", "r", encoding="utf-8") as fh: setup( name="scrapling", - version="0.2.95", + version="0.2.96", description="""Scrapling is an undetectable, powerful, flexible, high-performance Python library that makes Web Scraping easy again! In an internet filled with complications, it simplifies web scraping, even when websites' design changes, while providing impressive speed that surpasses almost all alternatives.""", long_description=long_description, From 7ed189aa4e42f7916c887299b9698ab0a9fc5e79 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Wed, 5 Mar 2025 03:15:11 +0200 Subject: [PATCH 3/5] docs: Updating the README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f299b3b..c08d02f 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# 🕷️ Scrapling: Undetectable, Lightning-Fast, and Adaptive Web Scraping for Python +# 🕷️ Scrapling: Undetectable, Lightning-Fast, and Easy Web Scraping with Python [![Tests](https://github.com/D4Vinci/Scrapling/actions/workflows/tests.yml/badge.svg)](https://github.com/D4Vinci/Scrapling/actions/workflows/tests.yml) [![PyPI version](https://badge.fury.io/py/Scrapling.svg)](https://badge.fury.io/py/Scrapling) [![Supported Python versions](https://img.shields.io/pypi/pyversions/scrapling.svg)](https://pypi.org/project/scrapling/) [![PyPI Downloads](https://static.pepy.tech/badge/scrapling)](https://pepy.tech/project/scrapling) Dealing with failing web scrapers due to anti-bot protections or website changes? Meet Scrapling. From 087a85d615bb8aa4ffd0579f380e505b6757fdbb Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Wed, 5 Mar 2025 03:28:07 +0200 Subject: [PATCH 4/5] fix(cli): Fix issue where install drops user into Python Shell --- scrapling/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapling/cli.py b/scrapling/cli.py index 7d22f0c..be2c257 100644 --- a/scrapling/cli.py +++ b/scrapling/cli.py @@ -12,7 +12,7 @@ def get_package_dir(): def run_command(command, line): print(f"Installing {line}...") - _ = subprocess.check_call(command, shell=True) + _ = subprocess.check_call(' '.join(command), shell=True) # I meant to not use try except here From f8662bee29ad7cf312a805812798d55df394437f Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Wed, 5 Mar 2025 03:32:43 +0200 Subject: [PATCH 5/5] feat(cli): Adding the `-f` flag to `install` command --- scrapling/cli.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scrapling/cli.py b/scrapling/cli.py index be2c257..58ad43c 100644 --- a/scrapling/cli.py +++ b/scrapling/cli.py @@ -17,8 +17,9 @@ def run_command(command, line): @click.command(help="Install all Scrapling's Fetchers dependencies") -def install(): - if not get_package_dir().joinpath(".scrapling_dependencies_installed").exists(): +@click.option('-f', '--force', 'force', is_flag=True, default=False, type=bool, help="Force Scrapling to reinstall all Fetchers dependencies") +def install(force): + if force or not get_package_dir().joinpath(".scrapling_dependencies_installed").exists(): run_command([sys.executable, "-m", "playwright", "install", 'chromium'], 'Playwright browsers') run_command([sys.executable, "-m", "playwright", "install-deps", 'chromium', 'firefox'], 'Playwright dependencies') run_command([sys.executable, "-m", "camoufox", "fetch", '--browserforge'], 'Camoufox browser and databases')