From 21291958f178f88b107c5370f424c1b470b0b627 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Tue, 28 Jan 2025 23:10:56 +0200 Subject: [PATCH 01/30] docs(README file): Better more-friendly description --- README.md | 47 ++++++++++++++++++++++++----------------------- 1 file changed, 24 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index ba123fd..81702a5 100644 --- a/README.md +++ b/README.md @@ -92,27 +92,27 @@ Scrapling is a high-performance, intelligent web scraping library for Python tha ## Key Features ### Fetch websites as you prefer with async support -- **HTTP requests**: Stealthy and fast HTTP requests with `Fetcher` -- **Stealthy fetcher**: Annoying anti-bot protection? No problem! Scrapling can bypass almost all of them with `StealthyFetcher` with default configuration! -- **Your preferred browser**: Use your real browser with CDP, [NSTbrowser](https://app.nstbrowser.io/r/1vO5e5)'s browserless, PlayWright with stealth mode, or even vanilla PlayWright - All is possible with `PlayWrightFetcher`! +- **HTTP Requests**: Fast and stealthy HTTP requests with the `Fetcher` class. +- **Dynamic Loading & Automation**: Fetch dynamic websites with the `PlayWrightFetcher` class through your real browser, Scrapling's stealth mode, Playwright's Chrome browser, or [NSTbrowser](https://app.nstbrowser.io/r/1vO5e5)'s browserless! +- **Anti-bot Protections Bypass**: Easily bypass protections with `StealthyFetcher` and `PlayWrightFetcher` classes. ### Adaptive Scraping -- 🔄 **Smart Element Tracking**: Locate previously identified elements after website structure changes, using an intelligent similarity system and integrated storage. -- 🎯 **Flexible Querying**: Use CSS selectors, XPath, Elements filters, text search, or regex - chain them however you want! -- 🔍 **Find Similar Elements**: Automatically locate elements similar to the element you want on the page (Ex: other products like the product you found on the page). +- 🔄 **Smart Element Tracking**: Relocate elements after website changes, using an intelligent similarity system and integrated storage. +- 🎯 **Flexible Selection**: CSS selectors, XPath selectors, filters-based search, text search, regex search and more. +- 🔍 **Find Similar Elements**: Automatically locate elements similar to the element you found! - 🧠 **Smart Content Scraping**: Extract data from multiple websites without specific selectors using Scrapling powerful features. -### Performance -- 🚀 **Lightning Fast**: Built from the ground up with performance in mind, outperforming most popular Python scraping libraries (outperforming BeautifulSoup in parsing by up to 620x in our tests). +### High Performance +- 🚀 **Lightning Fast**: Built from the ground up with performance in mind, outperforming most popular Python scraping libraries. - 🔋 **Memory Efficient**: Optimized data structures for minimal memory footprint. -- ⚡ **Fast JSON serialization**: 10x faster JSON serialization than the standard json library with more options. +- ⚡ **Fast JSON serialization**: 10x faster than standard library. -### Developing Experience -- 🛠️ **Powerful Navigation API**: Traverse the DOM tree easily in all directions and get the info you want (parent, ancestors, sibling, children, next/previous element, and more). -- 🧬 **Rich Text Processing**: All strings have built-in methods for regex matching, cleaning, and more. All elements' attributes are read-only dictionaries that are faster than standard dictionaries with added methods. -- 📝 **Automatic Selector Generation**: Create robust CSS/XPath selectors for any element. -- 🔌 **API Similar to Scrapy/BeautifulSoup**: Familiar methods and similar pseudo-elements for Scrapy and BeautifulSoup users. -- 📘 **Type hints and test coverage**: Complete type coverage and almost full test coverage for better IDE support and fewer bugs, respectively. +### Developer Friendly +- 🛠️ **Powerful Navigation API**: Easy DOM traversal in all directions. +- 🧬 **Rich Text Processing**: All strings have built-in regex, cleaning methods, and more. All elements' attributes are optimized dictionaries that takes less memory than standard dictionaries with added methods. +- 📝 **Auto Selectors Generation**: Generate robust short and full CSS/XPath selectors for any element. +- 🔌 **Familiar API**: Similar to Scrapy/BeautifulSoup and the same pseudo-elements used in Scrapy. +- 📘 **Type hints**: Complete type/doc-strings coverage for future-proofing and best autocompletion support. ## Getting Started @@ -121,21 +121,22 @@ from scrapling import Fetcher fetcher = Fetcher(auto_match=False) -# Fetch a web page and create an Adaptor instance +# Do http GET request to a web page and create an Adaptor instance page = fetcher.get('https://quotes.toscrape.com/', stealthy_headers=True) -# Get all strings in the full page +# Get all text content from all HTML tags in the page except `script` and `style` tags page.get_all_text(ignore_tags=('script', 'style')) -# Get all quotes, any of these methods will return a list of strings (TextHandlers) +# Get all quotes elements, any of these methods will return a list of strings directly (TextHandlers) quotes = page.css('.quote .text::text') # CSS selector quotes = page.xpath('//span[@class="text"]/text()') # XPath quotes = page.css('.quote').css('.text::text') # Chained selectors quotes = [element.text for element in page.css('.quote .text')] # Slower than bulk query above # Get the first quote element -quote = page.css_first('.quote') # / page.css('.quote').first / page.css('.quote')[0] +quote = page.css_first('.quote') # same as page.css('.quote').first or page.css('.quote')[0] # Tired of selectors? Use find_all/find +# Get all 'div' HTML tags that one of its 'class' values is 'quote' quotes = page.find_all('div', {'class': 'quote'}) # Same as quotes = page.find_all('div', class_='quote') @@ -143,10 +144,10 @@ quotes = page.find_all(['div'], class_='quote') quotes = page.find_all(class_='quote') # and so on... # Working with elements -quote.html_content # Inner HTML -quote.prettify() # Prettified version of Inner HTML -quote.attrib # Element attributes -quote.path # DOM path to element (List) +quote.html_content # Get Inner HTML of this element +quote.prettify() # Prettified version of Inner HTML above +quote.attrib # Get that element's attributes +quote.path # DOM path to element (List of all ancestors from tag till the element itself) ``` To keep it simple, all methods can be chained on top of each other! From 99fe842bc91ee8842cd3a59e99b638efbd9d7094 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Tue, 28 Jan 2025 23:40:56 +0200 Subject: [PATCH 02/30] build: updating dependencies --- setup.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 3af52b9..de8751b 100644 --- a/setup.py +++ b/setup.py @@ -52,8 +52,7 @@ setup( ], # Instead of using requirements file to dodge possible errors from tox? install_requires=[ - "requests>=2.3", - "lxml>=4.5", + "lxml>=5.0", "cssselect>=1.2", 'click', "w3lib", @@ -62,7 +61,7 @@ setup( 'httpx[brotli,zstd, socks]', 'playwright>=1.49.1', 'rebrowser-playwright>=1.49.1', - 'camoufox[geoip]>=0.4.9' + 'camoufox[geoip]>=0.4.10' ], python_requires=">=3.9", url="https://github.com/D4Vinci/Scrapling", From c2447e069e9d8dbc01f5a3e4415d7693079a0764 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Tue, 28 Jan 2025 23:41:48 +0200 Subject: [PATCH 03/30] build: Pumping version up --- scrapling/__init__.py | 2 +- setup.cfg | 2 +- setup.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/scrapling/__init__.py b/scrapling/__init__.py index a80c50f..7e099c8 100644 --- a/scrapling/__init__.py +++ b/scrapling/__init__.py @@ -5,7 +5,7 @@ from scrapling.fetchers import (AsyncFetcher, CustomFetcher, Fetcher, from scrapling.parser import Adaptor, Adaptors __author__ = "Karim Shoair (karim.shoair@pm.me)" -__version__ = "0.2.92" +__version__ = "0.2.93" __copyright__ = "Copyright (c) 2024 Karim Shoair" diff --git a/setup.cfg b/setup.cfg index 4b945f7..780469c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = scrapling -version = 0.2.92 +version = 0.2.93 author = Karim Shoair author_email = karim.shoair@pm.me description = Scrapling is an undetectable, powerful, flexible, adaptive, and high-performance web scraping library for Python. diff --git a/setup.py b/setup.py index de8751b..fa11763 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ with open("README.md", "r", encoding="utf-8") as fh: setup( name="scrapling", - version="0.2.92", + version="0.2.93", description="""Scrapling is a powerful, flexible, and high-performance web scraping library for Python. It simplifies the process of extracting data from websites, even when they undergo structural changes, and offers impressive speed improvements over many popular scraping tools.""", From 1332be7e63c3f71115dd95634c5804c6d0e9f9fc Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Tue, 28 Jan 2025 23:44:58 +0200 Subject: [PATCH 04/30] fix(autocompletion): Fix the defaults import It runs autocompletion on some IDEs --- scrapling/defaults.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/scrapling/defaults.py b/scrapling/defaults.py index 64fd1b7..20f0c44 100644 --- a/scrapling/defaults.py +++ b/scrapling/defaults.py @@ -1,7 +1,10 @@ -from .fetchers import AsyncFetcher, Fetcher, PlayWrightFetcher, StealthyFetcher +from .fetchers import AsyncFetcher as _AsyncFetcher +from .fetchers import Fetcher as _Fetcher +from .fetchers import PlayWrightFetcher as _PlayWrightFetcher +from .fetchers import StealthyFetcher as _StealthyFetcher # If you are going to use Fetchers with the default settings, import them from this file instead for a cleaner looking code -Fetcher = Fetcher() -AsyncFetcher = AsyncFetcher() -StealthyFetcher = StealthyFetcher() -PlayWrightFetcher = PlayWrightFetcher() +Fetcher = _Fetcher() +AsyncFetcher = _AsyncFetcher() +StealthyFetcher = _StealthyFetcher() +PlayWrightFetcher = _PlayWrightFetcher() From 52b66c98ece792375fe75231e1b44ac924464601 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Tue, 28 Jan 2025 23:47:47 +0200 Subject: [PATCH 05/30] feat(StealthyFetcher): Disable Cross-Origin-Opener-Policy by default This allows elements in cross-origin iframes, such as the Turnstile checkbox, to be clicked. --- scrapling/engines/camo.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scrapling/engines/camo.py b/scrapling/engines/camo.py index 6d28634..1368b36 100644 --- a/scrapling/engines/camo.py +++ b/scrapling/engines/camo.py @@ -95,6 +95,7 @@ class CamoufoxEngine: with Camoufox( geoip=self.geoip, proxy=self.proxy, + disable_coop=True, addons=self.addons, exclude_addons=addons, headless=self.headless, From 675c0110e6c3135b041a33681ce785b2697b4eea Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Tue, 28 Jan 2025 23:51:31 +0200 Subject: [PATCH 06/30] feat(StealthyFetcher): Make ads enabled by default for performance boost --- README.md | 2 +- scrapling/engines/camo.py | 4 ++-- scrapling/fetchers.py | 8 ++++---- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 81702a5..e76d507 100644 --- a/README.md +++ b/README.md @@ -263,7 +263,7 @@ True | humanize | Humanize the cursor movement. Takes either True or the MAX duration in seconds of the cursor movement. The cursor typically takes up to 1.5 seconds to move across the window. | ✔️ | | allow_webgl | Enabled by default. Disabling it WebGL not recommended as many WAFs now checks if WebGL is enabled. | ✔️ | | geoip | Recommended to use with proxies; Automatically use IP's longitude, latitude, timezone, country, locale, & spoof the WebRTC IP address. It will also calculate and spoof the browser's language based on the distribution of language speakers in the target region. | ✔️ | -| disable_ads | Enabled by default, this installs `uBlock Origin` addon on the browser if enabled. | ✔️ | +| disable_ads | Disabled by default, this installs `uBlock Origin` addon on the browser if enabled. | ✔️ | | network_idle | Wait for the page until there are no network connections for at least 500 ms. | ✔️ | | timeout | The timeout in milliseconds that is used in all operations and waits through the page. The default is 30000. | ✔️ | | wait_selector | Wait for a specific css selector to be in a specific state. | ✔️ | diff --git a/scrapling/engines/camo.py b/scrapling/engines/camo.py index 1368b36..1a550c2 100644 --- a/scrapling/engines/camo.py +++ b/scrapling/engines/camo.py @@ -19,7 +19,7 @@ class CamoufoxEngine: block_webrtc: Optional[bool] = False, allow_webgl: Optional[bool] = True, network_idle: Optional[bool] = False, humanize: Optional[Union[bool, float]] = True, timeout: Optional[float] = 30000, page_action: Callable = None, wait_selector: Optional[str] = None, addons: Optional[List[str]] = None, wait_selector_state: Optional[SelectorWaitStates] = 'attached', google_search: Optional[bool] = True, extra_headers: Optional[Dict[str, str]] = None, - proxy: Optional[Union[str, Dict[str, str]]] = None, os_randomize: Optional[bool] = None, disable_ads: Optional[bool] = True, + proxy: Optional[Union[str, Dict[str, str]]] = None, os_randomize: Optional[bool] = None, disable_ads: Optional[bool] = False, geoip: Optional[bool] = False, adaptor_arguments: Dict = None, ): @@ -36,7 +36,7 @@ class CamoufoxEngine: :param humanize: Humanize the cursor movement. Takes either True or the MAX duration in seconds of the cursor movement. The cursor typically takes up to 1.5 seconds to move across the window. :param allow_webgl: Enabled by default. Disabling it WebGL not recommended as many WAFs now checks if WebGL is enabled. :param network_idle: Wait for the page until there are no network connections for at least 500 ms. - :param disable_ads: Enabled by default, this installs `uBlock Origin` addon on the browser if enabled. + :param disable_ads: Disabled by default, this installs `uBlock Origin` addon on the browser if enabled. :param os_randomize: If enabled, Scrapling will randomize the OS fingerprints used. The default is Scrapling matching the fingerprints with the current OS. :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30000 :param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again. diff --git a/scrapling/fetchers.py b/scrapling/fetchers.py index bdf87a6..ef7a1b6 100644 --- a/scrapling/fetchers.py +++ b/scrapling/fetchers.py @@ -177,7 +177,7 @@ class StealthyFetcher(BaseFetcher): block_webrtc: Optional[bool] = False, allow_webgl: Optional[bool] = True, network_idle: Optional[bool] = False, addons: Optional[List[str]] = None, timeout: Optional[float] = 30000, page_action: Callable = None, wait_selector: Optional[str] = None, humanize: Optional[Union[bool, float]] = True, wait_selector_state: SelectorWaitStates = 'attached', google_search: Optional[bool] = True, extra_headers: Optional[Dict[str, str]] = None, - proxy: Optional[Union[str, Dict[str, str]]] = None, os_randomize: Optional[bool] = None, disable_ads: Optional[bool] = True, geoip: Optional[bool] = False, + proxy: Optional[Union[str, Dict[str, str]]] = None, os_randomize: Optional[bool] = None, disable_ads: Optional[bool] = False, geoip: Optional[bool] = False, ) -> Response: """ Opens up a browser and do your request based on your chosen options below. @@ -191,7 +191,7 @@ class StealthyFetcher(BaseFetcher): This can help save your proxy usage but be careful with this option as it makes some websites never finish loading. :param block_webrtc: Blocks WebRTC entirely. :param addons: List of Firefox addons to use. Must be paths to extracted addons. - :param disable_ads: Enabled by default, this installs `uBlock Origin` addon on the browser if enabled. + :param disable_ads: Disabled by default, this installs `uBlock Origin` addon on the browser if enabled. :param humanize: Humanize the cursor movement. Takes either True or the MAX duration in seconds of the cursor movement. The cursor typically takes up to 1.5 seconds to move across the window. :param allow_webgl: Enabled by default. Disabling it WebGL not recommended as many WAFs now checks if WebGL is enabled. :param geoip: Recommended to use with proxies; Automatically use IP's longitude, latitude, timezone, country, locale, & spoof the WebRTC IP address. @@ -235,7 +235,7 @@ class StealthyFetcher(BaseFetcher): block_webrtc: Optional[bool] = False, allow_webgl: Optional[bool] = True, network_idle: Optional[bool] = False, addons: Optional[List[str]] = None, timeout: Optional[float] = 30000, page_action: Callable = None, wait_selector: Optional[str] = None, humanize: Optional[Union[bool, float]] = True, wait_selector_state: SelectorWaitStates = 'attached', google_search: Optional[bool] = True, extra_headers: Optional[Dict[str, str]] = None, - proxy: Optional[Union[str, Dict[str, str]]] = None, os_randomize: Optional[bool] = None, disable_ads: Optional[bool] = True, geoip: Optional[bool] = False, + proxy: Optional[Union[str, Dict[str, str]]] = None, os_randomize: Optional[bool] = None, disable_ads: Optional[bool] = False, geoip: Optional[bool] = False, ) -> Response: """ Opens up a browser and do your request based on your chosen options below. @@ -249,7 +249,7 @@ class StealthyFetcher(BaseFetcher): This can help save your proxy usage but be careful with this option as it makes some websites never finish loading. :param block_webrtc: Blocks WebRTC entirely. :param addons: List of Firefox addons to use. Must be paths to extracted addons. - :param disable_ads: Enabled by default, this installs `uBlock Origin` addon on the browser if enabled. + :param disable_ads: Disabled by default, this installs `uBlock Origin` addon on the browser if enabled. :param humanize: Humanize the cursor movement. Takes either True or the MAX duration in seconds of the cursor movement. The cursor typically takes up to 1.5 seconds to move across the window. :param allow_webgl: Enabled by default. Disabling it WebGL not recommended as many WAFs now checks if WebGL is enabled. :param geoip: Recommended to use with proxies; Automatically use IP's longitude, latitude, timezone, country, locale, & spoof the WebRTC IP address. From e9e606bca51e07b7d56bea6e86d225517d8e06fa Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Tue, 28 Jan 2025 23:53:12 +0200 Subject: [PATCH 07/30] feat(StealthyFetcher): Disable Cross-Origin-Opener-Policy by default for async_fetch This allows elements in cross-origin iframes, such as the Turnstile checkbox, to be clicked. . Forgot to add it with the other commit --- scrapling/engines/camo.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scrapling/engines/camo.py b/scrapling/engines/camo.py index 1a550c2..af3c61c 100644 --- a/scrapling/engines/camo.py +++ b/scrapling/engines/camo.py @@ -175,6 +175,7 @@ class CamoufoxEngine: async with AsyncCamoufox( geoip=self.geoip, proxy=self.proxy, + disable_coop=True, addons=self.addons, exclude_addons=addons, headless=self.headless, From 85f98708b2e5ab6200b9e8ae49f74ec44a6a546b Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Tue, 28 Jan 2025 23:55:24 +0200 Subject: [PATCH 08/30] feat(StealthyFetcher): Use memory cache by default for performance boost This is instead of the default disk-based cache. --- scrapling/engines/camo.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scrapling/engines/camo.py b/scrapling/engines/camo.py index af3c61c..e4ee4bb 100644 --- a/scrapling/engines/camo.py +++ b/scrapling/engines/camo.py @@ -96,6 +96,7 @@ class CamoufoxEngine: geoip=self.geoip, proxy=self.proxy, disable_coop=True, + enable_cache=True, addons=self.addons, exclude_addons=addons, headless=self.headless, @@ -176,6 +177,7 @@ class CamoufoxEngine: geoip=self.geoip, proxy=self.proxy, disable_coop=True, + enable_cache=True, addons=self.addons, exclude_addons=addons, headless=self.headless, From 5047e886a52db86b34f40b2f6ec87d10dcea633c Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Wed, 29 Jan 2025 00:58:14 +0200 Subject: [PATCH 09/30] style(autocompletion): Improving type hints for custom types This will provide a better autocompletion experience inside IDEs, commit affects: - TextHandler - AttributesHandler --- scrapling/core/_types.py | 3 +- scrapling/core/custom_types.py | 57 ++++++++++++++++++---------------- 2 files changed, 32 insertions(+), 28 deletions(-) diff --git a/scrapling/core/_types.py b/scrapling/core/_types.py index 84e9a51..a175077 100644 --- a/scrapling/core/_types.py +++ b/scrapling/core/_types.py @@ -3,7 +3,8 @@ Type definitions for type checking purposes. """ from typing import (TYPE_CHECKING, Any, Callable, Dict, Generator, Iterable, - List, Literal, Optional, Pattern, Tuple, Type, Union) + List, Literal, Optional, Pattern, Tuple, Type, TypeVar, + Union) SelectorWaitStates = Literal["attached", "detached", "hidden", "visible"] diff --git a/scrapling/core/custom_types.py b/scrapling/core/custom_types.py index 0419406..4a6a4c3 100644 --- a/scrapling/core/custom_types.py +++ b/scrapling/core/custom_types.py @@ -5,9 +5,13 @@ from types import MappingProxyType from orjson import dumps, loads from w3lib.html import replace_entities as _replace_entities -from scrapling.core._types import Dict, List, Pattern, SupportsIndex, Union +from scrapling.core._types import (Dict, Iterable, List, Optional, Pattern, + SupportsIndex, TypeVar, Union) from scrapling.core.utils import _is_iterable, flatten +# Define type variable for AttributeHandler value type +VT = TypeVar('VT', bound='TextHandler') + class TextHandler(str): """Extends standard Python string by adding more functionality""" @@ -19,71 +23,70 @@ class TextHandler(str): return super().__new__(cls, '') # Make methods from original `str` class return `TextHandler` instead of returning `str` again - # Of course, this stupid workaround is only so we can keep the auto-completion working without issues in your IDE - # and I made sonnet write it for me :) - def strip(self, chars=None): + # Of course, I made sonnet write it for me :) + def strip(self, chars: str = None) -> Union[str, 'TextHandler']: return TextHandler(super().strip(chars)) - def lstrip(self, chars=None): + def lstrip(self, chars: str = None) -> Union[str, 'TextHandler']: return TextHandler(super().lstrip(chars)) - def rstrip(self, chars=None): + def rstrip(self, chars: str = None) -> Union[str, 'TextHandler']: return TextHandler(super().rstrip(chars)) - def capitalize(self): + def capitalize(self) -> Union[str, 'TextHandler']: return TextHandler(super().capitalize()) - def casefold(self): + def casefold(self) -> Union[str, 'TextHandler']: return TextHandler(super().casefold()) - def center(self, width, fillchar=' '): + def center(self, width: SupportsIndex, fillchar: str = ' ') -> Union[str, 'TextHandler']: return TextHandler(super().center(width, fillchar)) - def expandtabs(self, tabsize=8): + def expandtabs(self, tabsize: SupportsIndex = 8) -> Union[str, 'TextHandler']: return TextHandler(super().expandtabs(tabsize)) - def format(self, *args, **kwargs): + def format(self, *args: str, **kwargs: str) -> Union[str, 'TextHandler']: return TextHandler(super().format(*args, **kwargs)) - def format_map(self, mapping): + def format_map(self, mapping) -> Union[str, 'TextHandler']: return TextHandler(super().format_map(mapping)) - def join(self, iterable): + def join(self, iterable: Iterable[str]) -> Union[str, 'TextHandler']: return TextHandler(super().join(iterable)) - def ljust(self, width, fillchar=' '): + def ljust(self, width: SupportsIndex, fillchar: str = ' ') -> Union[str, 'TextHandler']: return TextHandler(super().ljust(width, fillchar)) - def rjust(self, width, fillchar=' '): + def rjust(self, width: SupportsIndex, fillchar: str = ' ') -> Union[str, 'TextHandler']: return TextHandler(super().rjust(width, fillchar)) - def swapcase(self): + def swapcase(self) -> Union[str, 'TextHandler']: return TextHandler(super().swapcase()) - def title(self): + def title(self) -> Union[str, 'TextHandler']: return TextHandler(super().title()) - def translate(self, table): + def translate(self, table) -> Union[str, 'TextHandler']: return TextHandler(super().translate(table)) - def zfill(self, width): + def zfill(self, width: SupportsIndex) -> Union[str, 'TextHandler']: return TextHandler(super().zfill(width)) - def replace(self, old, new, count=-1): + def replace(self, old: str, new: str, count: SupportsIndex = -1) -> Union[str, 'TextHandler']: return TextHandler(super().replace(old, new, count)) - def upper(self): + def upper(self) -> Union[str, 'TextHandler']: return TextHandler(super().upper()) - def lower(self): + def lower(self) -> Union[str, 'TextHandler']: return TextHandler(super().lower()) ############## - def sort(self, reverse: bool = False) -> str: + def sort(self, reverse: bool = False) -> Union[str, 'TextHandler']: """Return a sorted version of the string""" return self.__class__("".join(sorted(self, reverse=reverse))) - def clean(self) -> str: + def clean(self) -> Union[str, 'TextHandler']: """Return a new version of the string after removing all white spaces and consecutive spaces""" data = re.sub(r'[\t|\r|\n]', '', self) data = re.sub(' +', ' ', data) @@ -210,7 +213,7 @@ class TextHandlers(List[TextHandler]): get_all = extract -class AttributesHandler(Mapping): +class AttributesHandler(Mapping[str, VT]): """A read-only mapping to use instead of the standard dictionary for the speed boost but at the same time I use it to add more functionalities. If standard dictionary is needed, just convert this class to dictionary with `dict` function """ @@ -231,7 +234,7 @@ class AttributesHandler(Mapping): # Fastest read-only mapping type self._data = MappingProxyType(mapping) - def get(self, key, default=None): + def get(self, key: str, default: Optional[str] = None) -> Union[VT, None]: """Acts like standard dictionary `.get()` method""" return self._data.get(key, default) @@ -253,7 +256,7 @@ class AttributesHandler(Mapping): """Convert current attributes to JSON string if the attributes are JSON serializable otherwise throws error""" return dumps(dict(self._data)) - def __getitem__(self, key): + def __getitem__(self, key: str) -> VT: return self._data[key] def __iter__(self): From 70ffdd09e2f04b9c8853b3bf493f7f806b53f1d4 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Wed, 29 Jan 2025 01:22:07 +0200 Subject: [PATCH 10/30] feat(TextHandler): Make split string method return Texthandlers --- scrapling/core/custom_types.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/scrapling/core/custom_types.py b/scrapling/core/custom_types.py index 4a6a4c3..0ac6ed8 100644 --- a/scrapling/core/custom_types.py +++ b/scrapling/core/custom_types.py @@ -1,4 +1,5 @@ import re +import typing from collections.abc import Mapping from types import MappingProxyType @@ -10,7 +11,7 @@ from scrapling.core._types import (Dict, Iterable, List, Optional, Pattern, from scrapling.core.utils import _is_iterable, flatten # Define type variable for AttributeHandler value type -VT = TypeVar('VT', bound='TextHandler') +_TextHandlerType = TypeVar('_TextHandlerType', bound='TextHandler') class TextHandler(str): @@ -24,6 +25,11 @@ class TextHandler(str): # Make methods from original `str` class return `TextHandler` instead of returning `str` again # Of course, I made sonnet write it for me :) + def split(self, sep: str = None, maxsplit: SupportsIndex = -1) -> 'TextHandlers[_TextHandlerType]': + return TextHandlers([ + typing.cast("_TextHandlerType", s) for s in super().split(sep, maxsplit) + ]) + def strip(self, chars: str = None) -> Union[str, 'TextHandler']: return TextHandler(super().strip(chars)) @@ -155,7 +161,7 @@ class TextHandler(str): return result[0] if result else default -class TextHandlers(List[TextHandler]): +class TextHandlers(List[_TextHandlerType]): """ The :class:`TextHandlers` class is a subclass of the builtin ``List`` class, which provides a few additional methods. """ @@ -213,7 +219,7 @@ class TextHandlers(List[TextHandler]): get_all = extract -class AttributesHandler(Mapping[str, VT]): +class AttributesHandler(Mapping[str, _TextHandlerType]): """A read-only mapping to use instead of the standard dictionary for the speed boost but at the same time I use it to add more functionalities. If standard dictionary is needed, just convert this class to dictionary with `dict` function """ @@ -234,7 +240,7 @@ class AttributesHandler(Mapping[str, VT]): # Fastest read-only mapping type self._data = MappingProxyType(mapping) - def get(self, key: str, default: Optional[str] = None) -> Union[VT, None]: + def get(self, key: str, default: Optional[str] = None) -> Union[_TextHandlerType, None]: """Acts like standard dictionary `.get()` method""" return self._data.get(key, default) @@ -256,7 +262,7 @@ class AttributesHandler(Mapping[str, VT]): """Convert current attributes to JSON string if the attributes are JSON serializable otherwise throws error""" return dumps(dict(self._data)) - def __getitem__(self, key: str) -> VT: + def __getitem__(self, key: str) -> _TextHandlerType: return self._data[key] def __iter__(self): From 1d5fcc060d69cdc0ede7afeda5ca0c1c9dc8f3cd Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Wed, 29 Jan 2025 14:22:47 +0200 Subject: [PATCH 11/30] feat(TextHandler): Make slicing return `TextHandlers` + autocompletion fixes and access by index return `TextHandler` --- scrapling/core/custom_types.py | 43 +++++++++++++++++++++++++--------- 1 file changed, 32 insertions(+), 11 deletions(-) diff --git a/scrapling/core/custom_types.py b/scrapling/core/custom_types.py index 0ac6ed8..136dbb4 100644 --- a/scrapling/core/custom_types.py +++ b/scrapling/core/custom_types.py @@ -23,12 +23,25 @@ class TextHandler(str): return super().__new__(cls, string) return super().__new__(cls, '') - # Make methods from original `str` class return `TextHandler` instead of returning `str` again - # Of course, I made sonnet write it for me :) - def split(self, sep: str = None, maxsplit: SupportsIndex = -1) -> 'TextHandlers[_TextHandlerType]': - return TextHandlers([ - typing.cast("_TextHandlerType", s) for s in super().split(sep, maxsplit) - ]) + @typing.overload + def __getitem__(self, key: SupportsIndex) -> 'TextHandler': + pass + + @typing.overload + def __getitem__(self, key: slice) -> "TextHandlers": + pass + + def __getitem__(self, key: Union[SupportsIndex, slice]) -> Union["TextHandler", "TextHandlers"]: + lst = super().__getitem__(key) + if isinstance(key, slice): + lst = [TextHandler(s) for s in lst] + return TextHandlers(typing.cast(List[_TextHandlerType], lst)) + return typing.cast(_TextHandlerType, TextHandler(lst)) + + def split(self, sep: str = None, maxsplit: SupportsIndex = -1) -> 'TextHandlers': + return TextHandlers( + typing.cast(List[_TextHandlerType], [TextHandler(s) for s in super().split(sep, maxsplit)]) + ) def strip(self, chars: str = None) -> Union[str, 'TextHandler']: return TextHandler(super().strip(chars)) @@ -161,18 +174,26 @@ class TextHandler(str): return result[0] if result else default -class TextHandlers(List[_TextHandlerType]): +class TextHandlers(List[TextHandler]): """ The :class:`TextHandlers` class is a subclass of the builtin ``List`` class, which provides a few additional methods. """ __slots__ = () - def __getitem__(self, pos: Union[SupportsIndex, slice]) -> Union[TextHandler, "TextHandlers[TextHandler]"]: + @typing.overload + def __getitem__(self, pos: SupportsIndex) -> TextHandler: + pass + + @typing.overload + def __getitem__(self, pos: slice) -> "TextHandlers": + pass + + def __getitem__(self, pos: Union[SupportsIndex, slice]) -> Union[TextHandler, "TextHandlers"]: lst = super().__getitem__(pos) if isinstance(pos, slice): - return self.__class__(lst) - else: - return lst + lst = [TextHandler(s) for s in lst] + return TextHandlers(typing.cast(List[_TextHandlerType], lst)) + return typing.cast(_TextHandlerType, TextHandler(lst)) def re(self, regex: Union[str, Pattern[str]], replace_entities: bool = True, clean_match: bool = False, case_sensitive: bool = False) -> 'List[str]': From 7ee0f6114d8b586bdc1b2826ed7f0c492080ed0b Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Thu, 30 Jan 2025 01:46:33 +0200 Subject: [PATCH 12/30] fix(parser): Code restructure for speed boost & better type hints - Now return types are consistent across all the parser engine - Parser got a 5-30% performance boost across different methods. - Renamed some of the internal methods for clearer code. - A lot better auto-completion experience after a lot of adjustments. --- scrapling/parser.py | 168 +++++++++++++++++++++++--------------------- 1 file changed, 88 insertions(+), 80 deletions(-) diff --git a/scrapling/parser.py b/scrapling/parser.py index 1a46f1a..27a822b 100644 --- a/scrapling/parser.py +++ b/scrapling/parser.py @@ -1,6 +1,7 @@ import inspect import os import re +import typing from difflib import SequenceMatcher from urllib.parse import urljoin @@ -145,47 +146,46 @@ class Adaptor(SelectorsGeneration): # Faster than checking `element.is_attribute or element.is_text or element.is_tail` return issubclass(type(element), etree._ElementUnicodeResult) - def __get_correct_result( - self, element: Union[html.HtmlElement, etree._ElementUnicodeResult] - ) -> Union[TextHandler, html.HtmlElement, 'Adaptor', str]: - """Used internally in all functions to convert results to type (Adaptor|Adaptors) when possible""" - if self._is_text_node(element): - # etree._ElementUnicodeResult basically inherit from `str` so it's fine - return TextHandler(str(element)) - else: - if issubclass(type(element), html.HtmlMixin): + @staticmethod + def __content_convertor(element: Union[html.HtmlElement, etree._ElementUnicodeResult]) -> TextHandler: + """Used internally to convert a single element's text content to TextHandler directly without checks - return Adaptor( - root=element, - text='', body=b'', # Since root argument is provided, both `text` and `body` will be ignored so this is just a filler - url=self.url, encoding=self.encoding, auto_match=self.__auto_match_enabled, - keep_comments=self.__keep_comments, keep_cdata=self.__keep_cdata, - huge_tree=self.__huge_tree_enabled, - **self.__response_data - ) - return element + This single line has been isolated like this so when it's used with map we get that slight performance boost vs list comprehension + """ + return TextHandler(str(element)) - def __convert_results( - self, result: Union[List[html.HtmlElement], html.HtmlElement] - ) -> Union['Adaptors[Adaptor]', 'Adaptor', List, None]: - """Used internally in all functions to convert results to type (Adaptor|Adaptors) in bulk when possible""" - if result is None: + def __element_convertor(self, element: html.HtmlElement) -> 'Adaptor': + """Used internally to convert a single HtmlElement to Adaptor directly without checks""" + return Adaptor( + root=element, + text='', body=b'', # Since root argument is provided, both `text` and `body` will be ignored so this is just a filler + url=self.url, encoding=self.encoding, auto_match=self.__auto_match_enabled, + keep_comments=self.__keep_comments, keep_cdata=self.__keep_cdata, + huge_tree=self.__huge_tree_enabled, + **self.__response_data + ) + + def __handle_element(self, element: Union[html.HtmlElement, etree._ElementUnicodeResult]) -> Union[TextHandler, 'Adaptor', None]: + """Used internally in all functions to convert a single element to type (Adaptor|TextHandler) when possible""" + if element is None: return None - elif result == []: # Lxml will give a warning if I used something like `not result` - return [] + elif self._is_text_node(element): + # etree._ElementUnicodeResult basically inherit from `str` so it's fine + return self.__content_convertor(element) + else: + return self.__element_convertor(element) - if isinstance(result, Adaptors): - return result + def __handle_elements(self, result: List[Union[html.HtmlElement, etree._ElementUnicodeResult]]) -> Union['Adaptors', 'TextHandlers', List]: + """Used internally in all functions to convert results to type (Adaptors|TextHandlers) in bulk when possible""" + if not len(result): # Lxml will give a warning if I used something like `not result` + return Adaptors([]) - if type(result) is list: - results = [self.__get_correct_result(n) for n in result] - if all(isinstance(res, self.__class__) for res in results): - return Adaptors(results) - elif all(isinstance(res, TextHandler) for res in results): - return TextHandlers(results) - return results + # From within the code, this method will always get a list of the same type + # so we will continue without checks for slight performance boost + if self._is_text_node(result[0]): + return TextHandlers(list(map(self.__content_convertor, result))) - return self.__get_correct_result(result) + return Adaptors(list(map(self.__element_convertor, result))) def __getstate__(self) -> Any: # lxml don't like it :) @@ -282,14 +282,14 @@ class Adaptor(SelectorsGeneration): @property def parent(self) -> Union['Adaptor', None]: """Return the direct parent of the element or ``None`` otherwise""" - return self.__convert_results(self._root.getparent()) + return self.__handle_element(self._root.getparent()) @property def children(self) -> Union['Adaptors[Adaptor]', List]: """Return the children elements of the current element or empty list otherwise""" - return self.__convert_results(list( - child for child in self._root.iterchildren() if type(child) not in html_forbidden - )) + return Adaptors([ + self.__element_convertor(child) for child in self._root.iterchildren() if type(child) not in html_forbidden + ]) @property def siblings(self) -> Union['Adaptors[Adaptor]', List]: @@ -301,7 +301,7 @@ class Adaptor(SelectorsGeneration): def iterancestors(self) -> Generator['Adaptor', None, None]: """Return a generator that loops over all ancestors of the element, starting with element's parent.""" for ancestor in self._root.iterancestors(): - yield self.__convert_results(ancestor) + yield self.__element_convertor(ancestor) def find_ancestor(self, func: Callable[['Adaptor'], bool]) -> Union['Adaptor', None]: """Loop over all ancestors of the element till one match the passed function @@ -328,7 +328,7 @@ class Adaptor(SelectorsGeneration): # Ignore html comments and unwanted types next_element = next_element.getnext() - return self.__convert_results(next_element) + return self.__handle_element(next_element) @property def previous(self) -> Union['Adaptor', None]: @@ -339,7 +339,7 @@ class Adaptor(SelectorsGeneration): # Ignore html comments and unwanted types prev_element = prev_element.getprevious() - return self.__convert_results(prev_element) + return self.__handle_element(prev_element) # For easy copy-paste from Scrapy/parsel code when needed :) def get(self, default=None): @@ -413,13 +413,16 @@ class Adaptor(SelectorsGeneration): if score_table: highest_probability = max(score_table.keys()) if score_table[highest_probability] and highest_probability >= percentage: - log.debug(f'Highest probability was {highest_probability}%') - log.debug('Top 5 best matching elements are: ') - for percent in tuple(sorted(score_table.keys(), reverse=True))[:5]: - log.debug(f'{percent} -> {self.__convert_results(score_table[percent])}') + if log.getEffectiveLevel() < 20: + # No need to execute this part if logging level is not debugging + log.debug(f'Highest probability was {highest_probability}%') + log.debug('Top 5 best matching elements are: ') + for percent in tuple(sorted(score_table.keys(), reverse=True))[:5]: + log.debug(f'{percent} -> {self.__handle_elements(score_table[percent])}') + if not adaptor_type: return score_table[highest_probability] - return self.__convert_results(score_table[highest_probability]) + return self.__handle_elements(score_table[highest_probability]) return [] def css_first(self, selector: str, identifier: str = '', @@ -493,7 +496,7 @@ class Adaptor(SelectorsGeneration): :return: List as :class:`Adaptors` """ try: - if not self.__auto_match_enabled: + if not self.__auto_match_enabled or ',' not in selector: # No need to split selectors in this case, let's save some CPU cycles :) xpath_selector = HTMLTranslator().css_to_xpath(selector) return self.xpath(xpath_selector, identifier or selector, auto_match, auto_save, percentage) @@ -507,11 +510,8 @@ class Adaptor(SelectorsGeneration): results += self.xpath( xpath_selector, identifier or single_selector.canonical(), auto_match, auto_save, percentage ) - else: - xpath_selector = HTMLTranslator().css_to_xpath(selector) - return self.xpath(xpath_selector, identifier or selector, auto_match, auto_save, percentage) - return self.__convert_results(results) + return results except (SelectorError, SelectorSyntaxError,): raise SelectorSyntaxError(f"Invalid CSS selector: {selector}") @@ -538,37 +538,37 @@ class Adaptor(SelectorsGeneration): :return: List as :class:`Adaptors` """ try: - selected_elements = self._root.xpath(selector, **kwargs) + elements = self._root.xpath(selector, **kwargs) - if selected_elements: - if not self.__auto_match_enabled and auto_save: - log.warning("Argument `auto_save` will be ignored because `auto_match` wasn't enabled on initialization. Check docs for more info.") + if elements: + if auto_save: + if not self.__auto_match_enabled: + log.warning("Argument `auto_save` will be ignored because `auto_match` wasn't enabled on initialization. Check docs for more info.") + else: + self.save(elements[0], identifier or selector) - elif self.__auto_match_enabled and auto_save: - self.save(selected_elements[0], identifier or selector) - - return self.__convert_results(selected_elements) - else: - if self.__auto_match_enabled and auto_match: + return self.__handle_elements(elements) + elif self.__auto_match_enabled: + if auto_match: element_data = self.retrieve(identifier or selector) if element_data: - relocated = self.relocate(element_data, percentage) - if relocated is not None and auto_save: - self.save(relocated[0], identifier or selector) + elements = self.relocate(element_data, percentage) + if elements is not None and auto_save: + self.save(elements[0], identifier or selector) - return self.__convert_results(relocated) - else: - return self.__convert_results(selected_elements) - - elif not self.__auto_match_enabled and auto_match: + return self.__handle_elements(elements) + else: + if auto_match: log.warning("Argument `auto_match` will be ignored because `auto_match` wasn't enabled on initialization. Check docs for more info.") + elif auto_save: + log.warning("Argument `auto_save` will be ignored because `auto_match` wasn't enabled on initialization. Check docs for more info.") - return self.__convert_results(selected_elements) + return self.__handle_elements(elements) except (SelectorError, SelectorSyntaxError, etree.XPathError, etree.XPathEvalError): raise SelectorSyntaxError(f"Invalid XPath selector: {selector}") - def find_all(self, *args: Union[str, Iterable[str], Pattern, Callable, Dict[str, str]], **kwargs: str) -> Union['Adaptors[Adaptor]', List]: + def find_all(self, *args: Union[str, Iterable[str], Pattern, Callable, Dict[str, str]], **kwargs: str) -> 'Adaptors': """Find elements by filters of your creations for ease.. :param args: Tag name(s), an iterable of tag names, regex patterns, function, or a dictionary of elements' attributes. Leave empty for selecting all. @@ -588,7 +588,7 @@ class Adaptor(SelectorsGeneration): attributes = dict() tags, patterns = set(), set() - results, functions, selectors = [], [], [] + results, functions, selectors = Adaptors([]), [], [] def _search_tree(element: Adaptor, filter_function: Callable) -> None: """Collect element if it fulfills passed function otherwise, traverse the children tree and iterate""" @@ -662,7 +662,7 @@ class Adaptor(SelectorsGeneration): for function in functions: _search_tree(result, function) - return self.__convert_results(results) + return results def find(self, *args: Union[str, Iterable[str], Pattern, Callable, Dict[str, str]], **kwargs: str) -> Union['Adaptor', None]: """Find elements by filters of your creations for ease then return the first result. Otherwise return `None`. @@ -894,7 +894,7 @@ class Adaptor(SelectorsGeneration): if potential_match != root and are_alike(root, target_attrs, potential_match): similar_elements.append(potential_match) - return self.__convert_results(similar_elements) + return self.__handle_elements(similar_elements) def find_by_text( self, text: str, first_match: bool = True, partial: bool = False, @@ -908,7 +908,7 @@ class Adaptor(SelectorsGeneration): :param clean_match: if enabled, this will ignore all whitespaces and consecutive spaces while matching """ - results = [] + results = Adaptors([]) if not case_sensitive: text = text.lower() @@ -942,7 +942,7 @@ class Adaptor(SelectorsGeneration): if first_match: if results: return results[0] - return self.__convert_results(results) + return results def find_by_regex( self, query: Union[str, Pattern[str]], first_match: bool = True, case_sensitive: bool = False, clean_match: bool = True @@ -953,7 +953,7 @@ class Adaptor(SelectorsGeneration): :param case_sensitive: if enabled, letters case will be taken into consideration in the regex :param clean_match: if enabled, this will ignore all whitespaces and consecutive spaces while matching """ - results = [] + results = Adaptors([]) def _traverse(node: Adaptor) -> None: """Check if element matches given regex otherwise, traverse the children tree and iterate""" @@ -975,7 +975,7 @@ class Adaptor(SelectorsGeneration): if results and first_match: return results[0] - return self.__convert_results(results) + return results class Adaptors(List[Adaptor]): @@ -984,7 +984,15 @@ class Adaptors(List[Adaptor]): """ __slots__ = () - def __getitem__(self, pos: Union[SupportsIndex, slice]) -> Union[Adaptor, "Adaptors[Adaptor]"]: + @typing.overload + def __getitem__(self, pos: SupportsIndex) -> Adaptor: + pass + + @typing.overload + def __getitem__(self, pos: slice) -> "Adaptors": + pass + + def __getitem__(self, pos: Union[SupportsIndex, slice]) -> Union[Adaptor, "Adaptors"]: lst = super().__getitem__(pos) if isinstance(pos, slice): return self.__class__(lst) From 4e2e7a4b22ec53a0acd24ab5c445f3d4e5c15975 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Thu, 30 Jan 2025 02:03:09 +0200 Subject: [PATCH 13/30] fix(parser): More on correcting type hints and unifying return type --- scrapling/parser.py | 25 ++++++++++--------------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/scrapling/parser.py b/scrapling/parser.py index 27a822b..1afce3a 100644 --- a/scrapling/parser.py +++ b/scrapling/parser.py @@ -285,18 +285,18 @@ class Adaptor(SelectorsGeneration): return self.__handle_element(self._root.getparent()) @property - def children(self) -> Union['Adaptors[Adaptor]', List]: + def children(self) -> 'Adaptors[Adaptor]': """Return the children elements of the current element or empty list otherwise""" return Adaptors([ self.__element_convertor(child) for child in self._root.iterchildren() if type(child) not in html_forbidden ]) @property - def siblings(self) -> Union['Adaptors[Adaptor]', List]: + def siblings(self) -> 'Adaptors[Adaptor]': """Return other children of the current element's parent or empty list otherwise""" if self.parent: return Adaptors([child for child in self.parent.children if child._root != self._root]) - return [] + return Adaptors([]) def iterancestors(self) -> Generator['Adaptor', None, None]: """Return a generator that loops over all ancestors of the element, starting with element's parent.""" @@ -442,8 +442,6 @@ class Adaptor(SelectorsGeneration): :param percentage: The minimum percentage to accept while auto-matching and not going lower than that. Be aware that the percentage calculation depends solely on the page structure so don't play with this number unless you must know what you are doing! - - :return: List as :class:`Adaptors` """ for element in self.css(selector, identifier, auto_match, auto_save, percentage): return element @@ -468,8 +466,6 @@ class Adaptor(SelectorsGeneration): :param percentage: The minimum percentage to accept while auto-matching and not going lower than that. Be aware that the percentage calculation depends solely on the page structure so don't play with this number unless you must know what you are doing! - - :return: List as :class:`Adaptors` """ for element in self.xpath(selector, identifier, auto_match, auto_save, percentage, **kwargs): return element @@ -899,7 +895,7 @@ class Adaptor(SelectorsGeneration): def find_by_text( self, text: str, first_match: bool = True, partial: bool = False, case_sensitive: bool = False, clean_match: bool = True - ) -> Union['Adaptors[Adaptor]', 'Adaptor', List]: + ) -> Union['Adaptors[Adaptor]', 'Adaptor']: """Find elements that its text content fully/partially matches input. :param text: Text query to match :param first_match: Return first element that matches conditions, enabled by default @@ -946,7 +942,7 @@ class Adaptor(SelectorsGeneration): def find_by_regex( self, query: Union[str, Pattern[str]], first_match: bool = True, case_sensitive: bool = False, clean_match: bool = True - ) -> Union['Adaptors[Adaptor]', 'Adaptor', List]: + ) -> Union['Adaptors[Adaptor]', 'Adaptor']: """Find elements that its text content matches the input regex pattern. :param query: Regex query/pattern to match :param first_match: Return first element that matches conditions, enabled by default @@ -1001,7 +997,7 @@ class Adaptors(List[Adaptor]): def xpath( self, selector: str, identifier: str = '', auto_save: bool = False, percentage: int = 0, **kwargs: Any - ) -> Union["Adaptors[Adaptor]", List]: + ) -> "Adaptors[Adaptor]": """ Call the ``.xpath()`` method for each element in this list and return their results as another :class:`Adaptors`. @@ -1027,7 +1023,7 @@ class Adaptors(List[Adaptor]): ] return self.__class__(flatten(results)) - def css(self, selector: str, identifier: str = '', auto_save: bool = False, percentage: int = 0) -> Union["Adaptors[Adaptor]", List]: + def css(self, selector: str, identifier: str = '', auto_save: bool = False, percentage: int = 0) -> "Adaptors[Adaptor]": """ Call the ``.css()`` method for each element in this list and return their results flattened as another :class:`Adaptors`. @@ -1092,15 +1088,14 @@ class Adaptors(List[Adaptor]): return element return None - def filter(self, func: Callable[['Adaptor'], bool]) -> Union['Adaptors', List]: + def filter(self, func: Callable[['Adaptor'], bool]) -> 'Adaptors[Adaptor]': """Filter current elements based on the passed function :param func: A function that takes each element as an argument and returns True/False :return: The new `Adaptors` object or empty list otherwise. """ - results = [ + return self.__class__([ element for element in self if func(element) - ] - return self.__class__(results) if results else results + ]) # For easy copy-paste from Scrapy/parsel code when needed :) def get(self, default=None): From 93be41657257bdbbdd08f791c67550933ba3a67f Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Thu, 30 Jan 2025 02:15:09 +0200 Subject: [PATCH 14/30] feat(TextHandler): Now regex methods return TextHandler/TextHandlers --- scrapling/core/custom_types.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/scrapling/core/custom_types.py b/scrapling/core/custom_types.py index 136dbb4..d1ad2c1 100644 --- a/scrapling/core/custom_types.py +++ b/scrapling/core/custom_types.py @@ -130,7 +130,7 @@ class TextHandler(str): def re( self, regex: Union[str, Pattern[str]], replace_entities: bool = True, clean_match: bool = False, case_sensitive: bool = False, check_match: bool = False - ) -> Union[List[str], bool]: + ) -> Union["TextHandlers[TextHandler]", bool]: """Apply the given regex to the current text and return a list of strings with the matches. :param regex: Can be either a compiled regular expression or a string. @@ -155,12 +155,12 @@ class TextHandler(str): results = flatten(results) if not replace_entities: - return [TextHandler(string) for string in results] + return TextHandlers(typing.cast(List[_TextHandlerType], [TextHandler(string) for string in results])) - return [TextHandler(_replace_entities(s)) for s in results] + return TextHandlers(typing.cast(List[_TextHandlerType], [TextHandler(_replace_entities(s)) for s in results])) def re_first(self, regex: Union[str, Pattern[str]], default=None, replace_entities: bool = True, - clean_match: bool = False, case_sensitive: bool = False) -> Union[str, None]: + clean_match: bool = False, case_sensitive: bool = False) -> "TextHandler": """Apply the given regex to text and return the first match if found, otherwise return the default value. :param regex: Can be either a compiled regular expression or a string. @@ -196,7 +196,7 @@ class TextHandlers(List[TextHandler]): return typing.cast(_TextHandlerType, TextHandler(lst)) def re(self, regex: Union[str, Pattern[str]], replace_entities: bool = True, clean_match: bool = False, - case_sensitive: bool = False) -> 'List[str]': + case_sensitive: bool = False) -> 'TextHandlers[TextHandler]': """Call the ``.re()`` method for each element in this list and return their results flattened as TextHandlers. @@ -208,10 +208,10 @@ class TextHandlers(List[TextHandler]): results = [ n.re(regex, replace_entities, clean_match, case_sensitive) for n in self ] - return flatten(results) + return TextHandlers(flatten(results)) def re_first(self, regex: Union[str, Pattern[str]], default=None, replace_entities: bool = True, - clean_match: bool = False, case_sensitive: bool = False) -> Union[str, None]: + clean_match: bool = False, case_sensitive: bool = False) -> TextHandler: """Call the ``.re_first()`` method for each element in this list and return the first result or the default value otherwise. From e507778c48d6a9fe728c1f5361e2069739867cf0 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Thu, 30 Jan 2025 02:27:38 +0200 Subject: [PATCH 15/30] feat(Parser): Set regex methods new return types as TextHandlers --- scrapling/parser.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scrapling/parser.py b/scrapling/parser.py index 1afce3a..5743306 100644 --- a/scrapling/parser.py +++ b/scrapling/parser.py @@ -788,7 +788,7 @@ class Adaptor(SelectorsGeneration): return self.get_all_text(strip=True).json() def re(self, regex: Union[str, Pattern[str]], replace_entities: bool = True, - clean_match: bool = False, case_sensitive: bool = False) -> 'List[str]': + clean_match: bool = False, case_sensitive: bool = False) -> TextHandlers: """Apply the given regex to the current text and return a list of strings with the matches. :param regex: Can be either a compiled regular expression or a string. @@ -799,7 +799,7 @@ class Adaptor(SelectorsGeneration): return self.text.re(regex, replace_entities, clean_match, case_sensitive) def re_first(self, regex: Union[str, Pattern[str]], default=None, replace_entities: bool = True, - clean_match: bool = False, case_sensitive: bool = False) -> Union[str, None]: + clean_match: bool = False, case_sensitive: bool = False) -> TextHandler: """Apply the given regex to text and return the first match if found, otherwise return the default value. :param regex: Can be either a compiled regular expression or a string. @@ -1048,7 +1048,7 @@ class Adaptors(List[Adaptor]): return self.__class__(flatten(results)) def re(self, regex: Union[str, Pattern[str]], replace_entities: bool = True, - clean_match: bool = False, case_sensitive: bool = False) -> 'List[str]': + clean_match: bool = False, case_sensitive: bool = False) -> TextHandlers[TextHandler]: """Call the ``.re()`` method for each element in this list and return their results flattened as List of TextHandler. @@ -1060,10 +1060,10 @@ class Adaptors(List[Adaptor]): results = [ n.text.re(regex, replace_entities, clean_match, case_sensitive) for n in self ] - return flatten(results) + return TextHandlers(flatten(results)) def re_first(self, regex: Union[str, Pattern[str]], default=None, replace_entities: bool = True, - clean_match: bool = False, case_sensitive: bool = False) -> Union[str, None]: + clean_match: bool = False, case_sensitive: bool = False) -> TextHandler: """Call the ``.re_first()`` method for each element in this list and return the first result or the default value otherwise. From 7882cf25279d485582fa1aa2e9de4d71d6aadf6e Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Thu, 30 Jan 2025 02:44:57 +0200 Subject: [PATCH 16/30] style(TextHandler): Setting overload on check_match argument for auto completion --- scrapling/core/custom_types.py | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/scrapling/core/custom_types.py b/scrapling/core/custom_types.py index d1ad2c1..03257c3 100644 --- a/scrapling/core/custom_types.py +++ b/scrapling/core/custom_types.py @@ -6,8 +6,8 @@ from types import MappingProxyType from orjson import dumps, loads from w3lib.html import replace_entities as _replace_entities -from scrapling.core._types import (Dict, Iterable, List, Optional, Pattern, - SupportsIndex, TypeVar, Union) +from scrapling.core._types import (Dict, Iterable, List, Literal, Optional, + Pattern, SupportsIndex, TypeVar, Union) from scrapling.core.utils import _is_iterable, flatten # Define type variable for AttributeHandler value type @@ -127,6 +127,28 @@ class TextHandler(str): # Check this out: https://github.com/ijl/orjson/issues/445 return loads(str(self)) + @typing.overload + def re( + self, + regex: Union[str, Pattern[str]], + replace_entities: bool = True, + clean_match: bool = False, + case_sensitive: bool = False, + check_match: Literal[True] = True, + ) -> bool: + ... + + @typing.overload + def re( + self, + regex: Union[str, Pattern[str]], + replace_entities: bool = True, + clean_match: bool = False, + case_sensitive: bool = False, + check_match: Literal[False] = False, + ) -> "TextHandlers[TextHandler]": + ... + def re( self, regex: Union[str, Pattern[str]], replace_entities: bool = True, clean_match: bool = False, case_sensitive: bool = False, check_match: bool = False From c0d912d16ab12ab35cf01045a712f050d1b2dd49 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Thu, 30 Jan 2025 19:53:13 +0200 Subject: [PATCH 17/30] fix(asyncFetcher): A typo made async version of put do post request instead Awkward moment --- scrapling/fetchers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapling/fetchers.py b/scrapling/fetchers.py index ef7a1b6..eb8d1ad 100644 --- a/scrapling/fetchers.py +++ b/scrapling/fetchers.py @@ -143,7 +143,7 @@ class AsyncFetcher(Fetcher): :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers` """ adaptor_arguments = tuple(self.adaptor_arguments.items()) - response_object = await StaticEngine(url, proxy, stealthy_headers, follow_redirects, timeout, retries=retries, adaptor_arguments=adaptor_arguments).async_post(**kwargs) + response_object = await StaticEngine(url, proxy, stealthy_headers, follow_redirects, timeout, retries=retries, adaptor_arguments=adaptor_arguments).async_put(**kwargs) return response_object async def delete( From 148103f3c82c5bb218dd39021081ae5aad7f76c5 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Thu, 30 Jan 2025 20:01:25 +0200 Subject: [PATCH 18/30] fix(translator): increase CSS to XPath convertor cache limit --- scrapling/core/translator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapling/core/translator.py b/scrapling/core/translator.py index 263a24a..cb7f064 100644 --- a/scrapling/core/translator.py +++ b/scrapling/core/translator.py @@ -139,6 +139,6 @@ class TranslatorMixin: class HTMLTranslator(TranslatorMixin, OriginalHTMLTranslator): - @lru_cache(maxsize=256) + @lru_cache(maxsize=2048) def css_to_xpath(self, css: str, prefix: str = "descendant-or-self::") -> str: return super().css_to_xpath(css, prefix) From ae60c2c0ade0bd0c8b072d916ea0733cd60360b4 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Thu, 30 Jan 2025 20:07:19 +0200 Subject: [PATCH 19/30] fix(playwright engine): bug in checking `nstbrowser_config` type --- scrapling/engines/pw.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapling/engines/pw.py b/scrapling/engines/pw.py index 2cd93e5..c0ca34c 100644 --- a/scrapling/engines/pw.py +++ b/scrapling/engines/pw.py @@ -105,7 +105,7 @@ class PlaywrightEngine: """ cdp_url = self.cdp_url if self.nstbrowser_mode: - if self.nstbrowser_config and type(self.nstbrowser_config) is Dict: + if self.nstbrowser_config and isinstance(self.nstbrowser_config, dict): config = self.nstbrowser_config else: query = NSTBROWSER_DEFAULT_QUERY.copy() From f37538320fae3dec5145a870d3d16f16fb8fb4ca Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Thu, 30 Jan 2025 23:05:10 +0200 Subject: [PATCH 20/30] feat(parser): adding `below_elements` method --- scrapling/parser.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/scrapling/parser.py b/scrapling/parser.py index 5743306..cadfa82 100644 --- a/scrapling/parser.py +++ b/scrapling/parser.py @@ -19,7 +19,7 @@ from scrapling.core.storage_adaptors import (SQLiteStorageSystem, StorageSystemMixin, _StorageTools) from scrapling.core.translator import HTMLTranslator from scrapling.core.utils import (clean_spaces, flatten, html_forbidden, - is_jsonable, log) + is_jsonable, log, lru_cache) class Adaptor(SelectorsGeneration): @@ -284,6 +284,13 @@ class Adaptor(SelectorsGeneration): """Return the direct parent of the element or ``None`` otherwise""" return self.__handle_element(self._root.getparent()) + @property + @lru_cache(None, True) + def below_elements(self) -> 'Adaptors[Adaptor]': + """Return all elements under the current element in the DOM tree""" + below = self._root.xpath('.//*') + return self.__handle_elements(below) + @property def children(self) -> 'Adaptors[Adaptor]': """Return the children elements of the current element or empty list otherwise""" From 57090f90a4856a5be92553c824f4ebb1276e360a Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Thu, 30 Jan 2025 23:39:46 +0200 Subject: [PATCH 21/30] refactor(parser/find_all): about ~95% speed increase Boom! --- scrapling/parser.py | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/scrapling/parser.py b/scrapling/parser.py index cadfa82..4e96501 100644 --- a/scrapling/parser.py +++ b/scrapling/parser.py @@ -593,14 +593,6 @@ class Adaptor(SelectorsGeneration): tags, patterns = set(), set() results, functions, selectors = Adaptors([]), [], [] - def _search_tree(element: Adaptor, filter_function: Callable) -> None: - """Collect element if it fulfills passed function otherwise, traverse the children tree and iterate""" - if filter_function(element): - results.append(element) - - for branch in element.children: - _search_tree(branch, filter_function) - # Brace yourself for a wonderful journey! for arg in args: if type(arg) is str: @@ -661,9 +653,9 @@ class Adaptor(SelectorsGeneration): for pattern in patterns: results.extend(self.find_by_regex(pattern, first_match=False)) - for result in (results or [self]): - for function in functions: - _search_tree(result, function) + # Collect element if it fulfills passed function otherwise + for function in functions: + results.extend((results or self.below_elements).filter(function)) return results From 1962c11f64fdb7877afa6f37470430943704b7c2 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Fri, 31 Jan 2025 00:23:54 +0200 Subject: [PATCH 22/30] fix(parser/find_all): Logic issues made conditions used sometimes in a (and) fashion and other times (OR) Also, the speed boost is 6-14% not 95% as I said first :( --- scrapling/parser.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/scrapling/parser.py b/scrapling/parser.py index 4e96501..e01c680 100644 --- a/scrapling/parser.py +++ b/scrapling/parser.py @@ -19,7 +19,7 @@ from scrapling.core.storage_adaptors import (SQLiteStorageSystem, StorageSystemMixin, _StorageTools) from scrapling.core.translator import HTMLTranslator from scrapling.core.utils import (clean_spaces, flatten, html_forbidden, - is_jsonable, log, lru_cache) + is_jsonable, log) class Adaptor(SelectorsGeneration): @@ -285,7 +285,6 @@ class Adaptor(SelectorsGeneration): return self.__handle_element(self._root.getparent()) @property - @lru_cache(None, True) def below_elements(self) -> 'Adaptors[Adaptor]': """Return all elements under the current element in the DOM tree""" below = self._root.xpath('.//*') @@ -603,12 +602,12 @@ class Adaptor(SelectorsGeneration): raise TypeError('Nested Iterables are not accepted, only iterables of tag names are accepted') tags.update(set(arg)) - elif type(arg) is dict: + elif isinstance(arg, dict): if not all([(type(k) is str and type(v) is str) for k, v in arg.items()]): raise TypeError('Nested dictionaries are not accepted, only string keys and string values are accepted') attributes.update(arg) - elif type(arg) is re.Pattern: + elif isinstance(arg, re.Pattern): patterns.add(arg) elif callable(arg): @@ -629,14 +628,14 @@ class Adaptor(SelectorsGeneration): attributes[attribute_name] = value # It's easier and faster to build a selector than traversing the tree - tags = tags or [''] + tags = tags or ['*'] for tag in tags: selector = tag for key, value in attributes.items(): value = value.replace('"', r'\"') # Escape double quotes in user input # Not escaping anything with the key so the user can pass patterns like {'href*': '/p/'} or get errors :) selector += '[{}="{}"]'.format(key, value) - if selector: + if selector != '*': selectors.append(selector) if selectors: @@ -650,12 +649,13 @@ class Adaptor(SelectorsGeneration): for function in functions: results = results.filter(function) else: + results = results or self.below_elements for pattern in patterns: - results.extend(self.find_by_regex(pattern, first_match=False)) + results = results.filter(lambda e: e.text.re(pattern, check_match=True)) # Collect element if it fulfills passed function otherwise for function in functions: - results.extend((results or self.below_elements).filter(function)) + results = results.filter(function) return results From 3f23f18473a592e5e79c037c93bce859c1559e75 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Fri, 31 Jan 2025 00:48:27 +0200 Subject: [PATCH 23/30] style(TextHandler): fix re overload --- scrapling/core/custom_types.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapling/core/custom_types.py b/scrapling/core/custom_types.py index 03257c3..6e4ef50 100644 --- a/scrapling/core/custom_types.py +++ b/scrapling/core/custom_types.py @@ -131,10 +131,10 @@ class TextHandler(str): def re( self, regex: Union[str, Pattern[str]], + check_match: Literal[True], replace_entities: bool = True, clean_match: bool = False, case_sensitive: bool = False, - check_match: Literal[True] = True, ) -> bool: ... From 9ea245eab4350ec3bb5f0242485eb1f1d23cbdbf Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Fri, 31 Jan 2025 00:48:59 +0200 Subject: [PATCH 24/30] doc(README): be more specific about what the regex match --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index e76d507..7c67b49 100644 --- a/README.md +++ b/README.md @@ -545,7 +545,7 @@ Inspired by BeautifulSoup's `find_all` function you can find elements by using ` * Any string passed is considered a tag name * Any iterable passed like List/Tuple/Set is considered an iterable of tag names. * Any dictionary is considered a mapping of HTML element(s) attribute names and attribute values. - * Any regex patterns passed are used as filters + * Any regex patterns passed are used as filters to elements by their text content * Any functions passed are used as filters * Any keyword argument passed is considered as an HTML element attribute with its value. @@ -554,7 +554,7 @@ So the way it works is after collecting all passed arguments and keywords, each 1. All elements with the passed tag name(s). 2. All elements that match all passed attribute(s). -3. All elements that match all passed regex patterns. +3. All elements that its text content match all passed regex patterns. 4. All elements that fulfill all passed function(s). Note: The filtering process always starts from the first filter it finds in the filtering order above so if no tag name(s) are passed but attributes are passed, the process starts from that layer and so on. **But the order in which you pass the arguments doesn't matter.** From 2c6142d96d17ea97acf9a49bd99fa0d50265fcfb Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Fri, 31 Jan 2025 01:12:00 +0200 Subject: [PATCH 25/30] refactor(parser/find_by_text): Better implementation for ~20% speed boost --- scrapling/parser.py | 31 ++++++++++++------------------- 1 file changed, 12 insertions(+), 19 deletions(-) diff --git a/scrapling/parser.py b/scrapling/parser.py index e01c680..f5765ca 100644 --- a/scrapling/parser.py +++ b/scrapling/parser.py @@ -907,32 +907,25 @@ class Adaptor(SelectorsGeneration): if not case_sensitive: text = text.lower() - def _traverse(node: Adaptor) -> None: + # This selector gets all elements with text content + for node in self.__handle_elements(self._root.xpath('//*[normalize-space(text())]')): """Check if element matches given text otherwise, traverse the children tree and iterate""" node_text = node.text - # if there's already no text in this node, dodge it to save CPU cycles and time - if node_text: - if clean_match: - node_text = node_text.clean() + if clean_match: + node_text = node_text.clean() - if not case_sensitive: - node_text = node_text.lower() + if not case_sensitive: + node_text = node_text.lower() - if partial: - if text in node_text: - results.append(node) - elif text == node_text: + if partial: + if text in node_text: results.append(node) + elif text == node_text: + results.append(node) - if results and first_match: + if first_match and results: # we got an element so we should stop - return - - for branch in node.children: - _traverse(branch) - - # This will block until we traverse all children/branches - _traverse(self) + break if first_match: if results: From 37a31a120e471038c11dd923f77dec3ed3ecb0d7 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Fri, 31 Jan 2025 01:19:11 +0200 Subject: [PATCH 26/30] refactor(parser/find_by_regex): Better implementation for ~60% speed boost --- scrapling/parser.py | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/scrapling/parser.py b/scrapling/parser.py index f5765ca..d92ce53 100644 --- a/scrapling/parser.py +++ b/scrapling/parser.py @@ -943,23 +943,16 @@ class Adaptor(SelectorsGeneration): """ results = Adaptors([]) - def _traverse(node: Adaptor) -> None: + # This selector gets all elements with text content + for node in self.__handle_elements(self._root.xpath('//*[normalize-space(text())]')): """Check if element matches given regex otherwise, traverse the children tree and iterate""" node_text = node.text - # if there's already no text in this node, dodge it to save CPU cycles and time - if node_text: - if node_text.re(query, check_match=True, clean_match=clean_match, case_sensitive=case_sensitive): - results.append(node) + if node_text.re(query, check_match=True, clean_match=clean_match, case_sensitive=case_sensitive): + results.append(node) - if results and first_match: + if first_match and results: # we got an element so we should stop - return - - for branch in node.children: - _traverse(branch) - - # This will block until we traverse all children/branches - _traverse(self) + break if results and first_match: return results[0] From 3330e9f4750e6f320ab35bf04d35b9bdda6612a4 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Fri, 31 Jan 2025 01:34:02 +0200 Subject: [PATCH 27/30] refactor(parser/automatch): Cleaner and faster implementation ~5% speed boost --- scrapling/parser.py | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/scrapling/parser.py b/scrapling/parser.py index d92ce53..86c9f3e 100644 --- a/scrapling/parser.py +++ b/scrapling/parser.py @@ -398,23 +398,12 @@ class Adaptor(SelectorsGeneration): if issubclass(type(element), html.HtmlElement): element = _StorageTools.element_to_dict(element) - # TODO: Optimize the traverse logic a bit, maybe later - def _traverse(node: html.HtmlElement, ele: Dict) -> None: - """Get the matching score of the given element against the node then traverse the children - - :param node: Current node in the tree structure - :param ele: The element we are searching for as dictionary - :return: - """ + for node in self._root.xpath('.//*'): + # Collect all elements in the page then for each element get the matching score of it against the node. # Hence: the code doesn't stop even if the score was 100% # because there might be another element(s) left in page with the same score - score = self.__calculate_similarity_score(ele, node) + score = self.__calculate_similarity_score(element, node) score_table.setdefault(score, []).append(node) - for branch in node.iterchildren(): - _traverse(branch, ele) - - # This will block until we traverse all children/branches - _traverse(self._root, element) if score_table: highest_probability = max(score_table.keys()) From a8d475afb0462d9bcf0bbaae06517b5f4f578d92 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Fri, 31 Jan 2025 01:38:48 +0200 Subject: [PATCH 28/30] feat(parser): Make all returned html as TextHandler So you can do regex on raw html if wanted etc... --- scrapling/parser.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/scrapling/parser.py b/scrapling/parser.py index 86c9f3e..d3ec663 100644 --- a/scrapling/parser.py +++ b/scrapling/parser.py @@ -259,18 +259,18 @@ class Adaptor(SelectorsGeneration): return self.__attributes @property - def html_content(self) -> str: + def html_content(self) -> TextHandler: """Return the inner html code of the element""" - return etree.tostring(self._root, encoding='unicode', method='html', with_tail=False) + return TextHandler(etree.tostring(self._root, encoding='unicode', method='html', with_tail=False)) @property - def body(self) -> str: + def body(self) -> TextHandler: """Return raw HTML code of the element/page without any processing when possible or return `Adaptor.html_content`""" - return self.__raw_body or self.html_content + return TextHandler(self.__raw_body) or self.html_content - def prettify(self) -> str: + def prettify(self) -> TextHandler: """Return a prettified version of the element's inner html-code""" - return etree.tostring(self._root, encoding='unicode', pretty_print=True, method='html', with_tail=False) + return TextHandler(etree.tostring(self._root, encoding='unicode', pretty_print=True, method='html', with_tail=False)) def has_class(self, class_name: str) -> bool: """Check if element has a specific class From a6fd25f64c3a8eddd305458bb378d0682b4c365d Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Fri, 31 Jan 2025 01:57:19 +0200 Subject: [PATCH 29/30] fix(parser): fix traverse selectors --- scrapling/parser.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapling/parser.py b/scrapling/parser.py index d3ec663..eac0d64 100644 --- a/scrapling/parser.py +++ b/scrapling/parser.py @@ -897,7 +897,7 @@ class Adaptor(SelectorsGeneration): text = text.lower() # This selector gets all elements with text content - for node in self.__handle_elements(self._root.xpath('//*[normalize-space(text())]')): + for node in self.__handle_elements(self._root.xpath('.//*[normalize-space(text())]')): """Check if element matches given text otherwise, traverse the children tree and iterate""" node_text = node.text if clean_match: @@ -933,7 +933,7 @@ class Adaptor(SelectorsGeneration): results = Adaptors([]) # This selector gets all elements with text content - for node in self.__handle_elements(self._root.xpath('//*[normalize-space(text())]')): + for node in self.__handle_elements(self._root.xpath('.//*[normalize-space(text())]')): """Check if element matches given regex otherwise, traverse the children tree and iterate""" node_text = node.text if node_text.re(query, check_match=True, clean_match=clean_match, case_sensitive=case_sensitive): From 7c35341d85cf78f4ad825985ee9f5b7d6b0dcb3c Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Fri, 31 Jan 2025 02:47:49 +0200 Subject: [PATCH 30/30] refactor(parser/get_all_text): Cleaner and a bit faster implementation --- scrapling/parser.py | 21 ++++----------------- 1 file changed, 4 insertions(+), 17 deletions(-) diff --git a/scrapling/parser.py b/scrapling/parser.py index eac0d64..5a4d4f5 100644 --- a/scrapling/parser.py +++ b/scrapling/parser.py @@ -223,29 +223,16 @@ class Adaptor(SelectorsGeneration): :return: A TextHandler """ _all_strings = [] - - def _traverse(node: html.HtmlElement) -> None: - """Traverse element children and get text content of each - - :param node: Current node in the tree structure - :return: - """ + for node in self._root.xpath('.//*'): if node.tag not in ignore_tags: text = node.text if text and type(text) is str: - if valid_values: - if text.strip(): - _all_strings.append(text if not strip else text.strip()) + if valid_values and text.strip(): + _all_strings.append(text if not strip else text.strip()) else: _all_strings.append(text if not strip else text.strip()) - for branch in node.iterchildren(): - _traverse(branch) - - # We will start using Lxml directly for the speed boost - _traverse(self._root) - - return TextHandler(separator.join([s for s in _all_strings])) + return TextHandler(separator.join(_all_strings)) def urljoin(self, relative_url: str) -> str: """Join this Adaptor's url with a relative url to form an absolute full URL."""