From e5a022ec6acbfa66d7ac697beac96119a0897071 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Sun, 16 Mar 2025 22:33:46 +0200 Subject: [PATCH 1/7] fix(parser): Improving CSS selecting memory usage by ~144% and speed by ~2% --- scrapling/core/translator.py | 3 +++ scrapling/parser.py | 6 +++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/scrapling/core/translator.py b/scrapling/core/translator.py index 263a24a..d2d03fa 100644 --- a/scrapling/core/translator.py +++ b/scrapling/core/translator.py @@ -142,3 +142,6 @@ class HTMLTranslator(TranslatorMixin, OriginalHTMLTranslator): @lru_cache(maxsize=256) def css_to_xpath(self, css: str, prefix: str = "descendant-or-self::") -> str: return super().css_to_xpath(css, prefix) + + +translator_instance = HTMLTranslator() diff --git a/scrapling/parser.py b/scrapling/parser.py index f9e177e..146ea47 100644 --- a/scrapling/parser.py +++ b/scrapling/parser.py @@ -17,7 +17,7 @@ from scrapling.core.custom_types import (AttributesHandler, TextHandler, from scrapling.core.mixins import SelectorsGeneration from scrapling.core.storage_adaptors import (SQLiteStorageSystem, StorageSystemMixin, _StorageTools) -from scrapling.core.translator import HTMLTranslator +from scrapling.core.translator import translator_instance from scrapling.core.utils import (clean_spaces, flatten, html_forbidden, is_jsonable, log) @@ -476,7 +476,7 @@ class Adaptor(SelectorsGeneration): try: if not self.__auto_match_enabled or ',' not in selector: # No need to split selectors in this case, let's save some CPU cycles :) - xpath_selector = HTMLTranslator().css_to_xpath(selector) + xpath_selector = translator_instance.css_to_xpath(selector) return self.xpath(xpath_selector, identifier or selector, auto_match, auto_save, percentage) results = [] @@ -484,7 +484,7 @@ class Adaptor(SelectorsGeneration): for single_selector in split_selectors(selector): # I'm doing this only so the `save` function save data correctly for combined selectors # Like using the ',' to combine two different selectors that point to different elements. - xpath_selector = HTMLTranslator().css_to_xpath(single_selector.canonical()) + xpath_selector = translator_instance.css_to_xpath(single_selector.canonical()) results += self.xpath( xpath_selector, identifier or single_selector.canonical(), auto_match, auto_save, percentage ) From 68ac9d4d05cfbcea3f2646579f84156e15df301d Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Mon, 17 Mar 2025 05:58:40 +0200 Subject: [PATCH 2/7] fix(parser): Lowering memory with replacing `body` property with `html_content` --- scrapling/parser.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/scrapling/parser.py b/scrapling/parser.py index 146ea47..98e0864 100644 --- a/scrapling/parser.py +++ b/scrapling/parser.py @@ -26,7 +26,7 @@ class Adaptor(SelectorsGeneration): __slots__ = ( 'url', 'encoding', '__auto_match_enabled', '_root', '_storage', '__keep_comments', '__huge_tree_enabled', '__attributes', '__text', '__tag', - '__keep_cdata', '__raw_body' + '__keep_cdata' ) def __init__( @@ -72,20 +72,17 @@ class Adaptor(SelectorsGeneration): raise ValueError("Adaptor class needs text, body, or root arguments to work") self.__text = '' - self.__raw_body = '' if root is None: if text is None: if not body or not isinstance(body, bytes): raise TypeError(f"body argument must be valid and of type bytes, got {body.__class__}") body = body.replace(b"\x00", b"").strip() - self.__raw_body = body.replace(b"\x00", b"").strip().decode() else: if not isinstance(text, str): raise TypeError(f"text argument must be of type str, got {text.__class__}") body = text.strip().replace("\x00", "").encode(encoding) or b"" - self.__raw_body = text.strip() # https://lxml.de/api/lxml.etree.HTMLParser-class.html parser = html.HTMLParser( @@ -250,10 +247,7 @@ class Adaptor(SelectorsGeneration): """Return the inner html code of the element""" return TextHandler(etree.tostring(self._root, encoding='unicode', method='html', with_tail=False)) - @property - def body(self) -> TextHandler: - """Return raw HTML code of the element/page without any processing when possible or return `Adaptor.html_content`""" - return TextHandler(self.__raw_body) or self.html_content + body = html_content def prettify(self) -> TextHandler: """Return a prettified version of the element's inner html-code""" From bbc05ca1645d622ab27d144d1b36d81231e31f15 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Mon, 17 Mar 2025 06:24:42 +0200 Subject: [PATCH 3/7] fix(importing): Lazy Loading submodules to lower memory usage on importing Load what you use instead of all submodules together --- scrapling/__init__.py | 39 ++++++++++++++++++++++++++++++++++----- 1 file changed, 34 insertions(+), 5 deletions(-) diff --git a/scrapling/__init__.py b/scrapling/__init__.py index 0877054..e8461b5 100644 --- a/scrapling/__init__.py +++ b/scrapling/__init__.py @@ -1,12 +1,41 @@ -# Declare top-level shortcuts -from scrapling.core.custom_types import AttributesHandler, TextHandler -from scrapling.fetchers import (AsyncFetcher, CustomFetcher, Fetcher, - PlayWrightFetcher, StealthyFetcher) -from scrapling.parser import Adaptor, Adaptors __author__ = "Karim Shoair (karim.shoair@pm.me)" __version__ = "0.2.97" __copyright__ = "Copyright (c) 2024 Karim Shoair" +# A lightweight approach to create lazy loader for each import for backward compatibility +# This will reduces initial memory footprint significantly (only loads what's used) +def __getattr__(name): + if name == 'Fetcher': + from scrapling.fetchers import Fetcher as cls + return cls + elif name == 'Adaptor': + from scrapling.parser import Adaptor as cls + return cls + elif name == 'Adaptors': + from scrapling.parser import Adaptors as cls + return cls + elif name == 'AttributesHandler': + from scrapling.core.custom_types import AttributesHandler as cls + return cls + elif name == 'TextHandler': + from scrapling.core.custom_types import TextHandler as cls + return cls + elif name == 'AsyncFetcher': + from scrapling.fetchers import AsyncFetcher as cls + return cls + elif name == 'StealthyFetcher': + from scrapling.fetchers import StealthyFetcher as cls + return cls + elif name == 'PlayWrightFetcher': + from scrapling.fetchers import PlayWrightFetcher as cls + return cls + elif name == 'CustomFetcher': + from scrapling.fetchers import CustomFetcher as cls + return cls + else: + raise AttributeError(f"module 'scrapling' has no attribute '{name}'") + + __all__ = ['Adaptor', 'Fetcher', 'AsyncFetcher', 'StealthyFetcher', 'PlayWrightFetcher'] From 18d2d29fa887206261f56f070f91db73c31f9f2e Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Mon, 17 Mar 2025 06:29:32 +0200 Subject: [PATCH 4/7] docs: use a better import to reflect the new changes --- README.md | 12 ++++++------ docs/Core/using scrapling custom types.md | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index cb067a3..88a7fda 100644 --- a/README.md +++ b/README.md @@ -118,7 +118,7 @@ Deep SerpApi is a dedicated search engine designed for large language models (LL ## Getting Started ```python -from scrapling import Fetcher +from scrapling.fetchers import Fetcher fetcher = Fetcher(auto_match=False) @@ -200,7 +200,7 @@ Fetchers are interfaces built on top of other libraries with added features that ### Features You might be slightly confused by now so let me clear things up. All fetcher-type classes are imported in the same way ```python -from scrapling import Fetcher, StealthyFetcher, PlayWrightFetcher +from scrapling.fetchers import Fetcher, StealthyFetcher, PlayWrightFetcher ``` All of them can take these initialization arguments: `auto_match`, `huge_tree`, `keep_comments`, `keep_cdata`, `storage`, and `storage_args`, which are the same ones you give to the `Adaptor` class. @@ -232,7 +232,7 @@ You can route all traffic (HTTP and HTTPS) to a proxy for any of these methods i ``` For Async requests, you will just replace the import like below: ```python ->> from scrapling import AsyncFetcher +>> from scrapling.fetchers import AsyncFetcher >> page = await AsyncFetcher().get('https://httpbin.org/get', stealthy_headers=True, follow_redirects=True) >> page = await AsyncFetcher().post('https://httpbin.org/post', data={'key': 'value'}, proxy='http://username:password@localhost:8030') >> page = await AsyncFetcher().put('https://httpbin.org/put', data={'key': 'value'}) @@ -486,7 +486,7 @@ When website owners implement structural changes like The selector will no longer function and your code needs maintenance. That's where Scrapling's auto-matching feature comes into play. ```python -from scrapling import Adaptor +from scrapling.parser import Adaptor # Before the change page = Adaptor(page_source, url='example.com') element = page.css('#p1' auto_save=True) @@ -504,7 +504,7 @@ To solve this issue, I will use [The Web Archive](https://archive.org/)'s [Wayba If I want to extract the Questions button from the old design I can use a selector like this `#hmenus > div:nth-child(1) > ul > li:nth-child(1) > a` This selector is too specific because it was generated by Google Chrome. Now let's test the same selector in both versions ```python ->> from scrapling import Fetcher +>> from scrapling.fetchers import Fetcher >> selector = '#hmenus > div:nth-child(1) > ul > li:nth-child(1) > a' >> old_url = "https://web.archive.org/web/20100102003420/http://stackoverflow.com/" >> new_url = "https://stackoverflow.com/" @@ -565,7 +565,7 @@ Note: The filtering process always starts from the first filter it finds in the Examples to clear any confusion :) ```python ->> from scrapling import Fetcher +>> from scrapling.fetchers import Fetcher >> page = Fetcher().get('https://quotes.toscrape.com/') # Find all elements with tag name `div`. >> page.find_all('div') diff --git a/docs/Core/using scrapling custom types.md b/docs/Core/using scrapling custom types.md index 1c98216..202c91b 100644 --- a/docs/Core/using scrapling custom types.md +++ b/docs/Core/using scrapling custom types.md @@ -2,7 +2,7 @@ ### All current types can be imported alone like below ```python ->>> from scrapling import TextHandler, AttributesHandler +>>> from scrapling.core.custom_types import TextHandler, AttributesHandler >>> somestring = TextHandler('{}') >>> somestring.json() From ff3962b1adad331b11d38e8db643a0c6082d25d6 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Mon, 17 Mar 2025 06:46:41 +0200 Subject: [PATCH 5/7] fix: Adjusting cache size in all the library --- scrapling/core/storage_adaptors.py | 6 +++--- scrapling/core/utils.py | 2 +- scrapling/engines/pw.py | 4 ++-- scrapling/engines/static.py | 2 +- scrapling/engines/toolbelt/custom.py | 4 ++-- scrapling/engines/toolbelt/fingerprints.py | 4 ++-- scrapling/engines/toolbelt/navigation.py | 2 +- 7 files changed, 12 insertions(+), 12 deletions(-) diff --git a/scrapling/core/storage_adaptors.py b/scrapling/core/storage_adaptors.py index d991111..d6f67b9 100644 --- a/scrapling/core/storage_adaptors.py +++ b/scrapling/core/storage_adaptors.py @@ -19,7 +19,7 @@ class StorageSystemMixin(ABC): """ self.url = url - @lru_cache(126, typed=True) + @lru_cache(64, typed=True) def _get_base_url(self, default_value: str = 'default') -> str: if not self.url or type(self.url) is not str: return default_value @@ -51,7 +51,7 @@ class StorageSystemMixin(ABC): raise NotImplementedError('Storage system must implement `save` method') @staticmethod - @lru_cache(256, typed=True) + @lru_cache(128, typed=True) def _get_hash(identifier: str) -> str: """If you want to hash identifier in your storage system, use this safer""" identifier = identifier.lower().strip() @@ -63,7 +63,7 @@ class StorageSystemMixin(ABC): return f"{hash_value}_{len(identifier)}" # Length to reduce collision chance -@lru_cache(10, typed=True) +@lru_cache(1, typed=True) class SQLiteStorageSystem(StorageSystemMixin): """The recommended system to use, it's race condition safe and thread safe. Mainly built so the library can run in threaded frameworks like scrapy or threaded tools diff --git a/scrapling/core/utils.py b/scrapling/core/utils.py index e9de112..6555139 100644 --- a/scrapling/core/utils.py +++ b/scrapling/core/utils.py @@ -115,7 +115,7 @@ class _StorageTools: # return _impl -@lru_cache(256, typed=True) +@lru_cache(128, typed=True) def clean_spaces(string): string = string.replace('\t', ' ') string = re.sub('[\n|\r]', '', string) diff --git a/scrapling/engines/pw.py b/scrapling/engines/pw.py index 45a0ff3..0c71fec 100644 --- a/scrapling/engines/pw.py +++ b/scrapling/engines/pw.py @@ -126,7 +126,7 @@ class PlaywrightEngine: return cdp_url - @lru_cache(126, typed=True) + @lru_cache(32, typed=True) def __set_flags(self): """Returns the flags that will be used while launching the browser if stealth mode is enabled""" flags = DEFAULT_STEALTH_FLAGS @@ -169,7 +169,7 @@ class PlaywrightEngine: return context_kwargs - @lru_cache(10) + @lru_cache(1) def __stealth_scripts(self): # Basic bypasses nothing fancy as I'm still working on it # But with adding these bypasses to the above config, it bypasses many online tests like diff --git a/scrapling/engines/static.py b/scrapling/engines/static.py index f19b503..0aa4c2c 100644 --- a/scrapling/engines/static.py +++ b/scrapling/engines/static.py @@ -7,7 +7,7 @@ from scrapling.core.utils import log, lru_cache from .toolbelt import Response, generate_convincing_referer, generate_headers -@lru_cache(5, typed=True) # Singleton easily +@lru_cache(2, typed=True) # Singleton easily class StaticEngine: def __init__( self, url: str, proxy: Optional[str] = None, stealthy_headers: bool = True, follow_redirects: bool = True, diff --git a/scrapling/engines/toolbelt/custom.py b/scrapling/engines/toolbelt/custom.py index f00f649..c91a3a8 100644 --- a/scrapling/engines/toolbelt/custom.py +++ b/scrapling/engines/toolbelt/custom.py @@ -16,7 +16,7 @@ class ResponseEncoding: __ISO_8859_1_CONTENT_TYPES = {"text/plain", "text/html", "text/css", "text/javascript"} @classmethod - @lru_cache(maxsize=256) + @lru_cache(maxsize=128) def __parse_content_type(cls, header_value: str) -> Tuple[str, Dict[str, str]]: """Parse content type and parameters from a content-type header value. @@ -38,7 +38,7 @@ class ResponseEncoding: return content_type, params @classmethod - @lru_cache(maxsize=256) + @lru_cache(maxsize=128) def get_value(cls, content_type: Optional[str], text: Optional[str] = 'test') -> str: """Determine the appropriate character encoding from a content-type header. diff --git a/scrapling/engines/toolbelt/fingerprints.py b/scrapling/engines/toolbelt/fingerprints.py index cfdcf8a..dfbed5a 100644 --- a/scrapling/engines/toolbelt/fingerprints.py +++ b/scrapling/engines/toolbelt/fingerprints.py @@ -12,7 +12,7 @@ from scrapling.core._types import Dict, Union from scrapling.core.utils import lru_cache -@lru_cache(128, typed=True) +@lru_cache(10, typed=True) def generate_convincing_referer(url: str) -> str: """Takes the domain from the URL without the subdomain/suffix and make it look like you were searching google for this website @@ -26,7 +26,7 @@ def generate_convincing_referer(url: str) -> str: return f'https://www.google.com/search?q={website_name}' -@lru_cache(128, typed=True) +@lru_cache(1, typed=True) def get_os_name() -> Union[str, None]: """Get the current OS name in the same format needed for browserforge diff --git a/scrapling/engines/toolbelt/navigation.py b/scrapling/engines/toolbelt/navigation.py index 4d39fb5..fefb1e3 100644 --- a/scrapling/engines/toolbelt/navigation.py +++ b/scrapling/engines/toolbelt/navigation.py @@ -110,7 +110,7 @@ def construct_cdp_url(cdp_url: str, query_params: Optional[Dict] = None) -> str: raise ValueError(f"Invalid CDP URL: {str(e)}") -@lru_cache(126, typed=True) +@lru_cache(10, typed=True) def js_bypass_path(filename: str) -> str: """Takes the base filename of JS file inside the `bypasses` folder then return the full path of it From 94f16ca1af785938a11a1fd7753e768c8a7d5d5a Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Mon, 17 Mar 2025 06:47:43 +0200 Subject: [PATCH 6/7] build: Pumping up the version --- scrapling/__init__.py | 2 +- setup.cfg | 2 +- setup.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/scrapling/__init__.py b/scrapling/__init__.py index e8461b5..feda050 100644 --- a/scrapling/__init__.py +++ b/scrapling/__init__.py @@ -1,6 +1,6 @@ __author__ = "Karim Shoair (karim.shoair@pm.me)" -__version__ = "0.2.97" +__version__ = "0.2.98" __copyright__ = "Copyright (c) 2024 Karim Shoair" diff --git a/setup.cfg b/setup.cfg index 28004e3..35d934c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = scrapling -version = 0.2.97 +version = 0.2.98 author = Karim Shoair author_email = karim.shoair@pm.me description = Scrapling is an undetectable, powerful, flexible, high-performance Python library that makes Web Scraping easy again! diff --git a/setup.py b/setup.py index e5f36bf..b060b9b 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ with open("README.md", "r", encoding="utf-8") as fh: setup( name="scrapling", - version="0.2.97", + version="0.2.98", description="""Scrapling is an undetectable, powerful, flexible, high-performance Python library that makes Web Scraping easy again! In an internet filled with complications, it simplifies web scraping, even when websites' design changes, while providing impressive speed that surpasses almost all alternatives.""", long_description=long_description, From 9b0515b1cd2eed8e05c1a875e7cee07670e32651 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Mon, 17 Mar 2025 06:56:34 +0200 Subject: [PATCH 7/7] fix(defaults): Lazy Loading fetchers to lower memory usage on importing --- scrapling/defaults.py | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/scrapling/defaults.py b/scrapling/defaults.py index 20f0c44..4098f76 100644 --- a/scrapling/defaults.py +++ b/scrapling/defaults.py @@ -1,10 +1,19 @@ -from .fetchers import AsyncFetcher as _AsyncFetcher -from .fetchers import Fetcher as _Fetcher -from .fetchers import PlayWrightFetcher as _PlayWrightFetcher -from .fetchers import StealthyFetcher as _StealthyFetcher - # If you are going to use Fetchers with the default settings, import them from this file instead for a cleaner looking code -Fetcher = _Fetcher() -AsyncFetcher = _AsyncFetcher() -StealthyFetcher = _StealthyFetcher() -PlayWrightFetcher = _PlayWrightFetcher() + +# A lightweight approach to create lazy loader for each import for backward compatibility +# This will reduces initial memory footprint significantly (only loads what's used) +def __getattr__(name): + if name == 'Fetcher': + from scrapling.fetchers import Fetcher as cls + return cls() + elif name == 'AsyncFetcher': + from scrapling.fetchers import AsyncFetcher as cls + return cls() + elif name == 'StealthyFetcher': + from scrapling.fetchers import StealthyFetcher as cls + return cls() + elif name == 'PlayWrightFetcher': + from scrapling.fetchers import PlayWrightFetcher as cls + return cls() + else: + raise AttributeError(f"module 'scrapling' has no attribute '{name}'")