diff --git a/README.md b/README.md index cb067a3..88a7fda 100644 --- a/README.md +++ b/README.md @@ -118,7 +118,7 @@ Deep SerpApi is a dedicated search engine designed for large language models (LL ## Getting Started ```python -from scrapling import Fetcher +from scrapling.fetchers import Fetcher fetcher = Fetcher(auto_match=False) @@ -200,7 +200,7 @@ Fetchers are interfaces built on top of other libraries with added features that ### Features You might be slightly confused by now so let me clear things up. All fetcher-type classes are imported in the same way ```python -from scrapling import Fetcher, StealthyFetcher, PlayWrightFetcher +from scrapling.fetchers import Fetcher, StealthyFetcher, PlayWrightFetcher ``` All of them can take these initialization arguments: `auto_match`, `huge_tree`, `keep_comments`, `keep_cdata`, `storage`, and `storage_args`, which are the same ones you give to the `Adaptor` class. @@ -232,7 +232,7 @@ You can route all traffic (HTTP and HTTPS) to a proxy for any of these methods i ``` For Async requests, you will just replace the import like below: ```python ->> from scrapling import AsyncFetcher +>> from scrapling.fetchers import AsyncFetcher >> page = await AsyncFetcher().get('https://httpbin.org/get', stealthy_headers=True, follow_redirects=True) >> page = await AsyncFetcher().post('https://httpbin.org/post', data={'key': 'value'}, proxy='http://username:password@localhost:8030') >> page = await AsyncFetcher().put('https://httpbin.org/put', data={'key': 'value'}) @@ -486,7 +486,7 @@ When website owners implement structural changes like The selector will no longer function and your code needs maintenance. That's where Scrapling's auto-matching feature comes into play. ```python -from scrapling import Adaptor +from scrapling.parser import Adaptor # Before the change page = Adaptor(page_source, url='example.com') element = page.css('#p1' auto_save=True) @@ -504,7 +504,7 @@ To solve this issue, I will use [The Web Archive](https://archive.org/)'s [Wayba If I want to extract the Questions button from the old design I can use a selector like this `#hmenus > div:nth-child(1) > ul > li:nth-child(1) > a` This selector is too specific because it was generated by Google Chrome. Now let's test the same selector in both versions ```python ->> from scrapling import Fetcher +>> from scrapling.fetchers import Fetcher >> selector = '#hmenus > div:nth-child(1) > ul > li:nth-child(1) > a' >> old_url = "https://web.archive.org/web/20100102003420/http://stackoverflow.com/" >> new_url = "https://stackoverflow.com/" @@ -565,7 +565,7 @@ Note: The filtering process always starts from the first filter it finds in the Examples to clear any confusion :) ```python ->> from scrapling import Fetcher +>> from scrapling.fetchers import Fetcher >> page = Fetcher().get('https://quotes.toscrape.com/') # Find all elements with tag name `div`. >> page.find_all('div') diff --git a/docs/Core/using scrapling custom types.md b/docs/Core/using scrapling custom types.md index 1c98216..202c91b 100644 --- a/docs/Core/using scrapling custom types.md +++ b/docs/Core/using scrapling custom types.md @@ -2,7 +2,7 @@ ### All current types can be imported alone like below ```python ->>> from scrapling import TextHandler, AttributesHandler +>>> from scrapling.core.custom_types import TextHandler, AttributesHandler >>> somestring = TextHandler('{}') >>> somestring.json() diff --git a/scrapling/__init__.py b/scrapling/__init__.py index 0877054..feda050 100644 --- a/scrapling/__init__.py +++ b/scrapling/__init__.py @@ -1,12 +1,41 @@ -# Declare top-level shortcuts -from scrapling.core.custom_types import AttributesHandler, TextHandler -from scrapling.fetchers import (AsyncFetcher, CustomFetcher, Fetcher, - PlayWrightFetcher, StealthyFetcher) -from scrapling.parser import Adaptor, Adaptors __author__ = "Karim Shoair (karim.shoair@pm.me)" -__version__ = "0.2.97" +__version__ = "0.2.98" __copyright__ = "Copyright (c) 2024 Karim Shoair" +# A lightweight approach to create lazy loader for each import for backward compatibility +# This will reduces initial memory footprint significantly (only loads what's used) +def __getattr__(name): + if name == 'Fetcher': + from scrapling.fetchers import Fetcher as cls + return cls + elif name == 'Adaptor': + from scrapling.parser import Adaptor as cls + return cls + elif name == 'Adaptors': + from scrapling.parser import Adaptors as cls + return cls + elif name == 'AttributesHandler': + from scrapling.core.custom_types import AttributesHandler as cls + return cls + elif name == 'TextHandler': + from scrapling.core.custom_types import TextHandler as cls + return cls + elif name == 'AsyncFetcher': + from scrapling.fetchers import AsyncFetcher as cls + return cls + elif name == 'StealthyFetcher': + from scrapling.fetchers import StealthyFetcher as cls + return cls + elif name == 'PlayWrightFetcher': + from scrapling.fetchers import PlayWrightFetcher as cls + return cls + elif name == 'CustomFetcher': + from scrapling.fetchers import CustomFetcher as cls + return cls + else: + raise AttributeError(f"module 'scrapling' has no attribute '{name}'") + + __all__ = ['Adaptor', 'Fetcher', 'AsyncFetcher', 'StealthyFetcher', 'PlayWrightFetcher'] diff --git a/scrapling/core/storage_adaptors.py b/scrapling/core/storage_adaptors.py index d991111..d6f67b9 100644 --- a/scrapling/core/storage_adaptors.py +++ b/scrapling/core/storage_adaptors.py @@ -19,7 +19,7 @@ class StorageSystemMixin(ABC): """ self.url = url - @lru_cache(126, typed=True) + @lru_cache(64, typed=True) def _get_base_url(self, default_value: str = 'default') -> str: if not self.url or type(self.url) is not str: return default_value @@ -51,7 +51,7 @@ class StorageSystemMixin(ABC): raise NotImplementedError('Storage system must implement `save` method') @staticmethod - @lru_cache(256, typed=True) + @lru_cache(128, typed=True) def _get_hash(identifier: str) -> str: """If you want to hash identifier in your storage system, use this safer""" identifier = identifier.lower().strip() @@ -63,7 +63,7 @@ class StorageSystemMixin(ABC): return f"{hash_value}_{len(identifier)}" # Length to reduce collision chance -@lru_cache(10, typed=True) +@lru_cache(1, typed=True) class SQLiteStorageSystem(StorageSystemMixin): """The recommended system to use, it's race condition safe and thread safe. Mainly built so the library can run in threaded frameworks like scrapy or threaded tools diff --git a/scrapling/core/translator.py b/scrapling/core/translator.py index 263a24a..d2d03fa 100644 --- a/scrapling/core/translator.py +++ b/scrapling/core/translator.py @@ -142,3 +142,6 @@ class HTMLTranslator(TranslatorMixin, OriginalHTMLTranslator): @lru_cache(maxsize=256) def css_to_xpath(self, css: str, prefix: str = "descendant-or-self::") -> str: return super().css_to_xpath(css, prefix) + + +translator_instance = HTMLTranslator() diff --git a/scrapling/core/utils.py b/scrapling/core/utils.py index e9de112..6555139 100644 --- a/scrapling/core/utils.py +++ b/scrapling/core/utils.py @@ -115,7 +115,7 @@ class _StorageTools: # return _impl -@lru_cache(256, typed=True) +@lru_cache(128, typed=True) def clean_spaces(string): string = string.replace('\t', ' ') string = re.sub('[\n|\r]', '', string) diff --git a/scrapling/defaults.py b/scrapling/defaults.py index 20f0c44..4098f76 100644 --- a/scrapling/defaults.py +++ b/scrapling/defaults.py @@ -1,10 +1,19 @@ -from .fetchers import AsyncFetcher as _AsyncFetcher -from .fetchers import Fetcher as _Fetcher -from .fetchers import PlayWrightFetcher as _PlayWrightFetcher -from .fetchers import StealthyFetcher as _StealthyFetcher - # If you are going to use Fetchers with the default settings, import them from this file instead for a cleaner looking code -Fetcher = _Fetcher() -AsyncFetcher = _AsyncFetcher() -StealthyFetcher = _StealthyFetcher() -PlayWrightFetcher = _PlayWrightFetcher() + +# A lightweight approach to create lazy loader for each import for backward compatibility +# This will reduces initial memory footprint significantly (only loads what's used) +def __getattr__(name): + if name == 'Fetcher': + from scrapling.fetchers import Fetcher as cls + return cls() + elif name == 'AsyncFetcher': + from scrapling.fetchers import AsyncFetcher as cls + return cls() + elif name == 'StealthyFetcher': + from scrapling.fetchers import StealthyFetcher as cls + return cls() + elif name == 'PlayWrightFetcher': + from scrapling.fetchers import PlayWrightFetcher as cls + return cls() + else: + raise AttributeError(f"module 'scrapling' has no attribute '{name}'") diff --git a/scrapling/engines/pw.py b/scrapling/engines/pw.py index 45a0ff3..0c71fec 100644 --- a/scrapling/engines/pw.py +++ b/scrapling/engines/pw.py @@ -126,7 +126,7 @@ class PlaywrightEngine: return cdp_url - @lru_cache(126, typed=True) + @lru_cache(32, typed=True) def __set_flags(self): """Returns the flags that will be used while launching the browser if stealth mode is enabled""" flags = DEFAULT_STEALTH_FLAGS @@ -169,7 +169,7 @@ class PlaywrightEngine: return context_kwargs - @lru_cache(10) + @lru_cache(1) def __stealth_scripts(self): # Basic bypasses nothing fancy as I'm still working on it # But with adding these bypasses to the above config, it bypasses many online tests like diff --git a/scrapling/engines/static.py b/scrapling/engines/static.py index f19b503..0aa4c2c 100644 --- a/scrapling/engines/static.py +++ b/scrapling/engines/static.py @@ -7,7 +7,7 @@ from scrapling.core.utils import log, lru_cache from .toolbelt import Response, generate_convincing_referer, generate_headers -@lru_cache(5, typed=True) # Singleton easily +@lru_cache(2, typed=True) # Singleton easily class StaticEngine: def __init__( self, url: str, proxy: Optional[str] = None, stealthy_headers: bool = True, follow_redirects: bool = True, diff --git a/scrapling/engines/toolbelt/custom.py b/scrapling/engines/toolbelt/custom.py index f00f649..c91a3a8 100644 --- a/scrapling/engines/toolbelt/custom.py +++ b/scrapling/engines/toolbelt/custom.py @@ -16,7 +16,7 @@ class ResponseEncoding: __ISO_8859_1_CONTENT_TYPES = {"text/plain", "text/html", "text/css", "text/javascript"} @classmethod - @lru_cache(maxsize=256) + @lru_cache(maxsize=128) def __parse_content_type(cls, header_value: str) -> Tuple[str, Dict[str, str]]: """Parse content type and parameters from a content-type header value. @@ -38,7 +38,7 @@ class ResponseEncoding: return content_type, params @classmethod - @lru_cache(maxsize=256) + @lru_cache(maxsize=128) def get_value(cls, content_type: Optional[str], text: Optional[str] = 'test') -> str: """Determine the appropriate character encoding from a content-type header. diff --git a/scrapling/engines/toolbelt/fingerprints.py b/scrapling/engines/toolbelt/fingerprints.py index cfdcf8a..dfbed5a 100644 --- a/scrapling/engines/toolbelt/fingerprints.py +++ b/scrapling/engines/toolbelt/fingerprints.py @@ -12,7 +12,7 @@ from scrapling.core._types import Dict, Union from scrapling.core.utils import lru_cache -@lru_cache(128, typed=True) +@lru_cache(10, typed=True) def generate_convincing_referer(url: str) -> str: """Takes the domain from the URL without the subdomain/suffix and make it look like you were searching google for this website @@ -26,7 +26,7 @@ def generate_convincing_referer(url: str) -> str: return f'https://www.google.com/search?q={website_name}' -@lru_cache(128, typed=True) +@lru_cache(1, typed=True) def get_os_name() -> Union[str, None]: """Get the current OS name in the same format needed for browserforge diff --git a/scrapling/engines/toolbelt/navigation.py b/scrapling/engines/toolbelt/navigation.py index 4d39fb5..fefb1e3 100644 --- a/scrapling/engines/toolbelt/navigation.py +++ b/scrapling/engines/toolbelt/navigation.py @@ -110,7 +110,7 @@ def construct_cdp_url(cdp_url: str, query_params: Optional[Dict] = None) -> str: raise ValueError(f"Invalid CDP URL: {str(e)}") -@lru_cache(126, typed=True) +@lru_cache(10, typed=True) def js_bypass_path(filename: str) -> str: """Takes the base filename of JS file inside the `bypasses` folder then return the full path of it diff --git a/scrapling/parser.py b/scrapling/parser.py index f9e177e..98e0864 100644 --- a/scrapling/parser.py +++ b/scrapling/parser.py @@ -17,7 +17,7 @@ from scrapling.core.custom_types import (AttributesHandler, TextHandler, from scrapling.core.mixins import SelectorsGeneration from scrapling.core.storage_adaptors import (SQLiteStorageSystem, StorageSystemMixin, _StorageTools) -from scrapling.core.translator import HTMLTranslator +from scrapling.core.translator import translator_instance from scrapling.core.utils import (clean_spaces, flatten, html_forbidden, is_jsonable, log) @@ -26,7 +26,7 @@ class Adaptor(SelectorsGeneration): __slots__ = ( 'url', 'encoding', '__auto_match_enabled', '_root', '_storage', '__keep_comments', '__huge_tree_enabled', '__attributes', '__text', '__tag', - '__keep_cdata', '__raw_body' + '__keep_cdata' ) def __init__( @@ -72,20 +72,17 @@ class Adaptor(SelectorsGeneration): raise ValueError("Adaptor class needs text, body, or root arguments to work") self.__text = '' - self.__raw_body = '' if root is None: if text is None: if not body or not isinstance(body, bytes): raise TypeError(f"body argument must be valid and of type bytes, got {body.__class__}") body = body.replace(b"\x00", b"").strip() - self.__raw_body = body.replace(b"\x00", b"").strip().decode() else: if not isinstance(text, str): raise TypeError(f"text argument must be of type str, got {text.__class__}") body = text.strip().replace("\x00", "").encode(encoding) or b"" - self.__raw_body = text.strip() # https://lxml.de/api/lxml.etree.HTMLParser-class.html parser = html.HTMLParser( @@ -250,10 +247,7 @@ class Adaptor(SelectorsGeneration): """Return the inner html code of the element""" return TextHandler(etree.tostring(self._root, encoding='unicode', method='html', with_tail=False)) - @property - def body(self) -> TextHandler: - """Return raw HTML code of the element/page without any processing when possible or return `Adaptor.html_content`""" - return TextHandler(self.__raw_body) or self.html_content + body = html_content def prettify(self) -> TextHandler: """Return a prettified version of the element's inner html-code""" @@ -476,7 +470,7 @@ class Adaptor(SelectorsGeneration): try: if not self.__auto_match_enabled or ',' not in selector: # No need to split selectors in this case, let's save some CPU cycles :) - xpath_selector = HTMLTranslator().css_to_xpath(selector) + xpath_selector = translator_instance.css_to_xpath(selector) return self.xpath(xpath_selector, identifier or selector, auto_match, auto_save, percentage) results = [] @@ -484,7 +478,7 @@ class Adaptor(SelectorsGeneration): for single_selector in split_selectors(selector): # I'm doing this only so the `save` function save data correctly for combined selectors # Like using the ',' to combine two different selectors that point to different elements. - xpath_selector = HTMLTranslator().css_to_xpath(single_selector.canonical()) + xpath_selector = translator_instance.css_to_xpath(single_selector.canonical()) results += self.xpath( xpath_selector, identifier or single_selector.canonical(), auto_match, auto_save, percentage ) diff --git a/setup.cfg b/setup.cfg index 28004e3..35d934c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = scrapling -version = 0.2.97 +version = 0.2.98 author = Karim Shoair author_email = karim.shoair@pm.me description = Scrapling is an undetectable, powerful, flexible, high-performance Python library that makes Web Scraping easy again! diff --git a/setup.py b/setup.py index e5f36bf..b060b9b 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ with open("README.md", "r", encoding="utf-8") as fh: setup( name="scrapling", - version="0.2.97", + version="0.2.98", description="""Scrapling is an undetectable, powerful, flexible, high-performance Python library that makes Web Scraping easy again! In an internet filled with complications, it simplifies web scraping, even when websites' design changes, while providing impressive speed that surpasses almost all alternatives.""", long_description=long_description,