Merge pull request #51 from D4Vinci/dev

v0.2.98

README.md

@@ -118,7 +118,7 @@ Deep SerpApi is a dedicated search engine designed for large language models (LL
 ## Getting Started
 
 ```python
-from scrapling import Fetcher
+from scrapling.fetchers import Fetcher
 
 fetcher = Fetcher(auto_match=False)
 
@@ -200,7 +200,7 @@ Fetchers are interfaces built on top of other libraries with added features that
 ### Features
 You might be slightly confused by now so let me clear things up. All fetcher-type classes are imported in the same way
 ```python
-from scrapling import Fetcher, StealthyFetcher, PlayWrightFetcher
+from scrapling.fetchers import Fetcher, StealthyFetcher, PlayWrightFetcher
 ```
 All of them can take these initialization arguments: `auto_match`, `huge_tree`, `keep_comments`, `keep_cdata`, `storage`, and `storage_args`, which are the same ones you give to the `Adaptor` class.
 
@@ -232,7 +232,7 @@ You can route all traffic (HTTP and HTTPS) to a proxy for any of these methods i
 ```
 For Async requests, you will just replace the import like below:
 ```python
->> from scrapling import AsyncFetcher
+>> from scrapling.fetchers import AsyncFetcher
 >> page = await AsyncFetcher().get('https://httpbin.org/get', stealthy_headers=True, follow_redirects=True)
 >> page = await AsyncFetcher().post('https://httpbin.org/post', data={'key': 'value'}, proxy='http://username:password@localhost:8030')
 >> page = await AsyncFetcher().put('https://httpbin.org/put', data={'key': 'value'})
@@ -486,7 +486,7 @@ When website owners implement structural changes like
 The selector will no longer function and your code needs maintenance. That's where Scrapling's auto-matching feature comes into play.
 
 ```python
-from scrapling import Adaptor
+from scrapling.parser import Adaptor
 # Before the change
 page = Adaptor(page_source, url='example.com')
 element = page.css('#p1' auto_save=True)
@@ -504,7 +504,7 @@ To solve this issue, I will use [The Web Archive](https://archive.org/)'s [Wayba
 If I want to extract the Questions button from the old design I can use a selector like this `#hmenus > div:nth-child(1) > ul > li:nth-child(1) > a` This selector is too specific because it was generated by Google Chrome.
 Now let's test the same selector in both versions
 ```python
->> from scrapling import Fetcher
+>> from scrapling.fetchers import Fetcher
 >> selector = '#hmenus > div:nth-child(1) > ul > li:nth-child(1) > a'
 >> old_url = "https://web.archive.org/web/20100102003420/http://stackoverflow.com/"
 >> new_url = "https://stackoverflow.com/"
@@ -565,7 +565,7 @@ Note: The filtering process always starts from the first filter it finds in the
 Examples to clear any confusion :)
 
 ```python
->> from scrapling import Fetcher
+>> from scrapling.fetchers import Fetcher
 >> page = Fetcher().get('https://quotes.toscrape.com/')
 # Find all elements with tag name `div`.
 >> page.find_all('div')

docs/Core/using scrapling custom types.md

@@ -2,7 +2,7 @@
 
 ### All current types can be imported alone like below
 ```python
->>> from scrapling import TextHandler, AttributesHandler
+>>> from scrapling.core.custom_types import TextHandler, AttributesHandler
 
 >>> somestring = TextHandler('{}')
 >>> somestring.json()

scrapling/__init__.py

@@ -1,12 +1,41 @@
-# Declare top-level shortcuts
-from scrapling.core.custom_types import AttributesHandler, TextHandler
-from scrapling.fetchers import (AsyncFetcher, CustomFetcher, Fetcher,
-                                PlayWrightFetcher, StealthyFetcher)
-from scrapling.parser import Adaptor, Adaptors
 
 __author__ = "Karim Shoair (karim.shoair@pm.me)"
-__version__ = "0.2.97"
+__version__ = "0.2.98"
 __copyright__ = "Copyright (c) 2024 Karim Shoair"
 
 
+# A lightweight approach to create lazy loader for each import for backward compatibility
+# This will reduces initial memory footprint significantly (only loads what's used)
+def __getattr__(name):
+    if name == 'Fetcher':
+        from scrapling.fetchers import Fetcher as cls
+        return cls
+    elif name == 'Adaptor':
+        from scrapling.parser import Adaptor as cls
+        return cls
+    elif name == 'Adaptors':
+        from scrapling.parser import Adaptors as cls
+        return cls
+    elif name == 'AttributesHandler':
+        from scrapling.core.custom_types import AttributesHandler as cls
+        return cls
+    elif name == 'TextHandler':
+        from scrapling.core.custom_types import TextHandler as cls
+        return cls
+    elif name == 'AsyncFetcher':
+        from scrapling.fetchers import AsyncFetcher as cls
+        return cls
+    elif name == 'StealthyFetcher':
+        from scrapling.fetchers import StealthyFetcher as cls
+        return cls
+    elif name == 'PlayWrightFetcher':
+        from scrapling.fetchers import PlayWrightFetcher as cls
+        return cls
+    elif name == 'CustomFetcher':
+        from scrapling.fetchers import CustomFetcher as cls
+        return cls
+    else:
+        raise AttributeError(f"module 'scrapling' has no attribute '{name}'")
+
+
 __all__ = ['Adaptor', 'Fetcher', 'AsyncFetcher', 'StealthyFetcher', 'PlayWrightFetcher']

scrapling/core/storage_adaptors.py

@@ -19,7 +19,7 @@ class StorageSystemMixin(ABC):
         """
         self.url = url
 
-    @lru_cache(126, typed=True)
+    @lru_cache(64, typed=True)
     def _get_base_url(self, default_value: str = 'default') -> str:
         if not self.url or type(self.url) is not str:
             return default_value
@@ -51,7 +51,7 @@ class StorageSystemMixin(ABC):
         raise NotImplementedError('Storage system must implement `save` method')
 
     @staticmethod
-    @lru_cache(256, typed=True)
+    @lru_cache(128, typed=True)
     def _get_hash(identifier: str) -> str:
         """If you want to hash identifier in your storage system, use this safer"""
         identifier = identifier.lower().strip()
@@ -63,7 +63,7 @@ class StorageSystemMixin(ABC):
         return f"{hash_value}_{len(identifier)}"  # Length to reduce collision chance
 
 
-@lru_cache(10, typed=True)
+@lru_cache(1, typed=True)
 class SQLiteStorageSystem(StorageSystemMixin):
     """The recommended system to use, it's race condition safe and thread safe.
     Mainly built so the library can run in threaded frameworks like scrapy or threaded tools

scrapling/core/translator.py

@@ -142,3 +142,6 @@ class HTMLTranslator(TranslatorMixin, OriginalHTMLTranslator):
     @lru_cache(maxsize=256)
     def css_to_xpath(self, css: str, prefix: str = "descendant-or-self::") -> str:
         return super().css_to_xpath(css, prefix)
+
+
+translator_instance = HTMLTranslator()

scrapling/core/utils.py

@@ -115,7 +115,7 @@ class _StorageTools:
 #     return _impl
 
 
-@lru_cache(256, typed=True)
+@lru_cache(128, typed=True)
 def clean_spaces(string):
     string = string.replace('\t', ' ')
     string = re.sub('[\n|\r]', '', string)

scrapling/defaults.py

@@ -1,10 +1,19 @@
-from .fetchers import AsyncFetcher as _AsyncFetcher
-from .fetchers import Fetcher as _Fetcher
-from .fetchers import PlayWrightFetcher as _PlayWrightFetcher
-from .fetchers import StealthyFetcher as _StealthyFetcher
-
 # If you are going to use Fetchers with the default settings, import them from this file instead for a cleaner looking code
-Fetcher = _Fetcher()
+
-AsyncFetcher = _AsyncFetcher()
+# A lightweight approach to create lazy loader for each import for backward compatibility
-StealthyFetcher = _StealthyFetcher()
+# This will reduces initial memory footprint significantly (only loads what's used)
-PlayWrightFetcher = _PlayWrightFetcher()
+def __getattr__(name):
+    if name == 'Fetcher':
+        from scrapling.fetchers import Fetcher as cls
+        return cls()
+    elif name == 'AsyncFetcher':
+        from scrapling.fetchers import AsyncFetcher as cls
+        return cls()
+    elif name == 'StealthyFetcher':
+        from scrapling.fetchers import StealthyFetcher as cls
+        return cls()
+    elif name == 'PlayWrightFetcher':
+        from scrapling.fetchers import PlayWrightFetcher as cls
+        return cls()
+    else:
+        raise AttributeError(f"module 'scrapling' has no attribute '{name}'")

scrapling/engines/pw.py

@@ -126,7 +126,7 @@ class PlaywrightEngine:
 
         return cdp_url
 
-    @lru_cache(126, typed=True)
+    @lru_cache(32, typed=True)
     def __set_flags(self):
         """Returns the flags that will be used while launching the browser if stealth mode is enabled"""
         flags = DEFAULT_STEALTH_FLAGS
@@ -169,7 +169,7 @@ class PlaywrightEngine:
 
         return context_kwargs
 
-    @lru_cache(10)
+    @lru_cache(1)
     def __stealth_scripts(self):
         # Basic bypasses nothing fancy as I'm still working on it
         # But with adding these bypasses to the above config, it bypasses many online tests like

scrapling/engines/static.py

@@ -7,7 +7,7 @@ from scrapling.core.utils import log, lru_cache
 from .toolbelt import Response, generate_convincing_referer, generate_headers
 
 
-@lru_cache(5, typed=True)  # Singleton easily
+@lru_cache(2, typed=True)  # Singleton easily
 class StaticEngine:
     def __init__(
             self, url: str, proxy: Optional[str] = None, stealthy_headers: bool = True, follow_redirects: bool = True,

scrapling/engines/toolbelt/custom.py

@@ -16,7 +16,7 @@ class ResponseEncoding:
     __ISO_8859_1_CONTENT_TYPES = {"text/plain", "text/html", "text/css", "text/javascript"}
 
     @classmethod
-    @lru_cache(maxsize=256)
+    @lru_cache(maxsize=128)
     def __parse_content_type(cls, header_value: str) -> Tuple[str, Dict[str, str]]:
         """Parse content type and parameters from a content-type header value.
 
@@ -38,7 +38,7 @@ class ResponseEncoding:
         return content_type, params
 
     @classmethod
-    @lru_cache(maxsize=256)
+    @lru_cache(maxsize=128)
     def get_value(cls, content_type: Optional[str], text: Optional[str] = 'test') -> str:
         """Determine the appropriate character encoding from a content-type header.
 

scrapling/engines/toolbelt/fingerprints.py

@@ -12,7 +12,7 @@ from scrapling.core._types import Dict, Union
 from scrapling.core.utils import lru_cache
 
 
-@lru_cache(128, typed=True)
+@lru_cache(10, typed=True)
 def generate_convincing_referer(url: str) -> str:
     """Takes the domain from the URL without the subdomain/suffix and make it look like you were searching google for this website
 
@@ -26,7 +26,7 @@ def generate_convincing_referer(url: str) -> str:
     return f'https://www.google.com/search?q={website_name}'
 
 
-@lru_cache(128, typed=True)
+@lru_cache(1, typed=True)
 def get_os_name() -> Union[str, None]:
     """Get the current OS name in the same format needed for browserforge
 

scrapling/engines/toolbelt/navigation.py

@@ -110,7 +110,7 @@ def construct_cdp_url(cdp_url: str, query_params: Optional[Dict] = None) -> str:
         raise ValueError(f"Invalid CDP URL: {str(e)}")
 
 
-@lru_cache(126, typed=True)
+@lru_cache(10, typed=True)
 def js_bypass_path(filename: str) -> str:
     """Takes the base filename of JS file inside the `bypasses` folder then return the full path of it
 

scrapling/parser.py

@@ -17,7 +17,7 @@ from scrapling.core.custom_types import (AttributesHandler, TextHandler,
 from scrapling.core.mixins import SelectorsGeneration
 from scrapling.core.storage_adaptors import (SQLiteStorageSystem,
                                              StorageSystemMixin, _StorageTools)
-from scrapling.core.translator import HTMLTranslator
+from scrapling.core.translator import translator_instance
 from scrapling.core.utils import (clean_spaces, flatten, html_forbidden,
                                   is_jsonable, log)
 
@@ -26,7 +26,7 @@ class Adaptor(SelectorsGeneration):
     __slots__ = (
         'url', 'encoding', '__auto_match_enabled', '_root', '_storage',
         '__keep_comments', '__huge_tree_enabled', '__attributes', '__text', '__tag',
-        '__keep_cdata', '__raw_body'
+        '__keep_cdata'
     )
 
     def __init__(
@@ -72,20 +72,17 @@ class Adaptor(SelectorsGeneration):
             raise ValueError("Adaptor class needs text, body, or root arguments to work")
 
         self.__text = ''
-        self.__raw_body = ''
         if root is None:
             if text is None:
                 if not body or not isinstance(body, bytes):
                     raise TypeError(f"body argument must be valid and of type bytes, got {body.__class__}")
 
                 body = body.replace(b"\x00", b"").strip()
-                self.__raw_body = body.replace(b"\x00", b"").strip().decode()
             else:
                 if not isinstance(text, str):
                     raise TypeError(f"text argument must be of type str, got {text.__class__}")
 
                 body = text.strip().replace("\x00", "").encode(encoding) or b"<html/>"
-                self.__raw_body = text.strip()
 
             # https://lxml.de/api/lxml.etree.HTMLParser-class.html
             parser = html.HTMLParser(
@@ -250,10 +247,7 @@ class Adaptor(SelectorsGeneration):
         """Return the inner html code of the element"""
         return TextHandler(etree.tostring(self._root, encoding='unicode', method='html', with_tail=False))
 
-    @property
+    body = html_content
-    def body(self) -> TextHandler:
-        """Return raw HTML code of the element/page without any processing when possible or return `Adaptor.html_content`"""
-        return TextHandler(self.__raw_body) or self.html_content
 
     def prettify(self) -> TextHandler:
         """Return a prettified version of the element's inner html-code"""
@@ -476,7 +470,7 @@ class Adaptor(SelectorsGeneration):
         try:
             if not self.__auto_match_enabled or ',' not in selector:
                 # No need to split selectors in this case, let's save some CPU cycles :)
-                xpath_selector = HTMLTranslator().css_to_xpath(selector)
+                xpath_selector = translator_instance.css_to_xpath(selector)
                 return self.xpath(xpath_selector, identifier or selector, auto_match, auto_save, percentage)
 
             results = []
@@ -484,7 +478,7 @@ class Adaptor(SelectorsGeneration):
                 for single_selector in split_selectors(selector):
                     # I'm doing this only so the `save` function save data correctly for combined selectors
                     # Like using the ',' to combine two different selectors that point to different elements.
-                    xpath_selector = HTMLTranslator().css_to_xpath(single_selector.canonical())
+                    xpath_selector = translator_instance.css_to_xpath(single_selector.canonical())
                     results += self.xpath(
                         xpath_selector, identifier or single_selector.canonical(), auto_match, auto_save, percentage
                     )

setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = scrapling
-version = 0.2.97
+version = 0.2.98
 author = Karim Shoair
 author_email = karim.shoair@pm.me
 description = Scrapling is an undetectable, powerful, flexible, high-performance Python library that makes Web Scraping easy again!

setup.py

@@ -6,7 +6,7 @@ with open("README.md", "r", encoding="utf-8") as fh:
 
 setup(
     name="scrapling",
-    version="0.2.97",
+    version="0.2.98",
     description="""Scrapling is an undetectable, powerful, flexible, high-performance Python library that makes Web Scraping easy again! In an internet filled with complications,
     it simplifies web scraping, even when websites' design changes, while providing impressive speed that surpasses almost all alternatives.""",
     long_description=long_description,