refactor(api)!: Unifying log under 1 logger and removing debug parameter

So now you control the logging and the debugging from the shell through the logger with the name 'scrapling'

.github/ISSUE_TEMPLATE/01-bug_report.yml

@@ -65,7 +65,7 @@ body:
 
   - type: textarea
     attributes:
-      label: "Actual behavior (Remember to use `debug` parameter)"
+      label: "Actual behavior"
     validations:
       required: true
 

CONTRIBUTING.md

@@ -19,7 +19,11 @@ tests/test_parser_functions.py ................          [100%]
 
 =============================== 16 passed in 0.22s ================================
 ```
-Also, consider setting `debug` to `True` while initializing the Adaptor object so it's easier to know what's happening in the background.
+Also, consider setting the scrapling logging level to `debug` so it's easier to know what's happening in the background.
+```python
+>>> import logging
+>>> logging.getLogger("scrapling").setLevel(logging.DEBUG)
+```
 
 ### The process is straight-forward.
 

README.md

@@ -219,7 +219,7 @@ You might be slightly confused by now so let me clear things up. All fetcher-typ
 ```python
 from scrapling import Fetcher, StealthyFetcher, PlayWrightFetcher
 ```
-All of them can take these initialization arguments: `auto_match`, `huge_tree`, `keep_comments`, `keep_cdata`, `storage`, `storage_args`, and `debug`, which are the same ones you give to the `Adaptor` class.
+All of them can take these initialization arguments: `auto_match`, `huge_tree`, `keep_comments`, `keep_cdata`, `storage`, and `storage_args`, which are the same ones you give to the `Adaptor` class.
 
 If you don't want to pass arguments to the generated `Adaptor` object and want to use the default values, you can use this import instead for cleaner code:
 ```python

benchmarks.py

@@ -64,9 +64,9 @@ def test_pyquery():
 @benchmark
 def test_scrapling():
     # No need to do `.extract()` like parsel to extract text
-    # Also, this is faster than `[t.text for t in Adaptor(large_html, auto_match=False, debug=False).css('.item')]`
+    # Also, this is faster than `[t.text for t in Adaptor(large_html, auto_match=False).css('.item')]`
     # for obvious reasons, of course.
-    return Adaptor(large_html, auto_match=False, debug=False).css('.item::text')
+    return Adaptor(large_html, auto_match=False).css('.item::text')
 
 
 @benchmark
@@ -103,7 +103,7 @@ def test_scrapling_text(request_html):
     # Will loop over resulted elements to get text too to make comparison even more fair otherwise Scrapling will be even faster
     return [
         element.text for element in Adaptor(
-            request_html, auto_match=False, debug=False
+            request_html, auto_match=False
         ).find_by_text('Tipping the Velvet', first_match=True).find_similar(ignore_attributes=['title'])
     ]
 

scrapling/core/storage_adaptors.py

@@ -1,4 +1,3 @@
-import logging
 import sqlite3
 import threading
 from abc import ABC, abstractmethod
@@ -9,7 +8,7 @@ from lxml import html
 from tldextract import extract as tld
 
 from scrapling.core._types import Dict, Optional, Union
-from scrapling.core.utils import _StorageTools, cache
+from scrapling.core.utils import _StorageTools, log, lru_cache
 
 
 class StorageSystemMixin(ABC):
@@ -20,7 +19,7 @@ class StorageSystemMixin(ABC):
         """
         self.url = url
 
-    @cache(None, typed=True)
+    @lru_cache(None, typed=True)
     def _get_base_url(self, default_value: str = 'default') -> str:
         if not self.url or type(self.url) is not str:
             return default_value
@@ -52,7 +51,7 @@ class StorageSystemMixin(ABC):
         raise NotImplementedError('Storage system must implement `save` method')
 
     @staticmethod
-    @cache(None, typed=True)
+    @lru_cache(None, typed=True)
     def _get_hash(identifier: str) -> str:
         """If you want to hash identifier in your storage system, use this safer"""
         identifier = identifier.lower().strip()
@@ -64,7 +63,7 @@ class StorageSystemMixin(ABC):
         return f"{hash_value}_{len(identifier)}"  # Length to reduce collision chance
 
 
-@cache(None, typed=True)
+@lru_cache(None, typed=True)
 class SQLiteStorageSystem(StorageSystemMixin):
     """The recommended system to use, it's race condition safe and thread safe.
     Mainly built so the library can run in threaded frameworks like scrapy or threaded tools
@@ -86,7 +85,7 @@ class SQLiteStorageSystem(StorageSystemMixin):
         self.connection.execute("PRAGMA journal_mode=WAL")
         self.cursor = self.connection.cursor()
         self._setup_database()
-        logging.debug(
+        log.debug(
             f'Storage system loaded with arguments (storage_file="{storage_file}", url="{url}")'
         )
 

scrapling/core/translator.py

@@ -17,7 +17,7 @@ from cssselect.xpath import XPathExpr as OriginalXPathExpr
 from w3lib.html import HTML5_WHITESPACE
 
 from scrapling.core._types import Any, Optional, Protocol, Self
-from scrapling.core.utils import cache
+from scrapling.core.utils import lru_cache
 
 regex = f"[{HTML5_WHITESPACE}]+"
 replace_html5_whitespaces = re.compile(regex).sub
@@ -139,6 +139,6 @@ class TranslatorMixin:
 
 
 class HTMLTranslator(TranslatorMixin, OriginalHTMLTranslator):
-    @cache(maxsize=256)
+    @lru_cache(maxsize=256)
     def css_to_xpath(self, css: str, prefix: str = "descendant-or-self::") -> str:
         return super().css_to_xpath(css, prefix)

scrapling/core/utils.py

@@ -9,18 +9,36 @@ from scrapling.core._types import Any, Dict, Iterable, Union
 
 # Using cache on top of a class is brilliant way to achieve Singleton design pattern without much code
 # functools.cache is available on Python 3.9+ only so let's keep lru_cache
-from functools import lru_cache as cache  # isort:skip
+from functools import lru_cache  # isort:skip
-
 
 html_forbidden = {html.HtmlComment, }
-logging.basicConfig(
+
-    level=logging.INFO,
+
-    format="[%(asctime)s] %(levelname)s: %(message)s",
+@lru_cache(1, typed=True)
-    datefmt="%Y-%m-%d %H:%M:%S",
+def setup_logger():
-    handlers=[
+    """Create and configure a logger with a standard format.
-        logging.StreamHandler()
+
-    ]
+    :returns: logging.Logger: Configured logger instance
-)
+    """
+    logger = logging.getLogger('scrapling')
+    logger.setLevel(logging.INFO)
+
+    formatter = logging.Formatter(
+        fmt="[%(asctime)s] %(levelname)s: %(message)s",
+        datefmt="%Y-%m-%d %H:%M:%S"
+    )
+
+    console_handler = logging.StreamHandler()
+    console_handler.setFormatter(formatter)
+
+    # Add handler to logger (if not already added)
+    if not logger.handlers:
+        logger.addHandler(console_handler)
+
+    return logger
+
+
+log = setup_logger()
 
 
 def is_jsonable(content: Union[bytes, str]) -> bool:
@@ -34,23 +52,6 @@ def is_jsonable(content: Union[bytes, str]) -> bool:
         return False
 
 
-@cache(None, typed=True)
-def setup_basic_logging(level: str = 'debug'):
-    levels = {
-        'debug': logging.DEBUG,
-        'info': logging.INFO,
-        'warning': logging.WARNING,
-        'error': logging.ERROR,
-        'critical': logging.CRITICAL
-    }
-    formatter = logging.Formatter("[%(asctime)s] %(levelname)s: %(message)s", "%Y-%m-%d %H:%M:%S")
-    lvl = levels[level.lower()]
-    handler = logging.StreamHandler()
-    handler.setFormatter(formatter)
-    # Configure the root logger
-    logging.basicConfig(level=lvl, handlers=[handler])
-
-
 def flatten(lst: Iterable):
     return list(chain.from_iterable(lst))
 
@@ -114,7 +115,7 @@ class _StorageTools:
 #     return _impl
 
 
-@cache(None, typed=True)
+@lru_cache(None, typed=True)
 def clean_spaces(string):
     string = string.replace('\t', ' ')
     string = re.sub('[\n|\r]', '', string)

scrapling/engines/camo.py

@@ -1,10 +1,9 @@
-import logging
-
 from camoufox import DefaultAddons
 from camoufox.sync_api import Camoufox
 
 from scrapling.core._types import (Callable, Dict, List, Literal, Optional,
                                    Union)
+from scrapling.core.utils import log
 from scrapling.engines.toolbelt import (Response, StatusText,
                                         check_type_validity,
                                         construct_proxy_dict, do_nothing,
@@ -63,7 +62,7 @@ class CamoufoxEngine:
             self.page_action = page_action
         else:
             self.page_action = do_nothing
-            logging.error('[Ignored] Argument "page_action" must be callable')
+            log.error('[Ignored] Argument "page_action" must be callable')
 
         self.wait_selector = wait_selector
         self.wait_selector_state = wait_selector_state

scrapling/engines/pw.py

@@ -1,7 +1,7 @@
 import json
-import logging
 
 from scrapling.core._types import Callable, Dict, List, Optional, Union
+from scrapling.core.utils import log
 from scrapling.engines.constants import (DEFAULT_STEALTH_FLAGS,
                                          NSTBROWSER_DEFAULT_QUERY)
 from scrapling.engines.toolbelt import (Response, StatusText,
@@ -78,7 +78,7 @@ class PlaywrightEngine:
             self.page_action = page_action
         else:
             self.page_action = do_nothing
-            logging.error('[Ignored] Argument "page_action" must be callable')
+            log.error('[Ignored] Argument "page_action" must be callable')
 
         self.wait_selector = wait_selector
         self.wait_selector_state = wait_selector_state

scrapling/engines/static.py

@@ -1,5 +1,3 @@
-import logging
-
 import httpx
 from httpx._models import Response as httpxResponse
 
@@ -36,7 +34,6 @@ class StaticEngine:
         # Validate headers
         if not headers.get('user-agent') and not headers.get('User-Agent'):
             headers['User-Agent'] = generate_headers(browser_mode=False).get('User-Agent')
-            logging.info(f"Can't find useragent in headers so '{headers['User-Agent']}' was used.")
 
         if stealth:
             extra_headers = generate_headers(browser_mode=False)

scrapling/engines/toolbelt/custom.py

@@ -2,13 +2,12 @@
 Functions related to custom types or type checking
 """
 import inspect
-import logging
 from email.message import Message
 
 from scrapling.core._types import (Any, Callable, Dict, List, Optional, Tuple,
                                    Type, Union)
 from scrapling.core.custom_types import MappingProxyType
-from scrapling.core.utils import cache, setup_basic_logging
+from scrapling.core.utils import log, lru_cache
 from scrapling.parser import Adaptor, SQLiteStorageSystem
 
 
@@ -17,7 +16,7 @@ class ResponseEncoding:
     __ISO_8859_1_CONTENT_TYPES = {"text/plain", "text/html", "text/css", "text/javascript"}
 
     @classmethod
-    @cache(maxsize=None)
+    @lru_cache(maxsize=None)
     def __parse_content_type(cls, header_value: str) -> Tuple[str, Dict[str, str]]:
         """Parse content type and parameters from a content-type header value.
 
@@ -39,7 +38,7 @@ class ResponseEncoding:
         return content_type, params
 
     @classmethod
-    @cache(maxsize=None)
+    @lru_cache(maxsize=None)
     def get_value(cls, content_type: Optional[str], text: Optional[str] = 'test') -> str:
         """Determine the appropriate character encoding from a content-type header.
 
@@ -98,7 +97,7 @@ class Response(Adaptor):
         # For back-ward compatibility
         self.adaptor = self
         # For easier debugging while working from a Python shell
-        logging.info(f'Fetched ({status}) <{method} {url}> (referer: {request_headers.get("referer")})')
+        log.info(f'Fetched ({status}) <{method} {url}> (referer: {request_headers.get("referer")})')
 
     # def __repr__(self):
     #     return f'<{self.__class__.__name__} [{self.status} {self.reason}]>'
@@ -107,7 +106,7 @@ class Response(Adaptor):
 class BaseFetcher:
     def __init__(
             self, huge_tree: bool = True, keep_comments: Optional[bool] = False, auto_match: Optional[bool] = True,
-            storage: Any = SQLiteStorageSystem, storage_args: Optional[Dict] = None, debug: Optional[bool] = False,
+            storage: Any = SQLiteStorageSystem, storage_args: Optional[Dict] = None,
             automatch_domain: Optional[str] = None, keep_cdata: Optional[bool] = False,
     ):
         """Arguments below are the same from the Adaptor class so you can pass them directly, the rest of Adaptor's arguments
@@ -124,7 +123,6 @@ class BaseFetcher:
             If empty, default values will be used.
         :param automatch_domain: For cases where you want to automatch selectors across different websites as if they were on the same website, use this argument to unify them.
             Otherwise, the domain of the request is used by default.
-        :param debug: Enable debug mode
         """
         # Adaptor class parameters
         # I won't validate Adaptor's class parameters here again, I will leave it to be validated later
@@ -134,14 +132,11 @@ class BaseFetcher:
             keep_cdata=keep_cdata,
             auto_match=auto_match,
             storage=storage,
-            storage_args=storage_args,
+            storage_args=storage_args
-            debug=debug,
         )
-        # If the user used fetchers first, then configure the logger from here instead of the `Adaptor` class
-        setup_basic_logging(level='debug' if debug else 'info')
         if automatch_domain:
             if type(automatch_domain) is not str:
-                logging.warning('[Ignored] The argument "automatch_domain" must be of string type')
+                log.warning('[Ignored] The argument "automatch_domain" must be of string type')
             else:
                 self.adaptor_arguments.update({'automatch_domain': automatch_domain})
 
@@ -217,7 +212,7 @@ class StatusText:
     })
 
     @classmethod
-    @cache(maxsize=128)
+    @lru_cache(maxsize=128)
     def get(cls, status_code: int) -> str:
         """Get the phrase for a given HTTP status code."""
         return cls._phrases.get(status_code, "Unknown Status Code")
@@ -284,7 +279,7 @@ def check_type_validity(variable: Any, valid_types: Union[List[Type], None], def
         error_msg = f'Argument "{var_name}" cannot be None'
         if critical:
             raise TypeError(error_msg)
-        logging.error(f'[Ignored] {error_msg}')
+        log.error(f'[Ignored] {error_msg}')
         return default_value
 
     # If no valid_types specified and variable has a value, return it
@@ -297,7 +292,7 @@ def check_type_validity(variable: Any, valid_types: Union[List[Type], None], def
         error_msg = f'Argument "{var_name}" must be of type {" or ".join(type_names)}'
         if critical:
             raise TypeError(error_msg)
-        logging.error(f'[Ignored] {error_msg}')
+        log.error(f'[Ignored] {error_msg}')
         return default_value
 
     return variable

scrapling/engines/toolbelt/fingerprints.py

@@ -9,10 +9,10 @@ from browserforge.headers import Browser, HeaderGenerator
 from tldextract import extract
 
 from scrapling.core._types import Dict, Union
-from scrapling.core.utils import cache
+from scrapling.core.utils import lru_cache
 
 
-@cache(None, typed=True)
+@lru_cache(None, typed=True)
 def generate_convincing_referer(url: str) -> str:
     """Takes the domain from the URL without the subdomain/suffix and make it look like you were searching google for this website
 
@@ -26,7 +26,7 @@ def generate_convincing_referer(url: str) -> str:
     return f'https://www.google.com/search?q={website_name}'
 
 
-@cache(None, typed=True)
+@lru_cache(None, typed=True)
 def get_os_name() -> Union[str, None]:
     """Get the current OS name in the same format needed for browserforge
 

scrapling/engines/toolbelt/navigation.py

@@ -1,15 +1,13 @@
 """
 Functions related to files and URLs
 """
-
-import logging
 import os
 from urllib.parse import urlencode, urlparse
 
 from playwright.sync_api import Route
 
 from scrapling.core._types import Dict, Optional, Union
-from scrapling.core.utils import cache
+from scrapling.core.utils import log, lru_cache
 from scrapling.engines.constants import DEFAULT_DISABLED_RESOURCES
 
 
@@ -20,7 +18,7 @@ def intercept_route(route: Route) -> Union[Route, None]:
     :return: PlayWright `Route` object
     """
     if route.request.resource_type in DEFAULT_DISABLED_RESOURCES:
-        logging.debug(f'Blocking background resource "{route.request.url}" of type "{route.request.resource_type}"')
+        log.debug(f'Blocking background resource "{route.request.url}" of type "{route.request.resource_type}"')
         return route.abort()
     return route.continue_()
 
@@ -97,7 +95,7 @@ def construct_cdp_url(cdp_url: str, query_params: Optional[Dict] = None) -> str:
         raise ValueError(f"Invalid CDP URL: {str(e)}")
 
 
-@cache(None, typed=True)
+@lru_cache(None, typed=True)
 def js_bypass_path(filename: str) -> str:
     """Takes the base filename of JS file inside the `bypasses` folder then return the full path of it
 

scrapling/parser.py

@@ -18,12 +18,12 @@ from scrapling.core.storage_adaptors import (SQLiteStorageSystem,
                                              StorageSystemMixin, _StorageTools)
 from scrapling.core.translator import HTMLTranslator
 from scrapling.core.utils import (clean_spaces, flatten, html_forbidden,
-                                  is_jsonable, logging, setup_basic_logging)
+                                  is_jsonable, log)
 
 
 class Adaptor(SelectorsGeneration):
     __slots__ = (
-        'url', 'encoding', '__auto_match_enabled', '_root', '_storage', '__debug',
+        'url', 'encoding', '__auto_match_enabled', '_root', '_storage',
         '__keep_comments', '__huge_tree_enabled', '__attributes', '__text', '__tag',
         '__keep_cdata', '__raw_body'
     )
@@ -41,7 +41,6 @@ class Adaptor(SelectorsGeneration):
             auto_match: Optional[bool] = True,
             storage: Any = SQLiteStorageSystem,
             storage_args: Optional[Dict] = None,
-            debug: Optional[bool] = True,
             **kwargs
     ):
         """The main class that works as a wrapper for the HTML input data. Using this class, you can search for elements
@@ -67,7 +66,6 @@ class Adaptor(SelectorsGeneration):
         :param storage: The storage class to be passed for auto-matching functionalities, see ``Docs`` for more info.
         :param storage_args: A dictionary of ``argument->value`` pairs to be passed for the storage class.
             If empty, default values will be used.
-        :param debug: Enable debug mode
         """
         if root is None and not body and text is None:
             raise ValueError("Adaptor class needs text, body, or root arguments to work")
@@ -106,7 +104,6 @@ class Adaptor(SelectorsGeneration):
 
             self._root = root
 
-        setup_basic_logging(level='debug' if debug else 'info')
         self.__auto_match_enabled = auto_match
 
         if self.__auto_match_enabled:
@@ -117,7 +114,7 @@ class Adaptor(SelectorsGeneration):
                 }
 
             if not hasattr(storage, '__wrapped__'):
-                raise ValueError("Storage class must be wrapped with cache decorator, see docs for info")
+                raise ValueError("Storage class must be wrapped with lru_cache decorator, see docs for info")
 
             if not issubclass(storage.__wrapped__, StorageSystemMixin):
                 raise ValueError("Storage system must be inherited from class `StorageSystemMixin`")
@@ -132,7 +129,6 @@ class Adaptor(SelectorsGeneration):
         # For selector stuff
         self.__attributes = None
         self.__tag = None
-        self.__debug = debug
         # No need to check if all response attributes exist or not because if `status` exist, then the rest exist (Save some CPU cycles for speed)
         self.__response_data = {
             key: getattr(self, key) for key in ('status', 'reason', 'cookies', 'headers', 'request_headers',)
@@ -164,7 +160,7 @@ class Adaptor(SelectorsGeneration):
                     text='', body=b'',  # Since root argument is provided, both `text` and `body` will be ignored so this is just a filler
                     url=self.url, encoding=self.encoding, auto_match=self.__auto_match_enabled,
                     keep_comments=self.__keep_comments, keep_cdata=self.__keep_cdata,
-                    huge_tree=self.__huge_tree_enabled, debug=self.__debug,
+                    huge_tree=self.__huge_tree_enabled,
                     **self.__response_data
                 )
             return element
@@ -417,10 +413,10 @@ class Adaptor(SelectorsGeneration):
         if score_table:
             highest_probability = max(score_table.keys())
             if score_table[highest_probability] and highest_probability >= percentage:
-                logging.debug(f'Highest probability was {highest_probability}%')
+                log.debug(f'Highest probability was {highest_probability}%')
-                logging.debug('Top 5 best matching elements are: ')
+                log.debug('Top 5 best matching elements are: ')
                 for percent in tuple(sorted(score_table.keys(), reverse=True))[:5]:
-                    logging.debug(f'{percent} -> {self.__convert_results(score_table[percent])}')
+                    log.debug(f'{percent} -> {self.__convert_results(score_table[percent])}')
                 if not adaptor_type:
                     return score_table[highest_probability]
                 return self.__convert_results(score_table[highest_probability])
@@ -546,7 +542,7 @@ class Adaptor(SelectorsGeneration):
 
             if selected_elements:
                 if not self.__auto_match_enabled and auto_save:
-                    logging.warning("Argument `auto_save` will be ignored because `auto_match` wasn't enabled on initialization. Check docs for more info.")
+                    log.warning("Argument `auto_save` will be ignored because `auto_match` wasn't enabled on initialization. Check docs for more info.")
 
                 elif self.__auto_match_enabled and auto_save:
                     self.save(selected_elements[0], identifier or selector)
@@ -565,7 +561,7 @@ class Adaptor(SelectorsGeneration):
                         return self.__convert_results(selected_elements)
 
                 elif not self.__auto_match_enabled and auto_match:
-                    logging.warning("Argument `auto_match` will be ignored because `auto_match` wasn't enabled on initialization. Check docs for more info.")
+                    log.warning("Argument `auto_match` will be ignored because `auto_match` wasn't enabled on initialization. Check docs for more info.")
 
                 return self.__convert_results(selected_elements)
 
@@ -769,7 +765,7 @@ class Adaptor(SelectorsGeneration):
 
             self._storage.save(element, identifier)
         else:
-            logging.critical(
+            log.critical(
                 "Can't use Auto-match features with disabled globally, you have to start a new class instance."
             )
 
@@ -783,7 +779,7 @@ class Adaptor(SelectorsGeneration):
         if self.__auto_match_enabled:
             return self._storage.retrieve(identifier)
 
-        logging.critical(
+        log.critical(
             "Can't use Auto-match features with disabled globally, you have to start a new class instance."
         )
 

tests/parser/test_automatch.py

@@ -42,8 +42,8 @@ class TestParserAutoMatch(unittest.TestCase):
                 </div>
                 '''
 
-        old_page = Adaptor(original_html, url='example.com', auto_match=True, debug=True)
+        old_page = Adaptor(original_html, url='example.com', auto_match=True)
-        new_page = Adaptor(changed_html, url='example.com', auto_match=True, debug=True)
+        new_page = Adaptor(changed_html, url='example.com', auto_match=True)
 
         # 'p1' was used as ID and now it's not and all the path elements have changes
         # Also at the same time testing auto-match vs combined selectors

tests/parser/test_general.py

@@ -74,7 +74,7 @@ class TestParser(unittest.TestCase):
         </body>
         </html>
         '''
-        self.page = Adaptor(self.html, auto_match=False, debug=False)
+        self.page = Adaptor(self.html, auto_match=False)
 
     def test_css_selector(self):
         """Test Selecting elements with complex CSS selectors"""
@@ -273,7 +273,7 @@ class TestParser(unittest.TestCase):
         large_html = '<html><body>' + '<div class="item">' * 5000 + '</div>' * 5000 + '</body></html>'
 
         start_time = time.time()
-        parsed = Adaptor(large_html, auto_match=False, debug=False)
+        parsed = Adaptor(large_html, auto_match=False)
         elements = parsed.css('.item')
         end_time = time.time()