feat(cli): Adding Scrapling Shell feature

scrapling/core/shell.py

@@ -0,0 +1,175 @@
+import os
+import logging
+import tempfile
+import webbrowser
+from functools import wraps
+
+from IPython.terminal.embed import InteractiveShellEmbed
+
+from scrapling import __version__
+from scrapling.core.utils import log
+from scrapling.parser import Adaptor, Adaptors
+from scrapling.fetchers import Fetcher, AsyncFetcher, PlayWrightFetcher, StealthyFetcher
+
+
+_known_logging_levels = {
+    "debug": logging.DEBUG,
+    "info": logging.INFO,
+    "warning": logging.WARNING,
+    "error": logging.ERROR,
+    "critical": logging.CRITICAL,
+    "fatal": logging.FATAL,
+}
+
+
+def show_page_in_browser(page):
+    if not page:
+        log.error("Input must be of type `Adaptor`")
+        return
+
+    fd, fname = tempfile.mkstemp(".html")
+    os.write(fd, page.body.encode("utf-8"))
+    os.close(fd)
+    webbrowser.open(f"file://{fname}")
+
+
+class CustomShell:
+    """A custom IPython shell with minimal dependencies"""
+
+    def __init__(self, code, log_level="debug"):
+        self.code = code
+        self.page = None
+        self.pages = Adaptors([])
+        log_level = log_level.strip().lower()
+
+        if _known_logging_levels.get(log_level):
+            self.log_level = _known_logging_levels[log_level]
+        else:
+            log.error(f'Unknown log level "{log_level}", defaulting to "DEBUG"')
+            self.log_level = logging.DEBUG
+
+        self.shell = None
+
+        # Initialize your application components
+        self.init_components()
+
+    def init_components(self):
+        """Initialize application components"""
+        # This is where you'd set up your application-specific objects
+        if self.log_level:
+            logging.getLogger("scrapling").setLevel(self.log_level)
+
+        settings = Fetcher.display_config()
+        _ = settings.pop("storage")
+        _ = settings.pop("storage_args")
+        log.info(f"Scrapling {__version__} shell started")
+        log.info(f"Logging level is set to '{logging.getLevelName(self.log_level)}'")
+        log.info(f"Fetchers' parsing settings: {settings}")
+
+    @staticmethod
+    def banner():
+        """Create a custom banner for the shell"""
+        return f"""
+-> Available Scrapling objects:
+   - Fetcher/AsyncFetcher
+   - PlayWrightFetcher
+   - StealthyFetcher
+   - Adaptor
+
+-> Useful shortcuts:
+   - {"get":<30} Shortcut for `Fetcher.get`
+   - {"post":<30} Shortcut for `Fetcher.post`
+   - {"put":<30} Shortcut for `Fetcher.put`
+   - {"delete":<30} Shortcut for `Fetcher.delete`
+   - {"fetch":<30} Shortcut for `PlayWrightFetcher.fetch`
+   - {"stealthy_fetch":<30} Shortcut for `StealthyFetcher.fetch`
+
+-> Useful commands
+   - {"page / response":<30} The response object of the last page you fetched
+   - {"pages":<30} Adaptors object of the last 5 response objects you fetched
+   - {"view(page)":<30} View page in a browser
+   - {"help()":<30} Show this help message (Shell help)
+
+Type 'exit' or press Ctrl+D to exit.
+        """
+
+    def update_page(self, result):
+        """Update current page and add to pages history"""
+        self.page = result
+        self.pages.append(result)
+        if len(self.pages) > 5:
+            self.pages.pop(0)  # Remove oldest item
+
+        # Update in IPython namespace too
+        if self.shell:
+            self.shell.user_ns["page"] = self.page
+            self.shell.user_ns["response"] = self.page
+            self.shell.user_ns["pages"] = self.pages
+
+        return result
+
+    def create_wrapper(self, func):
+        """Create a wrapper that preserves function signature but updates page"""
+
+        @wraps(func)
+        def wrapper(*args, **kwargs):
+            result = func(*args, **kwargs)
+            return self.update_page(result)
+
+        return wrapper
+
+    def get_namespace(self):
+        """Create a namespace with application-specific objects"""
+
+        # Create wrapped versions of fetch functions
+        get = self.create_wrapper(Fetcher.get)
+        post = self.create_wrapper(Fetcher.post)
+        put = self.create_wrapper(Fetcher.put)
+        delete = self.create_wrapper(Fetcher.delete)
+        dynamic_fetch = self.create_wrapper(PlayWrightFetcher.fetch)
+        stealthy_fetch = self.create_wrapper(StealthyFetcher.fetch)
+
+        # Create the namespace dictionary
+        return {
+            "get": get,
+            "post": post,
+            "put": put,
+            "delete": delete,
+            "Fetcher": Fetcher,
+            "AsyncFetcher": AsyncFetcher,
+            "fetch": dynamic_fetch,
+            "PlayWrightFetcher": PlayWrightFetcher,
+            "stealthy_fetch": stealthy_fetch,
+            "StealthyFetcher": StealthyFetcher,
+            "Adaptor": Adaptor,
+            "page": self.page,
+            "response": self.page,
+            "pages": self.pages,
+            "view": show_page_in_browser,
+            "help": self.show_help,
+        }
+
+    def show_help(self):
+        """Show help information"""
+        print(self.banner())
+
+    def start(self):
+        """Start the interactive shell"""
+        # Create the shell
+        ipython_shell = InteractiveShellEmbed(banner1=self.banner(), exit_msg="Bye Bye")
+
+        # Store reference to the shell
+        self.shell = ipython_shell
+
+        # Get our namespace with application objects
+        namespace = self.get_namespace()
+
+        ipython_shell.user_ns.update(namespace)
+        # If a command was provided, execute it and exit
+        if self.code:
+            # Execute the command in the namespace
+            ipython_shell.run_cell(self.code, store_history=False)
+            return
+
+        # Start the shell with our namespace
+        ipython_shell(local_ns=namespace)