From 49b7ae13f68e3b0bd9cb3d469c7eeff3f9828e5c Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Tue, 22 Apr 2025 05:15:57 +0200 Subject: [PATCH] feat(cli): Adding Scrapling Shell feature --- scrapling/cli.py | 29 +++++++ scrapling/core/shell.py | 175 ++++++++++++++++++++++++++++++++++++++++ setup.py | 1 + 3 files changed, 205 insertions(+) create mode 100644 scrapling/core/shell.py diff --git a/scrapling/cli.py b/scrapling/cli.py index 5b8e988..d1d4aa5 100644 --- a/scrapling/cli.py +++ b/scrapling/cli.py @@ -5,6 +5,8 @@ from pathlib import Path import click +from scrapling.core.shell import CustomShell + def get_package_dir(): return Path(os.path.dirname(__file__)) @@ -49,6 +51,32 @@ def install(force): print("The dependencies are already installed") +@click.command(help="Interactive scraping console") +@click.option( + "-c", + "--code", + "code", + is_flag=False, + default="", + type=str, + help="Evaluate the code in the shell, print the result and exit", +) +@click.option( + "-L", + "--loglevel", + "level", + is_flag=False, + default="debug", + type=click.Choice( + ["debug", "info", "warning", "error", "critical", "fatal"], case_sensitive=False + ), + help="Log level (default: DEBUG)", +) +def shell(code, level): + console = CustomShell(code=code, log_level=level) + console.start() + + @click.group() def main(): pass @@ -56,3 +84,4 @@ def main(): # Adding commands main.add_command(install) +main.add_command(shell) diff --git a/scrapling/core/shell.py b/scrapling/core/shell.py new file mode 100644 index 0000000..c3961e4 --- /dev/null +++ b/scrapling/core/shell.py @@ -0,0 +1,175 @@ +import os +import logging +import tempfile +import webbrowser +from functools import wraps + +from IPython.terminal.embed import InteractiveShellEmbed + +from scrapling import __version__ +from scrapling.core.utils import log +from scrapling.parser import Adaptor, Adaptors +from scrapling.fetchers import Fetcher, AsyncFetcher, PlayWrightFetcher, StealthyFetcher + + +_known_logging_levels = { + "debug": logging.DEBUG, + "info": logging.INFO, + "warning": logging.WARNING, + "error": logging.ERROR, + "critical": logging.CRITICAL, + "fatal": logging.FATAL, +} + + +def show_page_in_browser(page): + if not page: + log.error("Input must be of type `Adaptor`") + return + + fd, fname = tempfile.mkstemp(".html") + os.write(fd, page.body.encode("utf-8")) + os.close(fd) + webbrowser.open(f"file://{fname}") + + +class CustomShell: + """A custom IPython shell with minimal dependencies""" + + def __init__(self, code, log_level="debug"): + self.code = code + self.page = None + self.pages = Adaptors([]) + log_level = log_level.strip().lower() + + if _known_logging_levels.get(log_level): + self.log_level = _known_logging_levels[log_level] + else: + log.error(f'Unknown log level "{log_level}", defaulting to "DEBUG"') + self.log_level = logging.DEBUG + + self.shell = None + + # Initialize your application components + self.init_components() + + def init_components(self): + """Initialize application components""" + # This is where you'd set up your application-specific objects + if self.log_level: + logging.getLogger("scrapling").setLevel(self.log_level) + + settings = Fetcher.display_config() + _ = settings.pop("storage") + _ = settings.pop("storage_args") + log.info(f"Scrapling {__version__} shell started") + log.info(f"Logging level is set to '{logging.getLevelName(self.log_level)}'") + log.info(f"Fetchers' parsing settings: {settings}") + + @staticmethod + def banner(): + """Create a custom banner for the shell""" + return f""" +-> Available Scrapling objects: + - Fetcher/AsyncFetcher + - PlayWrightFetcher + - StealthyFetcher + - Adaptor + +-> Useful shortcuts: + - {"get":<30} Shortcut for `Fetcher.get` + - {"post":<30} Shortcut for `Fetcher.post` + - {"put":<30} Shortcut for `Fetcher.put` + - {"delete":<30} Shortcut for `Fetcher.delete` + - {"fetch":<30} Shortcut for `PlayWrightFetcher.fetch` + - {"stealthy_fetch":<30} Shortcut for `StealthyFetcher.fetch` + +-> Useful commands + - {"page / response":<30} The response object of the last page you fetched + - {"pages":<30} Adaptors object of the last 5 response objects you fetched + - {"view(page)":<30} View page in a browser + - {"help()":<30} Show this help message (Shell help) + +Type 'exit' or press Ctrl+D to exit. + """ + + def update_page(self, result): + """Update current page and add to pages history""" + self.page = result + self.pages.append(result) + if len(self.pages) > 5: + self.pages.pop(0) # Remove oldest item + + # Update in IPython namespace too + if self.shell: + self.shell.user_ns["page"] = self.page + self.shell.user_ns["response"] = self.page + self.shell.user_ns["pages"] = self.pages + + return result + + def create_wrapper(self, func): + """Create a wrapper that preserves function signature but updates page""" + + @wraps(func) + def wrapper(*args, **kwargs): + result = func(*args, **kwargs) + return self.update_page(result) + + return wrapper + + def get_namespace(self): + """Create a namespace with application-specific objects""" + + # Create wrapped versions of fetch functions + get = self.create_wrapper(Fetcher.get) + post = self.create_wrapper(Fetcher.post) + put = self.create_wrapper(Fetcher.put) + delete = self.create_wrapper(Fetcher.delete) + dynamic_fetch = self.create_wrapper(PlayWrightFetcher.fetch) + stealthy_fetch = self.create_wrapper(StealthyFetcher.fetch) + + # Create the namespace dictionary + return { + "get": get, + "post": post, + "put": put, + "delete": delete, + "Fetcher": Fetcher, + "AsyncFetcher": AsyncFetcher, + "fetch": dynamic_fetch, + "PlayWrightFetcher": PlayWrightFetcher, + "stealthy_fetch": stealthy_fetch, + "StealthyFetcher": StealthyFetcher, + "Adaptor": Adaptor, + "page": self.page, + "response": self.page, + "pages": self.pages, + "view": show_page_in_browser, + "help": self.show_help, + } + + def show_help(self): + """Show help information""" + print(self.banner()) + + def start(self): + """Start the interactive shell""" + # Create the shell + ipython_shell = InteractiveShellEmbed(banner1=self.banner(), exit_msg="Bye Bye") + + # Store reference to the shell + self.shell = ipython_shell + + # Get our namespace with application objects + namespace = self.get_namespace() + + ipython_shell.user_ns.update(namespace) + # If a command was provided, execute it and exit + if self.code: + # Execute the command in the namespace + ipython_shell.run_cell(self.code, store_history=False) + return + + # Start the shell with our namespace + ipython_shell(local_ns=namespace) diff --git a/setup.py b/setup.py index ea1f644..72b6585 100644 --- a/setup.py +++ b/setup.py @@ -52,6 +52,7 @@ setup( install_requires=[ "lxml>=5.0", "cssselect>=1.2", + "IPython", "click", "w3lib", "orjson>=3",