diff --git a/.bandit.yml b/.bandit.yml index 9d7a1f7..525749a 100644 --- a/.bandit.yml +++ b/.bandit.yml @@ -5,3 +5,5 @@ skips: - B410 - B113 # `Requests call without timeout` these requests are done in the benchmark and examples scripts only - B403 # We are using pickle for tests only +- B404 # Using subprocess library +- B602 # subprocess call with shell=True identified diff --git a/MANIFEST.in b/MANIFEST.in index b8fbc5b..8c168dd 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -4,7 +4,10 @@ include *.js include scrapling/engines/toolbelt/bypasses/*.js include scrapling/*.db include scrapling/*.db* +include scrapling/*.db-* include scrapling/py.typed +include scrapling/.scrapling_dependencies_installed +include .scrapling_dependencies_installed recursive-exclude * __pycache__ recursive-exclude * *.py[co] \ No newline at end of file diff --git a/scrapling/cli.py b/scrapling/cli.py new file mode 100644 index 0000000..7d22f0c --- /dev/null +++ b/scrapling/cli.py @@ -0,0 +1,37 @@ +import os +import subprocess +import sys +from pathlib import Path + +import click + + +def get_package_dir(): + return Path(os.path.dirname(__file__)) + + +def run_command(command, line): + print(f"Installing {line}...") + _ = subprocess.check_call(command, shell=True) + # I meant to not use try except here + + +@click.command(help="Install all Scrapling's Fetchers dependencies") +def install(): + if not get_package_dir().joinpath(".scrapling_dependencies_installed").exists(): + run_command([sys.executable, "-m", "playwright", "install", 'chromium'], 'Playwright browsers') + run_command([sys.executable, "-m", "playwright", "install-deps", 'chromium', 'firefox'], 'Playwright dependencies') + run_command([sys.executable, "-m", "camoufox", "fetch", '--browserforge'], 'Camoufox browser and databases') + # if no errors raised by above commands, then we add below file + get_package_dir().joinpath(".scrapling_dependencies_installed").touch() + else: + print('The dependencies are already installed') + + +@click.group() +def main(): + pass + + +# Adding commands +main.add_command(install) diff --git a/setup.py b/setup.py index 45b88ad..f092a64 100644 --- a/setup.py +++ b/setup.py @@ -20,6 +20,11 @@ setup( package_dir={ "scrapling": "scrapling", }, + entry_points={ + 'console_scripts': [ + 'scrapling=scrapling.cli:main' + ], + }, include_package_data=True, classifiers=[ "Operating System :: OS Independent", @@ -50,6 +55,7 @@ setup( "requests>=2.3", "lxml>=4.5", "cssselect>=1.2", + 'click', "w3lib", "orjson>=3", "tldextract",