From 1c9e48b1c6ebf19d55130e1b9fed0b228bc0203c Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Sun, 29 Jun 2025 18:56:09 +0300 Subject: [PATCH] feat(extract): Adding new command to CLI options + Optimizations Users can now fetch websites directly without code and extract full/selected HTML content as HTML, Markdown, or extract text content. --- scrapling/cli.py | 794 +++++++++++++++++++++++++++++++++++++++- scrapling/core/shell.py | 72 +++- 2 files changed, 846 insertions(+), 20 deletions(-) diff --git a/scrapling/cli.py b/scrapling/cli.py index 00e7c3f..9ea0ed0 100644 --- a/scrapling/cli.py +++ b/scrapling/cli.py @@ -1,21 +1,75 @@ -import os -import sys -import subprocess from pathlib import Path +from subprocess import check_output +from sys import executable as python_executable -from click import command, option, Choice, group +from scrapling.core.utils import log +from scrapling.core.shell import Convertor, _CookieParser +from scrapling.fetchers import Fetcher, DynamicFetcher, StealthyFetcher + +from orjson import loads as json_loads, JSONDecodeError +from click import command, option, Choice, group, argument + +__OUTPUT_FILE_HELP__ = "Output file path can be HTML content, Markdown of the HTML content, or the text content. Use file extensions (`.html`/`.md`/`.txt`) respectively." def get_package_dir(): - return Path(os.path.dirname(__file__)) + return Path(__file__).parent def run_command(cmd, line): print(f"Installing {line}...") - _ = subprocess.check_call(cmd, shell=False) # nosec B603 + _ = check_output(cmd, shell=False) # nosec B603 # I meant to not use try except here +def parse_headers(header_strings): + """Parse header strings into a dictionary""" + headers = {} + for header in header_strings: + if ":" in header: + key, value = header.split(":", 1) + headers[key.strip()] = value.strip() + else: + log.warning(f"Invalid header format '{header}', should be 'Key: Value'") + return headers + + +def parse_cookies(cookie_string): + """Parse cookie string into a dictionary""" + if not cookie_string: + return {} + + try: + cookies = {key: value for key, value in _CookieParser(cookie_string)} + except Exception as e: + raise ValueError(f"Could not parse cookies '{cookie_string}': {e}") + + return cookies + + +def parse_json_data(json_string): + """Parse JSON string into a Python object""" + if not json_string: + return None + + try: + return json_loads(json_string) + except JSONDecodeError as e: + raise ValueError(f"Invalid JSON data '{json_string}': {e}") + + +def make_request_and_save(fetcher_func, url, output_file, css_selector=None, **kwargs): + """Make a request using the specified fetcher function and save the result""" + # Handle relative paths - convert to an absolute path based on the current working directory + output_path = Path(output_file) + if not output_path.is_absolute(): + output_path = Path.cwd() / output_file + + response = fetcher_func(url, **kwargs) + Convertor.write_content_to_file(response, str(output_path), css_selector) + log.info(f"Content successfully saved to '{output_path}'") + + @command(help="Install all Scrapling's Fetchers dependencies") @option( "-f", @@ -32,15 +86,22 @@ def install(force): or not get_package_dir().joinpath(".scrapling_dependencies_installed").exists() ): run_command( - [sys.executable, "-m", "playwright", "install", "chromium"], + [python_executable, "-m", "playwright", "install", "chromium"], "Playwright browsers", ) run_command( - [sys.executable, "-m", "playwright", "install-deps", "chromium", "firefox"], + [ + python_executable, + "-m", + "playwright", + "install-deps", + "chromium", + "firefox", + ], "Playwright dependencies", ) run_command( - [sys.executable, "-m", "camoufox", "fetch", "--browserforge"], + [python_executable, "-m", "camoufox", "fetch", "--browserforge"], "Camoufox browser and databases", ) # if no errors raised by the above commands, then we add the below file @@ -77,6 +138,720 @@ def shell(code, level): console.start() +def parse_extract_arguments(headers, cookies, params, json=None): + """Parse arguments for extract command""" + parsed_headers = parse_headers(headers) + parsed_cookies = parse_cookies(cookies) + parsed_json = parse_json_data(json) + parsed_params = {} + for param in params: + if "=" in param: + key, value = param.split("=", 1) + parsed_params[key] = value + + return parsed_headers, parsed_cookies, parsed_params, parsed_json + + +@group( + help="Fetch web pages using various fetchers and extract full/selected HTML content as HTML, Markdown, or extract text content." +) +def extract(): + """Extract content from web pages and save to files""" + pass + + +@extract.command( + help=f"Perform a GET request and save content to file.\n\n{__OUTPUT_FILE_HELP__}" +) +@argument("url", required=True) +@argument("output_file", required=True) +@option( + "--headers", + "-H", + multiple=True, + help='HTTP headers in format "Key: Value" (can be used multiple times)', +) +@option("--cookies", help='Cookies string in format "name1=value1; name2=value2"') +@option( + "--timeout", type=int, default=30, help="Request timeout in seconds (default: 30)" +) +@option("--proxy", help='Proxy URL in format "http://username:password@host:port"') +@option( + "--css-selector", + "-s", + help="CSS selector to extract specific content from the page. It resolves to the first match if multiple matches are found.", +) +@option( + "--params", + "-p", + multiple=True, + help='Query parameters in format "key=value" (can be used multiple times)', +) +@option( + "--follow-redirects/--no-follow-redirects", + default=True, + help="Whether to follow redirects (default: True)", +) +@option( + "--verify/--no-verify", + default=True, + help="Whether to verify SSL certificates (default: True)", +) +@option("--impersonate", help="Browser to impersonate (e.g., chrome, firefox).") +@option( + "--stealthy-headers/--no-stealthy-headers", + default=True, + help="Use stealthy browser headers (default: True)", +) +def get( + url, + output_file, + headers, + cookies, + timeout, + proxy, + css_selector, + params, + follow_redirects, + verify, + impersonate, + stealthy_headers, +): + """ + Perform a GET request and save content to file. + + :param url: Target URL for the request. + :param output_file: Output file path (.md for Markdown, .html for HTML). + :param headers: HTTP headers to include in the request. + :param cookies: Cookies to use in the request. + :param timeout: Number of seconds to wait before timing out. + :param proxy: Proxy URL to use. (Format: "http://username:password@localhost:8030") + :param css_selector: CSS selector to extract specific content. + :param params: Query string parameters for the request. + :param follow_redirects: Whether to follow redirects. + :param verify: Whether to verify HTTPS certificates. + :param impersonate: Browser version to impersonate. + :param stealthy_headers: If enabled, creates and adds real browser headers. + """ + + # Parse parameters + parsed_headers, parsed_cookies, parsed_params, _ = parse_extract_arguments( + headers, cookies, params + ) + + # Build request arguments + kwargs = { + "headers": parsed_headers if parsed_headers else None, + "cookies": parsed_cookies if parsed_cookies else None, + "timeout": timeout, + "follow_redirects": follow_redirects, + "verify": verify, + "stealthy_headers": stealthy_headers, + "impersonate": impersonate, + } + + if parsed_params: + kwargs["params"] = parsed_params + if proxy: + kwargs["proxy"] = proxy + + make_request_and_save(Fetcher.get, url, output_file, css_selector, **kwargs) + + +@extract.command( + help=f"Perform a POST request and save content to file.\n\n{__OUTPUT_FILE_HELP__}" +) +@argument("url", required=True) +@argument("output_file", required=True) +@option( + "--data", + "-d", + help='Form data to include in the request body (as string, ex: "param1=value1¶m2=value2")', +) +@option("--json", "-j", help="JSON data to include in the request body (as string)") +@option( + "--headers", + "-H", + multiple=True, + help='HTTP headers in format "Key: Value" (can be used multiple times)', +) +@option("--cookies", help='Cookies string in format "name1=value1; name2=value2"') +@option( + "--timeout", type=int, default=30, help="Request timeout in seconds (default: 30)" +) +@option("--proxy", help='Proxy URL in format "http://username:password@host:port"') +@option( + "--css-selector", + "-s", + help="CSS selector to extract specific content from the page", +) +@option( + "--params", + "-p", + multiple=True, + help='Query parameters in format "key=value" (can be used multiple times)', +) +@option( + "--follow-redirects/--no-follow-redirects", + default=True, + help="Whether to follow redirects (default: True)", +) +@option( + "--verify/--no-verify", + default=True, + help="Whether to verify SSL certificates (default: True)", +) +@option("--impersonate", help="Browser to impersonate (e.g., chrome, firefox).") +@option( + "--stealthy-headers/--no-stealthy-headers", + default=True, + help="Use stealthy browser headers (default: True)", +) +def post( + url, + output_file, + data, + json, + headers, + cookies, + timeout, + proxy, + css_selector, + params, + follow_redirects, + verify, + impersonate, + stealthy_headers, +): + """ + Perform a POST request and save content to file. + + :param url: Target URL for the request. + :param output_file: Output file path (.md for Markdown, .html for HTML). + :param data: Form data to include in the request body. (as string, ex: "param1=value1¶m2=value2") + :param json: A JSON serializable object to include in the body of the request. + :param headers: Headers to include in the request. + :param cookies: Cookies to use in the request. + :param timeout: Number of seconds to wait before timing out. + :param proxy: Proxy URL to use. + :param css_selector: CSS selector to extract specific content. + :param params: Query string parameters for the request. + :param follow_redirects: Whether to follow redirects. + :param verify: Whether to verify HTTPS certificates. + :param impersonate: Browser version to impersonate. + :param stealthy_headers: If enabled, creates and adds real browser headers. + """ + + # Parse parameters + parsed_headers, parsed_cookies, parsed_params, parsed_json = ( + parse_extract_arguments(headers, cookies, params, json) + ) + + # Build request arguments + kwargs = { + "headers": parsed_headers if parsed_headers else None, + "cookies": parsed_cookies if parsed_cookies else None, + "timeout": timeout, + "follow_redirects": follow_redirects, + "verify": verify, + "stealthy_headers": stealthy_headers, + "impersonate": impersonate, + } + + if data: + kwargs["data"] = data + if parsed_json: + kwargs["json"] = parsed_json + if parsed_params: + kwargs["params"] = parsed_params + if proxy: + kwargs["proxy"] = proxy + + make_request_and_save(Fetcher.post, url, output_file, css_selector, **kwargs) + + +@extract.command( + help=f"Perform a PUT request and save content to file.\n\n{__OUTPUT_FILE_HELP__}" +) +@argument("url", required=True) +@argument("output_file", required=True) +@option("--data", "-d", help="Form data to include in the request body") +@option("--json", "-j", help="JSON data to include in the request body (as string)") +@option( + "--headers", + "-H", + multiple=True, + help='HTTP headers in format "Key: Value" (can be used multiple times)', +) +@option("--cookies", help='Cookies string in format "name1=value1; name2=value2"') +@option( + "--timeout", type=int, default=30, help="Request timeout in seconds (default: 30)" +) +@option("--proxy", help='Proxy URL in format "http://username:password@host:port"') +@option( + "--css-selector", + "-s", + help="CSS selector to extract specific content from the page", +) +@option( + "--params", + "-p", + multiple=True, + help='Query parameters in format "key=value" (can be used multiple times)', +) +@option( + "--follow-redirects/--no-follow-redirects", + default=True, + help="Whether to follow redirects (default: True)", +) +@option( + "--verify/--no-verify", + default=True, + help="Whether to verify SSL certificates (default: True)", +) +@option("--impersonate", help="Browser to impersonate (e.g., chrome, firefox).") +@option( + "--stealthy-headers/--no-stealthy-headers", + default=True, + help="Use stealthy browser headers (default: True)", +) +def put( + url, + output_file, + data, + json, + headers, + cookies, + timeout, + proxy, + css_selector, + params, + follow_redirects, + verify, + impersonate, + stealthy_headers, +): + """ + Perform a PUT request and save content to file. + + :param url: Target URL for the request. + :param output_file: Output file path (.md for Markdown, .html for HTML). + :param data: Form data to include in the request body. + :param json: A JSON serializable object to include in the body of the request. + :param headers: Headers to include in the request. + :param cookies: Cookies to use in the request. + :param timeout: Number of seconds to wait before timing out. + :param proxy: Proxy URL to use. + :param css_selector: CSS selector to extract specific content. + :param params: Query string parameters for the request. + :param follow_redirects: Whether to follow redirects. + :param verify: Whether to verify HTTPS certificates. + :param impersonate: Browser version to impersonate. + :param stealthy_headers: If enabled, creates and adds real browser headers. + """ + + # Parse parameters + parsed_headers, parsed_cookies, parsed_params, parsed_json = ( + parse_extract_arguments(headers, cookies, params, json) + ) + + # Build request arguments + kwargs = { + "headers": parsed_headers if parsed_headers else None, + "cookies": parsed_cookies if parsed_cookies else None, + "timeout": timeout, + "follow_redirects": follow_redirects, + "verify": verify, + "stealthy_headers": stealthy_headers, + "impersonate": impersonate, + } + + if data: + kwargs["data"] = data + if parsed_json: + kwargs["json"] = parsed_json + if parsed_params: + kwargs["params"] = parsed_params + if proxy: + kwargs["proxy"] = proxy + + make_request_and_save(Fetcher.put, url, output_file, css_selector, **kwargs) + + +@extract.command( + help=f"Perform a DELETE request and save content to file.\n\n{__OUTPUT_FILE_HELP__}" +) +@argument("url", required=True) +@argument("output_file", required=True) +@option( + "--headers", + "-H", + multiple=True, + help='HTTP headers in format "Key: Value" (can be used multiple times)', +) +@option("--cookies", help='Cookies string in format "name1=value1; name2=value2"') +@option( + "--timeout", type=int, default=30, help="Request timeout in seconds (default: 30)" +) +@option("--proxy", help='Proxy URL in format "http://username:password@host:port"') +@option( + "--css-selector", + "-s", + help="CSS selector to extract specific content from the page", +) +@option( + "--params", + "-p", + multiple=True, + help='Query parameters in format "key=value" (can be used multiple times)', +) +@option( + "--follow-redirects/--no-follow-redirects", + default=True, + help="Whether to follow redirects (default: True)", +) +@option( + "--verify/--no-verify", + default=True, + help="Whether to verify SSL certificates (default: True)", +) +@option("--impersonate", help="Browser to impersonate (e.g., chrome, firefox).") +@option( + "--stealthy-headers/--no-stealthy-headers", + default=True, + help="Use stealthy browser headers (default: True)", +) +def delete( + url, + output_file, + headers, + cookies, + timeout, + proxy, + css_selector, + params, + follow_redirects, + verify, + impersonate, + stealthy_headers, +): + """ + Perform a DELETE request and save content to file. + + :param url: Target URL for the request. + :param output_file: Output file path (.md for Markdown, .html for HTML). + :param headers: Headers to include in the request. + :param cookies: Cookies to use in the request. + :param timeout: Number of seconds to wait before timing out. + :param proxy: Proxy URL to use. + :param css_selector: CSS selector to extract specific content. + :param params: Query string parameters for the request. + :param follow_redirects: Whether to follow redirects. + :param verify: Whether to verify HTTPS certificates. + :param impersonate: Browser version to impersonate. + :param stealthy_headers: If enabled, creates and adds real browser headers. + """ + + # Parse parameters + parsed_headers, parsed_cookies, parsed_params, _ = parse_extract_arguments( + headers, cookies, params + ) + + # Build request arguments + kwargs = { + "headers": parsed_headers if parsed_headers else None, + "cookies": parsed_cookies if parsed_cookies else None, + "timeout": timeout, + "follow_redirects": follow_redirects, + "verify": verify, + "stealthy_headers": stealthy_headers, + "impersonate": impersonate, + } + + if parsed_params: + kwargs["params"] = parsed_params + if proxy: + kwargs["proxy"] = proxy + + make_request_and_save(Fetcher.delete, url, output_file, css_selector, **kwargs) + + +@extract.command( + help=f"Use DynamicFetcher to fetch content with browser automation.\n\n{__OUTPUT_FILE_HELP__}" +) +@argument("url", required=True) +@argument("output_file", required=True) +@option( + "--headless/--no-headless", + default=True, + help="Run browser in headless mode (default: True)", +) +@option( + "--disable-resources/--enable-resources", + default=False, + help="Drop unnecessary resources for speed boost (default: False)", +) +@option( + "--network-idle/--no-network-idle", + default=False, + help="Wait for network idle (default: False)", +) +@option( + "--timeout", + type=int, + default=30000, + help="Timeout in milliseconds (default: 30000)", +) +@option( + "--wait", + type=int, + default=0, + help="Additional wait time in milliseconds after page load (default: 0)", +) +@option( + "--css-selector", + "-s", + help="CSS selector to extract specific content from the page", +) +@option("--wait-selector", help="CSS selector to wait for before proceeding") +@option("--locale", default="en-US", help="Browser locale (default: en-US)") +@option( + "--stealth/--no-stealth", default=False, help="Enable stealth mode (default: False)" +) +@option( + "--hide-canvas/--show-canvas", + default=False, + help="Add noise to canvas operations (default: False)", +) +@option( + "--disable-webgl/--enable-webgl", + default=False, + help="Disable WebGL support (default: False)", +) +@option("--proxy", help='Proxy URL in format "http://username:password@host:port"') +@option( + "--extra-headers", + "-H", + multiple=True, + help='Extra headers in format "Key: Value" (can be used multiple times)', +) +def fetch( + url, + output_file, + headless, + disable_resources, + network_idle, + timeout, + wait, + css_selector, + wait_selector, + locale, + stealth, + hide_canvas, + disable_webgl, + proxy, + extra_headers, +): + """ + Opens up a browser and fetch content using DynamicFetcher. + + :param url: Target url. + :param output_file: Output file path (.md for Markdown, .html for HTML). + :param headless: Run the browser in headless/hidden or headful/visible mode. + :param disable_resources: Drop requests of unnecessary resources for a speed boost. + :param network_idle: Wait for the page until there are no network connections for at least 500 ms. + :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. + :param wait: The time (milliseconds) the fetcher will wait after everything finishes before returning. + :param css_selector: CSS selector to extract specific content. + :param wait_selector: Wait for a specific CSS selector to be in a specific state. + :param locale: Set the locale for the browser. + :param stealth: Enables stealth mode. + :param hide_canvas: Add random noise to canvas operations to prevent fingerprinting. + :param disable_webgl: Disables WebGL and WebGL 2.0 support entirely. + :param proxy: The proxy to be used with requests. + :param extra_headers: Extra headers to add to the request. + """ + + # Parse parameters + parsed_headers = parse_headers(extra_headers) + + # Build request arguments + kwargs = { + "headless": headless, + "disable_resources": disable_resources, + "network_idle": network_idle, + "timeout": timeout, + "locale": locale, + "stealth": stealth, + "hide_canvas": hide_canvas, + "disable_webgl": disable_webgl, + } + + if wait > 0: + kwargs["wait"] = wait + if wait_selector: + kwargs["wait_selector"] = wait_selector + if proxy: + kwargs["proxy"] = proxy + if parsed_headers: + kwargs["extra_headers"] = parsed_headers + + make_request_and_save( + DynamicFetcher.fetch, url, output_file, css_selector, **kwargs + ) + + +@extract.command( + help=f"Use StealthyFetcher to fetch content with advanced stealth features.\n\n{__OUTPUT_FILE_HELP__}" +) +@argument("url", required=True) +@argument("output_file", required=True) +@option( + "--headless/--no-headless", + default=True, + help="Run browser in headless mode (default: True)", +) +@option( + "--block-images/--allow-images", + default=False, + help="Block image loading (default: False)", +) +@option( + "--disable-resources/--enable-resources", + default=False, + help="Drop unnecessary resources for speed boost (default: False)", +) +@option( + "--block-webrtc/--allow-webrtc", + default=False, + help="Block WebRTC entirely (default: False)", +) +@option( + "--humanize/--no-humanize", + default=False, + help="Humanize cursor movement (default: False)", +) +@option( + "--solve-cloudflare/--no-solve-cloudflare", + default=False, + help="Solve Cloudflare challenges (default: False)", +) +@option("--allow-webgl/--block-webgl", default=True, help="Allow WebGL (default: True)") +@option( + "--network-idle/--no-network-idle", + default=False, + help="Wait for network idle (default: False)", +) +@option( + "--disable-ads/--allow-ads", + default=False, + help="Install uBlock Origin addon (default: False)", +) +@option( + "--timeout", + type=int, + default=30000, + help="Timeout in milliseconds (default: 30000)", +) +@option( + "--wait", + type=int, + default=0, + help="Additional wait time in milliseconds after page load (default: 0)", +) +@option( + "--css-selector", + "-s", + help="CSS selector to extract specific content from the page", +) +@option("--wait-selector", help="CSS selector to wait for before proceeding") +@option( + "--geoip/--no-geoip", + default=False, + help="Use IP geolocation for timezone/locale (default: False)", +) +@option("--proxy", help='Proxy URL in format "http://username:password@host:port"') +@option( + "--extra-headers", + "-H", + multiple=True, + help='Extra headers in format "Key: Value" (can be used multiple times)', +) +def stealthy_fetch( + url, + output_file, + headless, + block_images, + disable_resources, + block_webrtc, + humanize, + solve_cloudflare, + allow_webgl, + network_idle, + disable_ads, + timeout, + wait, + css_selector, + wait_selector, + geoip, + proxy, + extra_headers, +): + """ + Opens up a browser with advanced stealth features and fetch content using StealthyFetcher. + + :param url: Target url. + :param output_file: Output file path (.md for Markdown, .html for HTML). + :param headless: Run the browser in headless/hidden, virtual screen mode, or headful/visible mode. + :param block_images: Prevent the loading of images through Firefox preferences. + :param disable_resources: Drop requests of unnecessary resources for a speed boost. + :param block_webrtc: Blocks WebRTC entirely. + :param humanize: Humanize the cursor movement. + :param solve_cloudflare: Solves all 3 types of the Cloudflare's Turnstile wait page. + :param allow_webgl: Allow WebGL (recommended to keep enabled). + :param network_idle: Wait for the page until there are no network connections for at least 500 ms. + :param disable_ads: Install the uBlock Origin addon on the browser. + :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. + :param wait: The time (milliseconds) the fetcher will wait after everything finishes before returning. + :param css_selector: CSS selector to extract specific content. + :param wait_selector: Wait for a specific CSS selector to be in a specific state. + :param geoip: Automatically use IP's longitude, latitude, timezone, country, locale. + :param proxy: The proxy to be used with requests. + :param extra_headers: Extra headers to add to the request. + """ + + # Parse parameters + parsed_headers = parse_headers(extra_headers) + + # Build request arguments + kwargs = { + "headless": headless, + "block_images": block_images, + "disable_resources": disable_resources, + "block_webrtc": block_webrtc, + "humanize": humanize, + "solve_cloudflare": solve_cloudflare, + "allow_webgl": allow_webgl, + "network_idle": network_idle, + "disable_ads": disable_ads, + "timeout": timeout, + "geoip": geoip, + } + + if wait > 0: + kwargs["wait"] = wait + if wait_selector: + kwargs["wait_selector"] = wait_selector + if proxy: + kwargs["proxy"] = proxy + if parsed_headers: + kwargs["extra_headers"] = parsed_headers + + make_request_and_save( + StealthyFetcher.fetch, url, output_file, css_selector, **kwargs + ) + + @group() def main(): pass @@ -85,3 +860,4 @@ def main(): # Adding commands main.add_command(install) main.add_command(shell) +main.add_command(extract) diff --git a/scrapling/core/shell.py b/scrapling/core/shell.py index b56a063..63ec304 100644 --- a/scrapling/core/shell.py +++ b/scrapling/core/shell.py @@ -1,4 +1,5 @@ # -*- coding: utf-8 -*- +from re import sub as re_sub from sys import stderr from functools import wraps from http import cookies as Cookie @@ -24,6 +25,7 @@ from IPython.terminal.embed import InteractiveShellEmbed from orjson import loads as json_loads, JSONDecodeError from scrapling import __version__ +from scrapling.core.custom_types import TextHandler from scrapling.core.utils import log from scrapling.parser import Adaptor, Adaptors from scrapling.core._types import List, Optional, Dict, Tuple, Any, Union @@ -63,6 +65,14 @@ Request = namedtuple( ) +def _CookieParser(cookie_string): + # Errors will be handled on call so the log can be specified + cookie_parser = Cookie.SimpleCookie() + cookie_parser.load(cookie_string) + for key, morsel in cookie_parser.items(): + yield key, morsel.value + + # Suppress exit on error to handle parsing errors gracefully class NoExitArgumentParser(ArgumentParser): def error(self, message): @@ -156,12 +166,11 @@ class CurlParser: if header_key.lower() == "cookie": try: - cookie_parser = Cookie.SimpleCookie() - cookie_parser.load(header_value) - for key, morsel in cookie_parser.items(): - cookie_dict[key] = morsel.value + cookie_dict = { + key: value for key, value in _CookieParser(header_value) + } except Exception as e: - log.error( + raise ValueError( f"Could not parse cookie string from -H '{header_value}': {e}" ) else: @@ -221,12 +230,9 @@ class CurlParser: if parsed_args.cookie: # We are focusing on the string format from DevTools. try: - cookie_parser = Cookie.SimpleCookie() - cookie_parser.load(parsed_args.cookie) - for key, morsel in cookie_parser.items(): - # Update the cookie dict, potentially overwriting - # cookies with the same name from -H 'Cookie:' - cookies[key] = morsel.value + for key, value in _CookieParser(parsed_args.cookie): + # Update the cookie dict, potentially overwriting cookies with the same name from -H 'cookie:' + cookies[key] = value log.debug(f"Parsed cookies from -b argument: {list(cookies.keys())}") except Exception as e: log.error( @@ -545,3 +551,47 @@ Type 'exit' or press Ctrl+D to exit. return ipython_shell() + + +class Convertor: + """Utils for the extract shell command""" + + @classmethod + def __convert_to_markdown(cls, body: TextHandler) -> str: + """Convert HTML content to Markdown""" + from markdownify import markdownify + + return markdownify(body) + + @classmethod + def write_content_to_file( + cls, page: Adaptor, filename: str, css_selector: Optional[str] = None + ) -> None: + """Write an Adaptor's content to a file""" + if not page or not isinstance(page, Adaptor): + raise TypeError("Input must be of type `Adaptor`") + elif not filename or not isinstance(filename, str) or not filename.strip(): + raise ValueError("Filename must be provided") + elif not filename.endswith((".md", ".html", ".txt")): + raise ValueError( + "Unknown file type: filename must end with '.md', '.html', or '.txt'" + ) + else: + body = page if not css_selector else page.css_first(css_selector) + with open(filename, "w", encoding="utf-8") as f: + if filename.endswith(".md"): + f.write(cls.__convert_to_markdown(body.body)) + elif filename.endswith(".html"): + f.write(body.body) + elif filename.endswith(".txt"): + txt_content = body.get_all_text(strip=True) + for s in ( + "\n", + "\r", + "\t", + " ", + ): + # Remove consecutive white-spaces + txt_content = re_sub(f"[{s}]+", s, txt_content) + + f.write(txt_content)