From 4efbffa1dc3d636e17991f51082016453546ef79 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Mon, 30 Mar 2026 02:44:14 +0200 Subject: [PATCH] feat(cli): Add an option to make content safe/targets AI --- scrapling/cli.py | 39 ++++++++++++++++++++++++++++++--------- scrapling/core/shell.py | 5 ++++- 2 files changed, 34 insertions(+), 10 deletions(-) diff --git a/scrapling/cli.py b/scrapling/cli.py index 5a59e86..8ecee2d 100644 --- a/scrapling/cli.py +++ b/scrapling/cli.py @@ -42,6 +42,7 @@ def __Request_and_Save( url: str, output_file: str, css_selector: Optional[str] = None, + ai_targeted: bool = False, **kwargs, ) -> None: """Make a request using the specified fetcher function and save the result""" @@ -53,7 +54,7 @@ def __Request_and_Save( output_path = Path.cwd() / output_file response = fetcher_func(url, **kwargs) - Convertor.write_content_to_file(response, str(output_path), css_selector) + Convertor.write_content_to_file(response, str(output_path), css_selector, main_content_only=ai_targeted) log.info(f"Content successfully saved to '{output_path}'") @@ -202,6 +203,12 @@ def extract(): def _common_http_options(f): """Apply shared Click options for all HTTP extract commands (get/post/put/delete).""" decorators = [ + option( + "--ai-targeted", + is_flag=True, + default=False, + help="Extract only main content and sanitize hidden elements for AI consumption (default: False)", + ), option( "--stealthy-headers/--no-stealthy-headers", default=True, @@ -250,6 +257,12 @@ def _common_http_options(f): def _common_browser_options(f): """Apply shared Click options for browser-based commands (fetch/stealthy_fetch).""" decorators = [ + option( + "--ai-targeted", + is_flag=True, + default=False, + help="Extract only main content and sanitize hidden elements for AI consumption (default: False)", + ), option( "--extra-headers", "-H", @@ -317,11 +330,13 @@ def _data_options(f): return f -def __http_command(method_name: str, url: str, output_file: str, css_selector: Optional[str], **kwargs) -> None: +def __http_command( + method_name: str, url: str, output_file: str, css_selector: Optional[str], ai_targeted: bool = False, **kwargs +) -> None: """Shared implementation for HTTP extract commands.""" from scrapling.fetchers import Fetcher - __Request_and_Save(getattr(Fetcher, method_name), url, output_file, css_selector, **kwargs) + __Request_and_Save(getattr(Fetcher, method_name), url, output_file, css_selector, ai_targeted=ai_targeted, **kwargs) @extract.command(help=f"Perform a GET request and save the content to a file.\n\n{__OUTPUT_FILE_HELP__}") @@ -341,6 +356,7 @@ def get( verify, impersonate, stealthy_headers, + ai_targeted, ): """Perform a GET request and save the content to a file.""" kwargs = __BuildRequest( @@ -355,7 +371,7 @@ def get( impersonate=impersonate, proxy=proxy, ) - __http_command("get", url, output_file, css_selector, **kwargs) + __http_command("get", url, output_file, css_selector, ai_targeted=ai_targeted, **kwargs) @extract.command(help=f"Perform a POST request and save the content to a file.\n\n{__OUTPUT_FILE_HELP__}") @@ -378,6 +394,7 @@ def post( verify, impersonate, stealthy_headers, + ai_targeted, ): """Perform a POST request and save the content to a file.""" kwargs = __BuildRequest( @@ -393,7 +410,7 @@ def post( proxy=proxy, data=data, ) - __http_command("post", url, output_file, css_selector, **kwargs) + __http_command("post", url, output_file, css_selector, ai_targeted=ai_targeted, **kwargs) @extract.command(help=f"Perform a PUT request and save the content to a file.\n\n{__OUTPUT_FILE_HELP__}") @@ -416,6 +433,7 @@ def put( verify, impersonate, stealthy_headers, + ai_targeted, ): """Perform a PUT request and save the content to a file.""" kwargs = __BuildRequest( @@ -431,7 +449,7 @@ def put( proxy=proxy, data=data, ) - __http_command("put", url, output_file, css_selector, **kwargs) + __http_command("put", url, output_file, css_selector, ai_targeted=ai_targeted, **kwargs) @extract.command(help=f"Perform a DELETE request and save the content to a file.\n\n{__OUTPUT_FILE_HELP__}") @@ -451,6 +469,7 @@ def delete( verify, impersonate, stealthy_headers, + ai_targeted, ): """Perform a DELETE request and save the content to a file.""" kwargs = __BuildRequest( @@ -465,7 +484,7 @@ def delete( impersonate=impersonate, proxy=proxy, ) - __http_command("delete", url, output_file, css_selector, **kwargs) + __http_command("delete", url, output_file, css_selector, ai_targeted=ai_targeted, **kwargs) def __build_browser_kwargs( @@ -518,6 +537,7 @@ def fetch( real_chrome, proxy, extra_headers, + ai_targeted, ): """Opens up a browser and fetch content using DynamicFetcher.""" parsed_headers, _ = _ParseHeaders(extra_headers, False) @@ -535,7 +555,7 @@ def fetch( ) from scrapling.fetchers import DynamicFetcher - __Request_and_Save(DynamicFetcher.fetch, url, output_file, css_selector, **kwargs) + __Request_and_Save(DynamicFetcher.fetch, url, output_file, css_selector, ai_targeted=ai_targeted, **kwargs) @extract.command(help=f"Use StealthyFetcher to fetch content with advanced stealth features.\n\n{__OUTPUT_FILE_HELP__}") @@ -576,6 +596,7 @@ def stealthy_fetch( solve_cloudflare, allow_webgl, hide_canvas, + ai_targeted, ): """Opens up a browser with advanced stealth features and fetch content using StealthyFetcher.""" parsed_headers, _ = _ParseHeaders(extra_headers, False) @@ -601,7 +622,7 @@ def stealthy_fetch( ) from scrapling.fetchers import StealthyFetcher - __Request_and_Save(StealthyFetcher.fetch, url, output_file, css_selector, **kwargs) + __Request_and_Save(StealthyFetcher.fetch, url, output_file, css_selector, ai_targeted=ai_targeted, **kwargs) @group() diff --git a/scrapling/core/shell.py b/scrapling/core/shell.py index bed1c84..14a25fe 100644 --- a/scrapling/core/shell.py +++ b/scrapling/core/shell.py @@ -653,7 +653,9 @@ class Convertor: yield "" @classmethod - def write_content_to_file(cls, page: Selector, filename: str, css_selector: Optional[str] = None) -> None: + def write_content_to_file( + cls, page: Selector, filename: str, css_selector: Optional[str] = None, main_content_only: bool = False + ) -> None: """Write a Selector's content to a file""" if not page or not isinstance(page, Selector): # pragma: no cover raise TypeError("Input must be of type `Selector`") @@ -670,6 +672,7 @@ class Convertor: page, cls._extension_map[extension], css_selector=css_selector, + main_content_only=main_content_only, ) ) )