refactor(cli): Code cleaning for easier maintenance/adding new features
Shortened the code by 210 lines. Also, removed docstrings because they are not needed for CLI commands (more maintenance burden). - `_common_http_options`: shared decorator for 10 Click options used by get/post/put/delete (was repeated 4x) - `_common_browser_options`: shared decorator for 11 Click options used by fetch/stealthy_fetch (was repeated 2x) - `_data_options`: shared decorator for `--data`/`--json` options used by post/put - `__http_command()`: shared implementation body for all HTTP commands (was 4 separate `from scrapling.fetchers import Fetcher` + `__Request_and_Save` blocks) - `__build_browser_kwargs()`: shared kwargs builder for fetch/stealthy_fetch (was duplicated)
This commit is contained in:
+217
-427
@@ -194,48 +194,140 @@ def extract():
|
|||||||
pass
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
####
|
||||||
|
# Shared Click option decorator factories
|
||||||
|
####
|
||||||
|
|
||||||
|
|
||||||
|
def _common_http_options(f):
|
||||||
|
"""Apply shared Click options for all HTTP extract commands (get/post/put/delete)."""
|
||||||
|
decorators = [
|
||||||
|
option(
|
||||||
|
"--stealthy-headers/--no-stealthy-headers",
|
||||||
|
default=True,
|
||||||
|
help="Use stealthy browser headers (default: True)",
|
||||||
|
),
|
||||||
|
option(
|
||||||
|
"--impersonate",
|
||||||
|
help="Browser to impersonate. Can be a single browser (e.g., chrome) or comma-separated list for random selection (e.g., chrome,firefox,safari).",
|
||||||
|
),
|
||||||
|
option(
|
||||||
|
"--verify/--no-verify",
|
||||||
|
default=True,
|
||||||
|
help="Whether to verify SSL certificates (default: True)",
|
||||||
|
),
|
||||||
|
option(
|
||||||
|
"--follow-redirects/--no-follow-redirects",
|
||||||
|
default=True,
|
||||||
|
help="Whether to follow redirects (default: True)",
|
||||||
|
),
|
||||||
|
option(
|
||||||
|
"--params",
|
||||||
|
"-p",
|
||||||
|
multiple=True,
|
||||||
|
help='Query parameters in format "key=value" (can be used multiple times)',
|
||||||
|
),
|
||||||
|
option(
|
||||||
|
"--css-selector",
|
||||||
|
"-s",
|
||||||
|
help="CSS selector to extract specific content from the page. It returns all matches.",
|
||||||
|
),
|
||||||
|
option("--proxy", help='Proxy URL in format "http://username:password@host:port"'),
|
||||||
|
option("--timeout", type=int, default=30, help="Request timeout in seconds (default: 30)"),
|
||||||
|
option("--cookies", help='Cookies string in format "name1=value1; name2=value2"'),
|
||||||
|
option(
|
||||||
|
"--headers",
|
||||||
|
"-H",
|
||||||
|
multiple=True,
|
||||||
|
help='HTTP headers in format "Key: Value" (can be used multiple times)',
|
||||||
|
),
|
||||||
|
]
|
||||||
|
for decorator in decorators:
|
||||||
|
f = decorator(f)
|
||||||
|
return f
|
||||||
|
|
||||||
|
|
||||||
|
def _common_browser_options(f):
|
||||||
|
"""Apply shared Click options for browser-based commands (fetch/stealthy_fetch)."""
|
||||||
|
decorators = [
|
||||||
|
option(
|
||||||
|
"--extra-headers",
|
||||||
|
"-H",
|
||||||
|
multiple=True,
|
||||||
|
help='Extra headers in format "Key: Value" (can be used multiple times)',
|
||||||
|
),
|
||||||
|
option("--proxy", help='Proxy URL in format "http://username:password@host:port"'),
|
||||||
|
option(
|
||||||
|
"--real-chrome/--no-real-chrome",
|
||||||
|
default=False,
|
||||||
|
help="If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch an instance of your browser and use it. (default: False)",
|
||||||
|
),
|
||||||
|
option("--locale", default=None, help="Specify user locale. Defaults to the system default locale."),
|
||||||
|
option("--wait-selector", help="CSS selector to wait for before proceeding"),
|
||||||
|
option(
|
||||||
|
"--css-selector",
|
||||||
|
"-s",
|
||||||
|
help="CSS selector to extract specific content from the page. It returns all matches.",
|
||||||
|
),
|
||||||
|
option(
|
||||||
|
"--wait",
|
||||||
|
type=int,
|
||||||
|
default=0,
|
||||||
|
help="Additional wait time in milliseconds after page load (default: 0)",
|
||||||
|
),
|
||||||
|
option(
|
||||||
|
"--timeout",
|
||||||
|
type=int,
|
||||||
|
default=30000,
|
||||||
|
help="Timeout in milliseconds (default: 30000)",
|
||||||
|
),
|
||||||
|
option(
|
||||||
|
"--network-idle/--no-network-idle",
|
||||||
|
default=False,
|
||||||
|
help="Wait for network idle (default: False)",
|
||||||
|
),
|
||||||
|
option(
|
||||||
|
"--disable-resources/--enable-resources",
|
||||||
|
default=False,
|
||||||
|
help="Drop unnecessary resources for speed boost (default: False)",
|
||||||
|
),
|
||||||
|
option(
|
||||||
|
"--headless/--no-headless",
|
||||||
|
default=True,
|
||||||
|
help="Run browser in headless mode (default: True)",
|
||||||
|
),
|
||||||
|
]
|
||||||
|
for decorator in decorators:
|
||||||
|
f = decorator(f)
|
||||||
|
return f
|
||||||
|
|
||||||
|
|
||||||
|
def _data_options(f):
|
||||||
|
"""Apply data/json options for POST and PUT commands."""
|
||||||
|
decorators = [
|
||||||
|
option("--json", "-j", help="JSON data to include in the request body (as string)"),
|
||||||
|
option(
|
||||||
|
"--data",
|
||||||
|
"-d",
|
||||||
|
help='Form data to include in the request body (as string, ex: "param1=value1¶m2=value2")',
|
||||||
|
),
|
||||||
|
]
|
||||||
|
for decorator in decorators:
|
||||||
|
f = decorator(f)
|
||||||
|
return f
|
||||||
|
|
||||||
|
|
||||||
|
def __http_command(method_name: str, url: str, output_file: str, css_selector: Optional[str], **kwargs) -> None:
|
||||||
|
"""Shared implementation for HTTP extract commands."""
|
||||||
|
from scrapling.fetchers import Fetcher
|
||||||
|
|
||||||
|
__Request_and_Save(getattr(Fetcher, method_name), url, output_file, css_selector, **kwargs)
|
||||||
|
|
||||||
|
|
||||||
@extract.command(help=f"Perform a GET request and save the content to a file.\n\n{__OUTPUT_FILE_HELP__}")
|
@extract.command(help=f"Perform a GET request and save the content to a file.\n\n{__OUTPUT_FILE_HELP__}")
|
||||||
@argument("url", required=True)
|
@argument("url", required=True)
|
||||||
@argument("output_file", required=True)
|
@argument("output_file", required=True)
|
||||||
@option(
|
@_common_http_options
|
||||||
"--headers",
|
|
||||||
"-H",
|
|
||||||
multiple=True,
|
|
||||||
help='HTTP headers in format "Key: Value" (can be used multiple times)',
|
|
||||||
)
|
|
||||||
@option("--cookies", help='Cookies string in format "name1=value1; name2=value2"')
|
|
||||||
@option("--timeout", type=int, default=30, help="Request timeout in seconds (default: 30)")
|
|
||||||
@option("--proxy", help='Proxy URL in format "http://username:password@host:port"')
|
|
||||||
@option(
|
|
||||||
"--css-selector",
|
|
||||||
"-s",
|
|
||||||
help="CSS selector to extract specific content from the page. It returns all matches.",
|
|
||||||
)
|
|
||||||
@option(
|
|
||||||
"--params",
|
|
||||||
"-p",
|
|
||||||
multiple=True,
|
|
||||||
help='Query parameters in format "key=value" (can be used multiple times)',
|
|
||||||
)
|
|
||||||
@option(
|
|
||||||
"--follow-redirects/--no-follow-redirects",
|
|
||||||
default=True,
|
|
||||||
help="Whether to follow redirects (default: True)",
|
|
||||||
)
|
|
||||||
@option(
|
|
||||||
"--verify/--no-verify",
|
|
||||||
default=True,
|
|
||||||
help="Whether to verify SSL certificates (default: True)",
|
|
||||||
)
|
|
||||||
@option(
|
|
||||||
"--impersonate",
|
|
||||||
help="Browser to impersonate. Can be a single browser (e.g., chrome) or comma-separated list for random selection (e.g., chrome,firefox,safari).",
|
|
||||||
)
|
|
||||||
@option(
|
|
||||||
"--stealthy-headers/--no-stealthy-headers",
|
|
||||||
default=True,
|
|
||||||
help="Use stealthy browser headers (default: True)",
|
|
||||||
)
|
|
||||||
def get(
|
def get(
|
||||||
url,
|
url,
|
||||||
output_file,
|
output_file,
|
||||||
@@ -250,23 +342,7 @@ def get(
|
|||||||
impersonate,
|
impersonate,
|
||||||
stealthy_headers,
|
stealthy_headers,
|
||||||
):
|
):
|
||||||
"""
|
"""Perform a GET request and save the content to a file."""
|
||||||
Perform a GET request and save the content to a file.
|
|
||||||
|
|
||||||
:param url: Target URL for the request.
|
|
||||||
:param output_file: Output file path (.md for Markdown, .html for HTML).
|
|
||||||
:param headers: HTTP headers to include in the request.
|
|
||||||
:param cookies: Cookies to use in the request.
|
|
||||||
:param timeout: Number of seconds to wait before timing out.
|
|
||||||
:param proxy: Proxy URL to use. (Format: "http://username:password@localhost:8030")
|
|
||||||
:param css_selector: CSS selector to extract specific content.
|
|
||||||
:param params: Query string parameters for the request.
|
|
||||||
:param follow_redirects: Whether to follow redirects.
|
|
||||||
:param verify: Whether to verify HTTPS certificates.
|
|
||||||
:param impersonate: Browser version to impersonate.
|
|
||||||
:param stealthy_headers: If enabled, creates and adds real browser headers.
|
|
||||||
"""
|
|
||||||
|
|
||||||
kwargs = __BuildRequest(
|
kwargs = __BuildRequest(
|
||||||
headers,
|
headers,
|
||||||
cookies,
|
cookies,
|
||||||
@@ -279,59 +355,14 @@ def get(
|
|||||||
impersonate=impersonate,
|
impersonate=impersonate,
|
||||||
proxy=proxy,
|
proxy=proxy,
|
||||||
)
|
)
|
||||||
from scrapling.fetchers import Fetcher
|
__http_command("get", url, output_file, css_selector, **kwargs)
|
||||||
|
|
||||||
__Request_and_Save(Fetcher.get, url, output_file, css_selector, **kwargs)
|
|
||||||
|
|
||||||
|
|
||||||
@extract.command(help=f"Perform a POST request and save the content to a file.\n\n{__OUTPUT_FILE_HELP__}")
|
@extract.command(help=f"Perform a POST request and save the content to a file.\n\n{__OUTPUT_FILE_HELP__}")
|
||||||
@argument("url", required=True)
|
@argument("url", required=True)
|
||||||
@argument("output_file", required=True)
|
@argument("output_file", required=True)
|
||||||
@option(
|
@_data_options
|
||||||
"--data",
|
@_common_http_options
|
||||||
"-d",
|
|
||||||
help='Form data to include in the request body (as string, ex: "param1=value1¶m2=value2")',
|
|
||||||
)
|
|
||||||
@option("--json", "-j", help="JSON data to include in the request body (as string)")
|
|
||||||
@option(
|
|
||||||
"--headers",
|
|
||||||
"-H",
|
|
||||||
multiple=True,
|
|
||||||
help='HTTP headers in format "Key: Value" (can be used multiple times)',
|
|
||||||
)
|
|
||||||
@option("--cookies", help='Cookies string in format "name1=value1; name2=value2"')
|
|
||||||
@option("--timeout", type=int, default=30, help="Request timeout in seconds (default: 30)")
|
|
||||||
@option("--proxy", help='Proxy URL in format "http://username:password@host:port"')
|
|
||||||
@option(
|
|
||||||
"--css-selector",
|
|
||||||
"-s",
|
|
||||||
help="CSS selector to extract specific content from the page. It returns all matches.",
|
|
||||||
)
|
|
||||||
@option(
|
|
||||||
"--params",
|
|
||||||
"-p",
|
|
||||||
multiple=True,
|
|
||||||
help='Query parameters in format "key=value" (can be used multiple times)',
|
|
||||||
)
|
|
||||||
@option(
|
|
||||||
"--follow-redirects/--no-follow-redirects",
|
|
||||||
default=True,
|
|
||||||
help="Whether to follow redirects (default: True)",
|
|
||||||
)
|
|
||||||
@option(
|
|
||||||
"--verify/--no-verify",
|
|
||||||
default=True,
|
|
||||||
help="Whether to verify SSL certificates (default: True)",
|
|
||||||
)
|
|
||||||
@option(
|
|
||||||
"--impersonate",
|
|
||||||
help="Browser to impersonate. Can be a single browser (e.g., chrome) or comma-separated list for random selection (e.g., chrome,firefox,safari).",
|
|
||||||
)
|
|
||||||
@option(
|
|
||||||
"--stealthy-headers/--no-stealthy-headers",
|
|
||||||
default=True,
|
|
||||||
help="Use stealthy browser headers (default: True)",
|
|
||||||
)
|
|
||||||
def post(
|
def post(
|
||||||
url,
|
url,
|
||||||
output_file,
|
output_file,
|
||||||
@@ -348,25 +379,7 @@ def post(
|
|||||||
impersonate,
|
impersonate,
|
||||||
stealthy_headers,
|
stealthy_headers,
|
||||||
):
|
):
|
||||||
"""
|
"""Perform a POST request and save the content to a file."""
|
||||||
Perform a POST request and save the content to a file.
|
|
||||||
|
|
||||||
:param url: Target URL for the request.
|
|
||||||
:param output_file: Output file path (.md for Markdown, .html for HTML).
|
|
||||||
:param data: Form data to include in the request body. (as string, ex: "param1=value1¶m2=value2")
|
|
||||||
:param json: A JSON serializable object to include in the body of the request.
|
|
||||||
:param headers: Headers to include in the request.
|
|
||||||
:param cookies: Cookies to use in the request.
|
|
||||||
:param timeout: Number of seconds to wait before timing out.
|
|
||||||
:param proxy: Proxy URL to use.
|
|
||||||
:param css_selector: CSS selector to extract specific content.
|
|
||||||
:param params: Query string parameters for the request.
|
|
||||||
:param follow_redirects: Whether to follow redirects.
|
|
||||||
:param verify: Whether to verify HTTPS certificates.
|
|
||||||
:param impersonate: Browser version to impersonate.
|
|
||||||
:param stealthy_headers: If enabled, creates and adds real browser headers.
|
|
||||||
"""
|
|
||||||
|
|
||||||
kwargs = __BuildRequest(
|
kwargs = __BuildRequest(
|
||||||
headers,
|
headers,
|
||||||
cookies,
|
cookies,
|
||||||
@@ -380,55 +393,14 @@ def post(
|
|||||||
proxy=proxy,
|
proxy=proxy,
|
||||||
data=data,
|
data=data,
|
||||||
)
|
)
|
||||||
from scrapling.fetchers import Fetcher
|
__http_command("post", url, output_file, css_selector, **kwargs)
|
||||||
|
|
||||||
__Request_and_Save(Fetcher.post, url, output_file, css_selector, **kwargs)
|
|
||||||
|
|
||||||
|
|
||||||
@extract.command(help=f"Perform a PUT request and save the content to a file.\n\n{__OUTPUT_FILE_HELP__}")
|
@extract.command(help=f"Perform a PUT request and save the content to a file.\n\n{__OUTPUT_FILE_HELP__}")
|
||||||
@argument("url", required=True)
|
@argument("url", required=True)
|
||||||
@argument("output_file", required=True)
|
@argument("output_file", required=True)
|
||||||
@option("--data", "-d", help="Form data to include in the request body")
|
@_data_options
|
||||||
@option("--json", "-j", help="JSON data to include in the request body (as string)")
|
@_common_http_options
|
||||||
@option(
|
|
||||||
"--headers",
|
|
||||||
"-H",
|
|
||||||
multiple=True,
|
|
||||||
help='HTTP headers in format "Key: Value" (can be used multiple times)',
|
|
||||||
)
|
|
||||||
@option("--cookies", help='Cookies string in format "name1=value1; name2=value2"')
|
|
||||||
@option("--timeout", type=int, default=30, help="Request timeout in seconds (default: 30)")
|
|
||||||
@option("--proxy", help='Proxy URL in format "http://username:password@host:port"')
|
|
||||||
@option(
|
|
||||||
"--css-selector",
|
|
||||||
"-s",
|
|
||||||
help="CSS selector to extract specific content from the page. It returns all matches.",
|
|
||||||
)
|
|
||||||
@option(
|
|
||||||
"--params",
|
|
||||||
"-p",
|
|
||||||
multiple=True,
|
|
||||||
help='Query parameters in format "key=value" (can be used multiple times)',
|
|
||||||
)
|
|
||||||
@option(
|
|
||||||
"--follow-redirects/--no-follow-redirects",
|
|
||||||
default=True,
|
|
||||||
help="Whether to follow redirects (default: True)",
|
|
||||||
)
|
|
||||||
@option(
|
|
||||||
"--verify/--no-verify",
|
|
||||||
default=True,
|
|
||||||
help="Whether to verify SSL certificates (default: True)",
|
|
||||||
)
|
|
||||||
@option(
|
|
||||||
"--impersonate",
|
|
||||||
help="Browser to impersonate. Can be a single browser (e.g., chrome) or comma-separated list for random selection (e.g., chrome,firefox,safari).",
|
|
||||||
)
|
|
||||||
@option(
|
|
||||||
"--stealthy-headers/--no-stealthy-headers",
|
|
||||||
default=True,
|
|
||||||
help="Use stealthy browser headers (default: True)",
|
|
||||||
)
|
|
||||||
def put(
|
def put(
|
||||||
url,
|
url,
|
||||||
output_file,
|
output_file,
|
||||||
@@ -445,25 +417,7 @@ def put(
|
|||||||
impersonate,
|
impersonate,
|
||||||
stealthy_headers,
|
stealthy_headers,
|
||||||
):
|
):
|
||||||
"""
|
"""Perform a PUT request and save the content to a file."""
|
||||||
Perform a PUT request and save the content to a file.
|
|
||||||
|
|
||||||
:param url: Target URL for the request.
|
|
||||||
:param output_file: Output file path (.md for Markdown, .html for HTML).
|
|
||||||
:param data: Form data to include in the request body.
|
|
||||||
:param json: A JSON serializable object to include in the body of the request.
|
|
||||||
:param headers: Headers to include in the request.
|
|
||||||
:param cookies: Cookies to use in the request.
|
|
||||||
:param timeout: Number of seconds to wait before timing out.
|
|
||||||
:param proxy: Proxy URL to use.
|
|
||||||
:param css_selector: CSS selector to extract specific content.
|
|
||||||
:param params: Query string parameters for the request.
|
|
||||||
:param follow_redirects: Whether to follow redirects.
|
|
||||||
:param verify: Whether to verify HTTPS certificates.
|
|
||||||
:param impersonate: Browser version to impersonate.
|
|
||||||
:param stealthy_headers: If enabled, creates and adds real browser headers.
|
|
||||||
"""
|
|
||||||
|
|
||||||
kwargs = __BuildRequest(
|
kwargs = __BuildRequest(
|
||||||
headers,
|
headers,
|
||||||
cookies,
|
cookies,
|
||||||
@@ -477,53 +431,13 @@ def put(
|
|||||||
proxy=proxy,
|
proxy=proxy,
|
||||||
data=data,
|
data=data,
|
||||||
)
|
)
|
||||||
from scrapling.fetchers import Fetcher
|
__http_command("put", url, output_file, css_selector, **kwargs)
|
||||||
|
|
||||||
__Request_and_Save(Fetcher.put, url, output_file, css_selector, **kwargs)
|
|
||||||
|
|
||||||
|
|
||||||
@extract.command(help=f"Perform a DELETE request and save the content to a file.\n\n{__OUTPUT_FILE_HELP__}")
|
@extract.command(help=f"Perform a DELETE request and save the content to a file.\n\n{__OUTPUT_FILE_HELP__}")
|
||||||
@argument("url", required=True)
|
@argument("url", required=True)
|
||||||
@argument("output_file", required=True)
|
@argument("output_file", required=True)
|
||||||
@option(
|
@_common_http_options
|
||||||
"--headers",
|
|
||||||
"-H",
|
|
||||||
multiple=True,
|
|
||||||
help='HTTP headers in format "Key: Value" (can be used multiple times)',
|
|
||||||
)
|
|
||||||
@option("--cookies", help='Cookies string in format "name1=value1; name2=value2"')
|
|
||||||
@option("--timeout", type=int, default=30, help="Request timeout in seconds (default: 30)")
|
|
||||||
@option("--proxy", help='Proxy URL in format "http://username:password@host:port"')
|
|
||||||
@option(
|
|
||||||
"--css-selector",
|
|
||||||
"-s",
|
|
||||||
help="CSS selector to extract specific content from the page. It returns all matches.",
|
|
||||||
)
|
|
||||||
@option(
|
|
||||||
"--params",
|
|
||||||
"-p",
|
|
||||||
multiple=True,
|
|
||||||
help='Query parameters in format "key=value" (can be used multiple times)',
|
|
||||||
)
|
|
||||||
@option(
|
|
||||||
"--follow-redirects/--no-follow-redirects",
|
|
||||||
default=True,
|
|
||||||
help="Whether to follow redirects (default: True)",
|
|
||||||
)
|
|
||||||
@option(
|
|
||||||
"--verify/--no-verify",
|
|
||||||
default=True,
|
|
||||||
help="Whether to verify SSL certificates (default: True)",
|
|
||||||
)
|
|
||||||
@option(
|
|
||||||
"--impersonate",
|
|
||||||
help="Browser to impersonate. Can be a single browser (e.g., chrome) or comma-separated list for random selection (e.g., chrome,firefox,safari).",
|
|
||||||
)
|
|
||||||
@option(
|
|
||||||
"--stealthy-headers/--no-stealthy-headers",
|
|
||||||
default=True,
|
|
||||||
help="Use stealthy browser headers (default: True)",
|
|
||||||
)
|
|
||||||
def delete(
|
def delete(
|
||||||
url,
|
url,
|
||||||
output_file,
|
output_file,
|
||||||
@@ -538,23 +452,7 @@ def delete(
|
|||||||
impersonate,
|
impersonate,
|
||||||
stealthy_headers,
|
stealthy_headers,
|
||||||
):
|
):
|
||||||
"""
|
"""Perform a DELETE request and save the content to a file."""
|
||||||
Perform a DELETE request and save the content to a file.
|
|
||||||
|
|
||||||
:param url: Target URL for the request.
|
|
||||||
:param output_file: Output file path (.md for Markdown, .html for HTML).
|
|
||||||
:param headers: Headers to include in the request.
|
|
||||||
:param cookies: Cookies to use in the request.
|
|
||||||
:param timeout: Number of seconds to wait before timing out.
|
|
||||||
:param proxy: Proxy URL to use.
|
|
||||||
:param css_selector: CSS selector to extract specific content.
|
|
||||||
:param params: Query string parameters for the request.
|
|
||||||
:param follow_redirects: Whether to follow redirects.
|
|
||||||
:param verify: Whether to verify HTTPS certificates.
|
|
||||||
:param impersonate: Browser version to impersonate.
|
|
||||||
:param stealthy_headers: If enabled, creates and adds real browser headers.
|
|
||||||
"""
|
|
||||||
|
|
||||||
kwargs = __BuildRequest(
|
kwargs = __BuildRequest(
|
||||||
headers,
|
headers,
|
||||||
cookies,
|
cookies,
|
||||||
@@ -567,60 +465,45 @@ def delete(
|
|||||||
impersonate=impersonate,
|
impersonate=impersonate,
|
||||||
proxy=proxy,
|
proxy=proxy,
|
||||||
)
|
)
|
||||||
from scrapling.fetchers import Fetcher
|
__http_command("delete", url, output_file, css_selector, **kwargs)
|
||||||
|
|
||||||
__Request_and_Save(Fetcher.delete, url, output_file, css_selector, **kwargs)
|
|
||||||
|
def __build_browser_kwargs(
|
||||||
|
headless,
|
||||||
|
disable_resources,
|
||||||
|
network_idle,
|
||||||
|
timeout,
|
||||||
|
wait,
|
||||||
|
wait_selector,
|
||||||
|
locale,
|
||||||
|
real_chrome,
|
||||||
|
proxy,
|
||||||
|
parsed_headers,
|
||||||
|
) -> Dict[str, Any]:
|
||||||
|
"""Build shared kwargs dict for browser-based commands."""
|
||||||
|
kwargs: Dict[str, Any] = {
|
||||||
|
"headless": headless,
|
||||||
|
"disable_resources": disable_resources,
|
||||||
|
"network_idle": network_idle,
|
||||||
|
"timeout": timeout,
|
||||||
|
"locale": locale,
|
||||||
|
"real_chrome": real_chrome,
|
||||||
|
}
|
||||||
|
if wait > 0:
|
||||||
|
kwargs["wait"] = wait
|
||||||
|
if wait_selector:
|
||||||
|
kwargs["wait_selector"] = wait_selector
|
||||||
|
if proxy:
|
||||||
|
kwargs["proxy"] = proxy
|
||||||
|
if parsed_headers:
|
||||||
|
kwargs["extra_headers"] = parsed_headers
|
||||||
|
return kwargs
|
||||||
|
|
||||||
|
|
||||||
@extract.command(help=f"Use DynamicFetcher to fetch content with browser automation.\n\n{__OUTPUT_FILE_HELP__}")
|
@extract.command(help=f"Use DynamicFetcher to fetch content with browser automation.\n\n{__OUTPUT_FILE_HELP__}")
|
||||||
@argument("url", required=True)
|
@argument("url", required=True)
|
||||||
@argument("output_file", required=True)
|
@argument("output_file", required=True)
|
||||||
@option(
|
@_common_browser_options
|
||||||
"--headless/--no-headless",
|
|
||||||
default=True,
|
|
||||||
help="Run browser in headless mode (default: True)",
|
|
||||||
)
|
|
||||||
@option(
|
|
||||||
"--disable-resources/--enable-resources",
|
|
||||||
default=False,
|
|
||||||
help="Drop unnecessary resources for speed boost (default: False)",
|
|
||||||
)
|
|
||||||
@option(
|
|
||||||
"--network-idle/--no-network-idle",
|
|
||||||
default=False,
|
|
||||||
help="Wait for network idle (default: False)",
|
|
||||||
)
|
|
||||||
@option(
|
|
||||||
"--timeout",
|
|
||||||
type=int,
|
|
||||||
default=30000,
|
|
||||||
help="Timeout in milliseconds (default: 30000)",
|
|
||||||
)
|
|
||||||
@option(
|
|
||||||
"--wait",
|
|
||||||
type=int,
|
|
||||||
default=0,
|
|
||||||
help="Additional wait time in milliseconds after page load (default: 0)",
|
|
||||||
)
|
|
||||||
@option(
|
|
||||||
"--css-selector",
|
|
||||||
"-s",
|
|
||||||
help="CSS selector to extract specific content from the page. It returns all matches.",
|
|
||||||
)
|
|
||||||
@option("--wait-selector", help="CSS selector to wait for before proceeding")
|
|
||||||
@option("--locale", default=None, help="Specify user locale. Defaults to the system default locale.")
|
|
||||||
@option(
|
|
||||||
"--real-chrome/--no-real-chrome",
|
|
||||||
default=False,
|
|
||||||
help="If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch an instance of your browser and use it. (default: False)",
|
|
||||||
)
|
|
||||||
@option("--proxy", help='Proxy URL in format "http://username:password@host:port"')
|
|
||||||
@option(
|
|
||||||
"--extra-headers",
|
|
||||||
"-H",
|
|
||||||
multiple=True,
|
|
||||||
help='Extra headers in format "Key: Value" (can be used multiple times)',
|
|
||||||
)
|
|
||||||
def fetch(
|
def fetch(
|
||||||
url,
|
url,
|
||||||
output_file,
|
output_file,
|
||||||
@@ -636,46 +519,20 @@ def fetch(
|
|||||||
proxy,
|
proxy,
|
||||||
extra_headers,
|
extra_headers,
|
||||||
):
|
):
|
||||||
"""
|
"""Opens up a browser and fetch content using DynamicFetcher."""
|
||||||
Opens up a browser and fetch content using DynamicFetcher.
|
|
||||||
|
|
||||||
:param url: Target url.
|
|
||||||
:param output_file: Output file path (.md for Markdown, .html for HTML).
|
|
||||||
:param headless: Run the browser in headless/hidden or headful/visible mode.
|
|
||||||
:param disable_resources: Drop requests of unnecessary resources for a speed boost.
|
|
||||||
:param network_idle: Wait for the page until there are no network connections for at least 500 ms.
|
|
||||||
:param timeout: The timeout in milliseconds that is used in all operations and waits through the page.
|
|
||||||
:param wait: The time (milliseconds) the fetcher will wait after everything finishes before returning.
|
|
||||||
:param css_selector: CSS selector to extract specific content.
|
|
||||||
:param wait_selector: Wait for a specific CSS selector to be in a specific state.
|
|
||||||
:param locale: Set the locale for the browser.
|
|
||||||
:param real_chrome: If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch an instance of your browser and use it.
|
|
||||||
:param proxy: The proxy to be used with requests.
|
|
||||||
:param extra_headers: Extra headers to add to the request.
|
|
||||||
"""
|
|
||||||
|
|
||||||
# Parse parameters
|
|
||||||
parsed_headers, _ = _ParseHeaders(extra_headers, False)
|
parsed_headers, _ = _ParseHeaders(extra_headers, False)
|
||||||
|
kwargs = __build_browser_kwargs(
|
||||||
# Build request arguments
|
headless,
|
||||||
kwargs = {
|
disable_resources,
|
||||||
"headless": headless,
|
network_idle,
|
||||||
"disable_resources": disable_resources,
|
timeout,
|
||||||
"network_idle": network_idle,
|
wait,
|
||||||
"timeout": timeout,
|
wait_selector,
|
||||||
"locale": locale,
|
locale,
|
||||||
"real_chrome": real_chrome,
|
real_chrome,
|
||||||
}
|
proxy,
|
||||||
|
parsed_headers,
|
||||||
if wait > 0:
|
)
|
||||||
kwargs["wait"] = wait
|
|
||||||
if wait_selector:
|
|
||||||
kwargs["wait_selector"] = wait_selector
|
|
||||||
if proxy:
|
|
||||||
kwargs["proxy"] = proxy
|
|
||||||
if parsed_headers:
|
|
||||||
kwargs["extra_headers"] = parsed_headers
|
|
||||||
|
|
||||||
from scrapling.fetchers import DynamicFetcher
|
from scrapling.fetchers import DynamicFetcher
|
||||||
|
|
||||||
__Request_and_Save(DynamicFetcher.fetch, url, output_file, css_selector, **kwargs)
|
__Request_and_Save(DynamicFetcher.fetch, url, output_file, css_selector, **kwargs)
|
||||||
@@ -684,16 +541,6 @@ def fetch(
|
|||||||
@extract.command(help=f"Use StealthyFetcher to fetch content with advanced stealth features.\n\n{__OUTPUT_FILE_HELP__}")
|
@extract.command(help=f"Use StealthyFetcher to fetch content with advanced stealth features.\n\n{__OUTPUT_FILE_HELP__}")
|
||||||
@argument("url", required=True)
|
@argument("url", required=True)
|
||||||
@argument("output_file", required=True)
|
@argument("output_file", required=True)
|
||||||
@option(
|
|
||||||
"--headless/--no-headless",
|
|
||||||
default=True,
|
|
||||||
help="Run browser in headless mode (default: True)",
|
|
||||||
)
|
|
||||||
@option(
|
|
||||||
"--disable-resources/--enable-resources",
|
|
||||||
default=False,
|
|
||||||
help="Drop unnecessary resources for speed boost (default: False)",
|
|
||||||
)
|
|
||||||
@option(
|
@option(
|
||||||
"--block-webrtc/--allow-webrtc",
|
"--block-webrtc/--allow-webrtc",
|
||||||
default=False,
|
default=False,
|
||||||
@@ -705,110 +552,53 @@ def fetch(
|
|||||||
help="Solve Cloudflare challenges (default: False)",
|
help="Solve Cloudflare challenges (default: False)",
|
||||||
)
|
)
|
||||||
@option("--allow-webgl/--block-webgl", default=True, help="Allow WebGL (default: True)")
|
@option("--allow-webgl/--block-webgl", default=True, help="Allow WebGL (default: True)")
|
||||||
@option(
|
|
||||||
"--network-idle/--no-network-idle",
|
|
||||||
default=False,
|
|
||||||
help="Wait for network idle (default: False)",
|
|
||||||
)
|
|
||||||
@option(
|
|
||||||
"--real-chrome/--no-real-chrome",
|
|
||||||
default=False,
|
|
||||||
help="If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch an instance of your browser and use it. (default: False)",
|
|
||||||
)
|
|
||||||
@option(
|
@option(
|
||||||
"--hide-canvas/--show-canvas",
|
"--hide-canvas/--show-canvas",
|
||||||
default=False,
|
default=False,
|
||||||
help="Add noise to canvas operations (default: False)",
|
help="Add noise to canvas operations (default: False)",
|
||||||
)
|
)
|
||||||
@option(
|
@_common_browser_options
|
||||||
"--timeout",
|
|
||||||
type=int,
|
|
||||||
default=30000,
|
|
||||||
help="Timeout in milliseconds (default: 30000)",
|
|
||||||
)
|
|
||||||
@option(
|
|
||||||
"--wait",
|
|
||||||
type=int,
|
|
||||||
default=0,
|
|
||||||
help="Additional wait time in milliseconds after page load (default: 0)",
|
|
||||||
)
|
|
||||||
@option(
|
|
||||||
"--css-selector",
|
|
||||||
"-s",
|
|
||||||
help="CSS selector to extract specific content from the page. It returns all matches.",
|
|
||||||
)
|
|
||||||
@option("--wait-selector", help="CSS selector to wait for before proceeding")
|
|
||||||
@option("--proxy", help='Proxy URL in format "http://username:password@host:port"')
|
|
||||||
@option(
|
|
||||||
"--extra-headers",
|
|
||||||
"-H",
|
|
||||||
multiple=True,
|
|
||||||
help='Extra headers in format "Key: Value" (can be used multiple times)',
|
|
||||||
)
|
|
||||||
def stealthy_fetch(
|
def stealthy_fetch(
|
||||||
url,
|
url,
|
||||||
output_file,
|
output_file,
|
||||||
headless,
|
headless,
|
||||||
disable_resources,
|
disable_resources,
|
||||||
block_webrtc,
|
|
||||||
solve_cloudflare,
|
|
||||||
allow_webgl,
|
|
||||||
network_idle,
|
network_idle,
|
||||||
real_chrome,
|
|
||||||
hide_canvas,
|
|
||||||
timeout,
|
timeout,
|
||||||
wait,
|
wait,
|
||||||
css_selector,
|
css_selector,
|
||||||
wait_selector,
|
wait_selector,
|
||||||
|
locale,
|
||||||
|
real_chrome,
|
||||||
proxy,
|
proxy,
|
||||||
extra_headers,
|
extra_headers,
|
||||||
|
block_webrtc,
|
||||||
|
solve_cloudflare,
|
||||||
|
allow_webgl,
|
||||||
|
hide_canvas,
|
||||||
):
|
):
|
||||||
"""
|
"""Opens up a browser with advanced stealth features and fetch content using StealthyFetcher."""
|
||||||
Opens up a browser with advanced stealth features and fetch content using StealthyFetcher.
|
|
||||||
|
|
||||||
:param url: Target url.
|
|
||||||
:param output_file: Output file path (.md for Markdown, .html for HTML).
|
|
||||||
:param headless: Run the browser in headless/hidden, or headful/visible mode.
|
|
||||||
:param disable_resources: Drop requests of unnecessary resources for a speed boost.
|
|
||||||
:param block_webrtc: Blocks WebRTC entirely.
|
|
||||||
:param solve_cloudflare: Solves all types of the Cloudflare's Turnstile/Interstitial challenges.
|
|
||||||
:param allow_webgl: Allow WebGL (recommended to keep enabled).
|
|
||||||
:param network_idle: Wait for the page until there are no network connections for at least 500 ms.
|
|
||||||
:param real_chrome: If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch an instance of your browser and use it.
|
|
||||||
:param hide_canvas: Add random noise to canvas operations to prevent fingerprinting.
|
|
||||||
:param timeout: The timeout in milliseconds that is used in all operations and waits through the page.
|
|
||||||
:param wait: The time (milliseconds) the fetcher will wait after everything finishes before returning.
|
|
||||||
:param css_selector: CSS selector to extract specific content.
|
|
||||||
:param wait_selector: Wait for a specific CSS selector to be in a specific state.
|
|
||||||
:param proxy: The proxy to be used with requests.
|
|
||||||
:param extra_headers: Extra headers to add to the request.
|
|
||||||
"""
|
|
||||||
|
|
||||||
# Parse parameters
|
|
||||||
parsed_headers, _ = _ParseHeaders(extra_headers, False)
|
parsed_headers, _ = _ParseHeaders(extra_headers, False)
|
||||||
|
kwargs = __build_browser_kwargs(
|
||||||
# Build request arguments
|
headless,
|
||||||
kwargs = {
|
disable_resources,
|
||||||
"headless": headless,
|
network_idle,
|
||||||
"disable_resources": disable_resources,
|
timeout,
|
||||||
"block_webrtc": block_webrtc,
|
wait,
|
||||||
"solve_cloudflare": solve_cloudflare,
|
wait_selector,
|
||||||
"allow_webgl": allow_webgl,
|
locale,
|
||||||
"network_idle": network_idle,
|
real_chrome,
|
||||||
"real_chrome": real_chrome,
|
proxy,
|
||||||
"hide_canvas": hide_canvas,
|
parsed_headers,
|
||||||
"timeout": timeout,
|
)
|
||||||
}
|
kwargs.update(
|
||||||
|
{
|
||||||
if wait > 0:
|
"block_webrtc": block_webrtc,
|
||||||
kwargs["wait"] = wait
|
"solve_cloudflare": solve_cloudflare,
|
||||||
if wait_selector:
|
"allow_webgl": allow_webgl,
|
||||||
kwargs["wait_selector"] = wait_selector
|
"hide_canvas": hide_canvas,
|
||||||
if proxy:
|
}
|
||||||
kwargs["proxy"] = proxy
|
)
|
||||||
if parsed_headers:
|
|
||||||
kwargs["extra_headers"] = parsed_headers
|
|
||||||
|
|
||||||
from scrapling.fetchers import StealthyFetcher
|
from scrapling.fetchers import StealthyFetcher
|
||||||
|
|
||||||
__Request_and_Save(StealthyFetcher.fetch, url, output_file, css_selector, **kwargs)
|
__Request_and_Save(StealthyFetcher.fetch, url, output_file, css_selector, **kwargs)
|
||||||
|
|||||||
Reference in New Issue
Block a user