refactor(fetchers)!: Replace Camoufox with patchright and many optimizations

- DynamicFetcher became 20% faster
- StealthyFetcher became 99% faster
- Scrapling size decreased
- Code became ~400 lines shorter
- Most importantly, scrapling is more stable and reliable now.
- Less confusing for new users.
- More...
This commit is contained in:
Karim shoair
2025-12-26 03:21:13 +02:00
parent f9f34e6ee9
commit cfc667f7dc
12 changed files with 469 additions and 707 deletions
+18 -51
View File
@@ -125,14 +125,9 @@ def install(force): # pragma: no cover
"playwright",
"install-deps",
"chromium",
"firefox",
],
"Playwright dependencies",
)
__Execute(
[python_executable, "-m", "camoufox", "fetch", "--browserforge"],
"Camoufox browser and databases",
)
# if no errors raised by the above commands, then we add the below file
__PACKAGE_DIR__.joinpath(".scrapling_dependencies_installed").touch()
else:
@@ -611,16 +606,10 @@ def delete(
)
@option("--wait-selector", help="CSS selector to wait for before proceeding")
@option("--locale", default="en-US", help="Browser locale (default: en-US)")
@option("--stealth/--no-stealth", default=False, help="Enable stealth mode (default: False)")
@option(
"--hide-canvas/--show-canvas",
"--real-chrome/--no-real-chrome",
default=False,
help="Add noise to canvas operations (default: False)",
)
@option(
"--disable-webgl/--enable-webgl",
default=False,
help="Disable WebGL support (default: False)",
help="If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch an instance of your browser and use it. (default: False)",
)
@option("--proxy", help='Proxy URL in format "http://username:password@host:port"')
@option(
@@ -640,9 +629,7 @@ def fetch(
css_selector,
wait_selector,
locale,
stealth,
hide_canvas,
disable_webgl,
real_chrome,
proxy,
extra_headers,
):
@@ -659,9 +646,7 @@ def fetch(
:param css_selector: CSS selector to extract specific content.
:param wait_selector: Wait for a specific CSS selector to be in a specific state.
:param locale: Set the locale for the browser.
:param stealth: Enables stealth mode.
:param hide_canvas: Add random noise to canvas operations to prevent fingerprinting.
:param disable_webgl: Disables WebGL and WebGL 2.0 support entirely.
:param real_chrome: If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch an instance of your browser and use it.
:param proxy: The proxy to be used with requests.
:param extra_headers: Extra headers to add to the request.
"""
@@ -676,9 +661,7 @@ def fetch(
"network_idle": network_idle,
"timeout": timeout,
"locale": locale,
"stealth": stealth,
"hide_canvas": hide_canvas,
"disable_webgl": disable_webgl,
"real_chrome": real_chrome,
}
if wait > 0:
@@ -703,11 +686,6 @@ def fetch(
default=True,
help="Run browser in headless mode (default: True)",
)
@option(
"--block-images/--allow-images",
default=False,
help="Block image loading (default: False)",
)
@option(
"--disable-resources/--enable-resources",
default=False,
@@ -718,11 +696,6 @@ def fetch(
default=False,
help="Block WebRTC entirely (default: False)",
)
@option(
"--humanize/--no-humanize",
default=False,
help="Humanize cursor movement (default: False)",
)
@option(
"--solve-cloudflare/--no-solve-cloudflare",
default=False,
@@ -735,9 +708,14 @@ def fetch(
help="Wait for network idle (default: False)",
)
@option(
"--disable-ads/--allow-ads",
"--real-chrome/--no-real-chrome",
default=False,
help="Install uBlock Origin addon (default: False)",
help="If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch an instance of your browser and use it. (default: False)",
)
@option(
"--hide-canvas/--show-canvas",
default=False,
help="Add noise to canvas operations (default: False)",
)
@option(
"--timeout",
@@ -757,11 +735,6 @@ def fetch(
help="CSS selector to extract specific content from the page. It returns all matches.",
)
@option("--wait-selector", help="CSS selector to wait for before proceeding")
@option(
"--geoip/--no-geoip",
default=False,
help="Use IP geolocation for timezone/locale (default: False)",
)
@option("--proxy", help='Proxy URL in format "http://username:password@host:port"')
@option(
"--extra-headers",
@@ -773,19 +746,17 @@ def stealthy_fetch(
url,
output_file,
headless,
block_images,
disable_resources,
block_webrtc,
humanize,
solve_cloudflare,
allow_webgl,
network_idle,
disable_ads,
real_chrome,
hide_canvas,
timeout,
wait,
css_selector,
wait_selector,
geoip,
proxy,
extra_headers,
):
@@ -795,19 +766,17 @@ def stealthy_fetch(
:param url: Target url.
:param output_file: Output file path (.md for Markdown, .html for HTML).
:param headless: Run the browser in headless/hidden, or headful/visible mode.
:param block_images: Prevent the loading of images through Firefox preferences.
:param disable_resources: Drop requests of unnecessary resources for a speed boost.
:param block_webrtc: Blocks WebRTC entirely.
:param humanize: Humanize the cursor movement.
:param solve_cloudflare: Solves all types of the Cloudflare's Turnstile/Interstitial challenges.
:param allow_webgl: Allow WebGL (recommended to keep enabled).
:param network_idle: Wait for the page until there are no network connections for at least 500 ms.
:param disable_ads: Install the uBlock Origin addon on the browser.
:param real_chrome: If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch an instance of your browser and use it.
:param hide_canvas: Add random noise to canvas operations to prevent fingerprinting.
:param timeout: The timeout in milliseconds that is used in all operations and waits through the page.
:param wait: The time (milliseconds) the fetcher will wait after everything finishes before returning.
:param css_selector: CSS selector to extract specific content.
:param wait_selector: Wait for a specific CSS selector to be in a specific state.
:param geoip: Automatically use IP's longitude, latitude, timezone, country, locale.
:param proxy: The proxy to be used with requests.
:param extra_headers: Extra headers to add to the request.
"""
@@ -818,16 +787,14 @@ def stealthy_fetch(
# Build request arguments
kwargs = {
"headless": headless,
"block_images": block_images,
"disable_resources": disable_resources,
"block_webrtc": block_webrtc,
"humanize": humanize,
"solve_cloudflare": solve_cloudflare,
"allow_webgl": allow_webgl,
"network_idle": network_idle,
"disable_ads": disable_ads,
"real_chrome": real_chrome,
"hide_canvas": hide_canvas,
"timeout": timeout,
"geoip": geoip,
}
if wait > 0: