diff --git a/scrapling/cli.py b/scrapling/cli.py index 99a2c63..122fd3d 100644 --- a/scrapling/cli.py +++ b/scrapling/cli.py @@ -125,14 +125,9 @@ def install(force): # pragma: no cover "playwright", "install-deps", "chromium", - "firefox", ], "Playwright dependencies", ) - __Execute( - [python_executable, "-m", "camoufox", "fetch", "--browserforge"], - "Camoufox browser and databases", - ) # if no errors raised by the above commands, then we add the below file __PACKAGE_DIR__.joinpath(".scrapling_dependencies_installed").touch() else: @@ -611,16 +606,10 @@ def delete( ) @option("--wait-selector", help="CSS selector to wait for before proceeding") @option("--locale", default="en-US", help="Browser locale (default: en-US)") -@option("--stealth/--no-stealth", default=False, help="Enable stealth mode (default: False)") @option( - "--hide-canvas/--show-canvas", + "--real-chrome/--no-real-chrome", default=False, - help="Add noise to canvas operations (default: False)", -) -@option( - "--disable-webgl/--enable-webgl", - default=False, - help="Disable WebGL support (default: False)", + help="If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch an instance of your browser and use it. (default: False)", ) @option("--proxy", help='Proxy URL in format "http://username:password@host:port"') @option( @@ -640,9 +629,7 @@ def fetch( css_selector, wait_selector, locale, - stealth, - hide_canvas, - disable_webgl, + real_chrome, proxy, extra_headers, ): @@ -659,9 +646,7 @@ def fetch( :param css_selector: CSS selector to extract specific content. :param wait_selector: Wait for a specific CSS selector to be in a specific state. :param locale: Set the locale for the browser. - :param stealth: Enables stealth mode. - :param hide_canvas: Add random noise to canvas operations to prevent fingerprinting. - :param disable_webgl: Disables WebGL and WebGL 2.0 support entirely. + :param real_chrome: If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch an instance of your browser and use it. :param proxy: The proxy to be used with requests. :param extra_headers: Extra headers to add to the request. """ @@ -676,9 +661,7 @@ def fetch( "network_idle": network_idle, "timeout": timeout, "locale": locale, - "stealth": stealth, - "hide_canvas": hide_canvas, - "disable_webgl": disable_webgl, + "real_chrome": real_chrome, } if wait > 0: @@ -703,11 +686,6 @@ def fetch( default=True, help="Run browser in headless mode (default: True)", ) -@option( - "--block-images/--allow-images", - default=False, - help="Block image loading (default: False)", -) @option( "--disable-resources/--enable-resources", default=False, @@ -718,11 +696,6 @@ def fetch( default=False, help="Block WebRTC entirely (default: False)", ) -@option( - "--humanize/--no-humanize", - default=False, - help="Humanize cursor movement (default: False)", -) @option( "--solve-cloudflare/--no-solve-cloudflare", default=False, @@ -735,9 +708,14 @@ def fetch( help="Wait for network idle (default: False)", ) @option( - "--disable-ads/--allow-ads", + "--real-chrome/--no-real-chrome", default=False, - help="Install uBlock Origin addon (default: False)", + help="If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch an instance of your browser and use it. (default: False)", +) +@option( + "--hide-canvas/--show-canvas", + default=False, + help="Add noise to canvas operations (default: False)", ) @option( "--timeout", @@ -757,11 +735,6 @@ def fetch( help="CSS selector to extract specific content from the page. It returns all matches.", ) @option("--wait-selector", help="CSS selector to wait for before proceeding") -@option( - "--geoip/--no-geoip", - default=False, - help="Use IP geolocation for timezone/locale (default: False)", -) @option("--proxy", help='Proxy URL in format "http://username:password@host:port"') @option( "--extra-headers", @@ -773,19 +746,17 @@ def stealthy_fetch( url, output_file, headless, - block_images, disable_resources, block_webrtc, - humanize, solve_cloudflare, allow_webgl, network_idle, - disable_ads, + real_chrome, + hide_canvas, timeout, wait, css_selector, wait_selector, - geoip, proxy, extra_headers, ): @@ -795,19 +766,17 @@ def stealthy_fetch( :param url: Target url. :param output_file: Output file path (.md for Markdown, .html for HTML). :param headless: Run the browser in headless/hidden, or headful/visible mode. - :param block_images: Prevent the loading of images through Firefox preferences. :param disable_resources: Drop requests of unnecessary resources for a speed boost. :param block_webrtc: Blocks WebRTC entirely. - :param humanize: Humanize the cursor movement. :param solve_cloudflare: Solves all types of the Cloudflare's Turnstile/Interstitial challenges. :param allow_webgl: Allow WebGL (recommended to keep enabled). :param network_idle: Wait for the page until there are no network connections for at least 500 ms. - :param disable_ads: Install the uBlock Origin addon on the browser. + :param real_chrome: If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch an instance of your browser and use it. + :param hide_canvas: Add random noise to canvas operations to prevent fingerprinting. :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. :param wait: The time (milliseconds) the fetcher will wait after everything finishes before returning. :param css_selector: CSS selector to extract specific content. :param wait_selector: Wait for a specific CSS selector to be in a specific state. - :param geoip: Automatically use IP's longitude, latitude, timezone, country, locale. :param proxy: The proxy to be used with requests. :param extra_headers: Extra headers to add to the request. """ @@ -818,16 +787,14 @@ def stealthy_fetch( # Build request arguments kwargs = { "headless": headless, - "block_images": block_images, "disable_resources": disable_resources, "block_webrtc": block_webrtc, - "humanize": humanize, "solve_cloudflare": solve_cloudflare, "allow_webgl": allow_webgl, "network_idle": network_idle, - "disable_ads": disable_ads, + "real_chrome": real_chrome, + "hide_canvas": hide_canvas, "timeout": timeout, - "geoip": geoip, } if wait > 0: diff --git a/scrapling/core/_types.py b/scrapling/core/_types.py index ac19a91..d7c1f8b 100644 --- a/scrapling/core/_types.py +++ b/scrapling/core/_types.py @@ -57,3 +57,17 @@ except ImportError: # pragma: no cover from typing_extensions import Self # Backport except ImportError: Self = object + + +# Copied from `playwright._impl._api_structures.SetCookieParam` +class SetCookieParam(TypedDict, total=False): + name: str + value: str + url: Optional[str] + domain: Optional[str] + path: Optional[str] + expires: Optional[float] + httpOnly: Optional[bool] + secure: Optional[bool] + sameSite: Optional[Literal["Lax", "None", "Strict"]] + partitionKey: Optional[str] diff --git a/scrapling/core/ai.py b/scrapling/core/ai.py index 4d5277a..ec061f5 100644 --- a/scrapling/core/ai.py +++ b/scrapling/core/ai.py @@ -213,13 +213,11 @@ class ScraplingMCPServer: main_content_only: bool = True, headless: bool = False, google_search: bool = True, - hide_canvas: bool = False, - disable_webgl: bool = False, real_chrome: bool = False, - stealth: bool = False, wait: int | float = 0, proxy: Optional[str | Dict[str, str]] = None, - locale: str = "en-US", + timezone_id: str | None = None, + locale: str | None = None, extra_headers: Optional[Dict[str, str]] = None, useragent: Optional[str] = None, cdp_url: Optional[str] = None, @@ -251,12 +249,11 @@ class ScraplingMCPServer: :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000 :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object. :param wait_selector: Wait for a specific CSS selector to be in a specific state. - :param locale: Set the locale for the browser if wanted. The default value is `en-US`. + :param timezone_id: Changes the timezone of the browser. Defaults to the system timezone. + :param locale: Specify user locale, for example, `en-GB`, `de-DE`, etc. Locale will affect navigator.language value, Accept-Language request header value as well as number and date formatting + rules. Defaults to the system default locale. :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`. - :param stealth: Enables stealth mode, check the documentation to see what stealth mode does currently. :param real_chrome: If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch an instance of your browser and use it. - :param hide_canvas: Add random noise to canvas operations to prevent fingerprinting. - :param disable_webgl: Disables WebGL and WebGL 2.0 support entirely. :param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers through CDP. :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name. :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ @@ -269,15 +266,13 @@ class ScraplingMCPServer: locale=locale, timeout=timeout, cookies=cookies, - stealth=stealth, cdp_url=cdp_url, headless=headless, useragent=useragent, - hide_canvas=hide_canvas, + timezone_id=timezone_id, real_chrome=real_chrome, network_idle=network_idle, wait_selector=wait_selector, - disable_webgl=disable_webgl, extra_headers=extra_headers, google_search=google_search, disable_resources=disable_resources, @@ -301,13 +296,11 @@ class ScraplingMCPServer: main_content_only: bool = True, headless: bool = False, google_search: bool = True, - hide_canvas: bool = False, - disable_webgl: bool = False, real_chrome: bool = False, - stealth: bool = False, wait: int | float = 0, proxy: Optional[str | Dict[str, str]] = None, - locale: str = "en-US", + timezone_id: str | None = None, + locale: str | None = None, extra_headers: Optional[Dict[str, str]] = None, useragent: Optional[str] = None, cdp_url: Optional[str] = None, @@ -339,12 +332,11 @@ class ScraplingMCPServer: :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000 :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object. :param wait_selector: Wait for a specific CSS selector to be in a specific state. - :param locale: Set the locale for the browser if wanted. The default value is `en-US`. + :param timezone_id: Changes the timezone of the browser. Defaults to the system timezone. + :param locale: Specify user locale, for example, `en-GB`, `de-DE`, etc. Locale will affect navigator.language value, Accept-Language request header value as well as number and date formatting + rules. Defaults to the system default locale. :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`. - :param stealth: Enables stealth mode, check the documentation to see what stealth mode does currently. :param real_chrome: If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch an instance of your browser and use it. - :param hide_canvas: Add random noise to canvas operations to prevent fingerprinting. - :param disable_webgl: Disables WebGL and WebGL 2.0 support entirely. :param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers through CDP. :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name. :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ @@ -356,17 +348,15 @@ class ScraplingMCPServer: locale=locale, timeout=timeout, cookies=cookies, - stealth=stealth, cdp_url=cdp_url, headless=headless, max_pages=len(urls), useragent=useragent, - hide_canvas=hide_canvas, + timezone_id=timezone_id, real_chrome=real_chrome, network_idle=network_idle, wait_selector=wait_selector, google_search=google_search, - disable_webgl=disable_webgl, extra_headers=extra_headers, disable_resources=disable_resources, wait_selector_state=wait_selector_state, @@ -393,29 +383,29 @@ class ScraplingMCPServer: css_selector: Optional[str] = None, main_content_only: bool = True, headless: bool = True, # noqa: F821 - block_images: bool = False, + google_search: bool = True, + real_chrome: bool = False, + wait: int | float = 0, + proxy: Optional[str | Dict[str, str]] = None, + timezone_id: str | None = None, + locale: str | None = None, + extra_headers: Optional[Dict[str, str]] = None, + useragent: Optional[str] = None, + hide_canvas: bool = False, + cdp_url: Optional[str] = None, + timeout: int | float = 30000, disable_resources: bool = False, + wait_selector: Optional[str] = None, + cookies: Optional[List[Dict]] = None, + network_idle: bool = False, + wait_selector_state: SelectorWaitStates = "attached", block_webrtc: bool = False, allow_webgl: bool = True, - network_idle: bool = False, - humanize: bool | float = True, solve_cloudflare: bool = False, - wait: int | float = 0, - timeout: int | float = 30000, - wait_selector: Optional[str] = None, - addons: Optional[List[str]] = None, - wait_selector_state: SelectorWaitStates = "attached", - cookies: Optional[List[Dict]] = None, - google_search: bool = True, - extra_headers: Optional[Dict[str, str]] = None, - proxy: Optional[str | Dict[str, str]] = None, - os_randomize: bool = False, - disable_ads: bool = False, - geoip: bool = False, additional_args: Optional[Dict] = None, ) -> ResponseModel: - """Use Scrapling's version of the Camoufox browser to fetch a URL and return a structured output of the result. - Note: This is best suitable for high protection levels. It's slower than the other tools. + """Use the stealthy fetcher to fetch a URL and return a structured output of the result. + Note: This is the only suitable fetcher for high protection levels. Note: If the `css_selector` resolves to more than one element, all the elements will be returned. :param url: The URL to request. @@ -426,54 +416,53 @@ class ScraplingMCPServer: :param css_selector: CSS selector to extract the content from the page. If main_content_only is True, then it will be executed on the main content of the page. Defaults to None. :param main_content_only: Whether to extract only the main content of the page. Defaults to True. The main content here is the data inside the `` tag. :param headless: Run the browser in headless/hidden (default), or headful/visible mode. - :param block_images: Prevent the loading of images through Firefox preferences. - This can help save your proxy usage but be careful with this option as it makes some websites never finish loading. :param disable_resources: Drop requests of unnecessary resources for a speed boost. It depends, but it made requests ~25% faster in my tests for some websites. Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`. This can help save your proxy usage but be careful with this option as it makes some websites never finish loading. - :param block_webrtc: Blocks WebRTC entirely. + :param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it. :param cookies: Set cookies for the next request. - :param addons: List of Firefox addons to use. Must be paths to extracted addons. - :param humanize: Humanize the cursor movement. Takes either True or the MAX duration in seconds of the cursor movement. The cursor typically takes up to 1.5 seconds to move across the window. :param solve_cloudflare: Solves all types of the Cloudflare's Turnstile/Interstitial challenges before returning the response to you. :param allow_webgl: Enabled by default. Disabling WebGL is not recommended as many WAFs now check if WebGL is enabled. :param network_idle: Wait for the page until there are no network connections for at least 500 ms. - :param disable_ads: Disabled by default, this installs the `uBlock Origin` addon on the browser if enabled. - :param os_randomize: If enabled, Scrapling will randomize the OS fingerprints used. The default is Scrapling matching the fingerprints with the current OS. :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object. :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000 :param wait_selector: Wait for a specific CSS selector to be in a specific state. - :param geoip: Recommended to use with proxies; Automatically use IP's longitude, latitude, timezone, country, locale, and spoof the WebRTC IP address. - It will also calculate and spoof the browser's language based on the distribution of language speakers in the target region. + :param timezone_id: Changes the timezone of the browser. Defaults to the system timezone. + :param locale: Specify user locale, for example, `en-GB`, `de-DE`, etc. Locale will affect navigator.language value, Accept-Language request header value as well as number and date formatting + rules. Defaults to the system default locale. :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`. + :param real_chrome: If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch an instance of your browser and use it. + :param hide_canvas: Add random noise to canvas operations to prevent fingerprinting. + :param block_webrtc: Forces WebRTC to respect proxy settings to prevent local IP address leak. + :param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers through CDP. :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name. :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only. - :param additional_args: Additional arguments to be passed to Camoufox as additional settings, and it takes higher priority than Scrapling's settings. + :param additional_args: Additional arguments to be passed to Playwright's context as additional settings, and it takes higher priority than Scrapling's settings. """ page = await StealthyFetcher.async_fetch( url, wait=wait, proxy=proxy, - geoip=geoip, - addons=addons, + locale=locale, + cdp_url=cdp_url, timeout=timeout, cookies=cookies, headless=headless, - humanize=humanize, + useragent=useragent, + timezone_id=timezone_id, + real_chrome=real_chrome, + hide_canvas=hide_canvas, allow_webgl=allow_webgl, - disable_ads=disable_ads, network_idle=network_idle, - block_images=block_images, block_webrtc=block_webrtc, - os_randomize=os_randomize, wait_selector=wait_selector, google_search=google_search, extra_headers=extra_headers, + additional_args=additional_args, solve_cloudflare=solve_cloudflare, disable_resources=disable_resources, wait_selector_state=wait_selector_state, - additional_args=additional_args, ) return _ContentTranslator( Convertor._extract_content( @@ -492,29 +481,29 @@ class ScraplingMCPServer: css_selector: Optional[str] = None, main_content_only: bool = True, headless: bool = True, # noqa: F821 - block_images: bool = False, + google_search: bool = True, + real_chrome: bool = False, + wait: int | float = 0, + proxy: Optional[str | Dict[str, str]] = None, + timezone_id: str | None = None, + locale: str | None = None, + extra_headers: Optional[Dict[str, str]] = None, + useragent: Optional[str] = None, + hide_canvas: bool = False, + cdp_url: Optional[str] = None, + timeout: int | float = 30000, disable_resources: bool = False, + wait_selector: Optional[str] = None, + cookies: Optional[List[Dict]] = None, + network_idle: bool = False, + wait_selector_state: SelectorWaitStates = "attached", block_webrtc: bool = False, allow_webgl: bool = True, - network_idle: bool = False, - humanize: bool | float = True, solve_cloudflare: bool = False, - wait: int | float = 0, - timeout: int | float = 30000, - wait_selector: Optional[str] = None, - addons: Optional[List[str]] = None, - wait_selector_state: SelectorWaitStates = "attached", - cookies: Optional[List[Dict]] = None, - google_search: bool = True, - extra_headers: Optional[Dict[str, str]] = None, - proxy: Optional[str | Dict[str, str]] = None, - os_randomize: bool = False, - disable_ads: bool = False, - geoip: bool = False, additional_args: Optional[Dict] = None, ) -> List[ResponseModel]: - """Use Scrapling's version of the Camoufox browser to fetch a group of URLs at the same time, and for each page return a structured output of the result. - Note: This is best suitable for high protection levels. It's slower than the other tools. + """Use the stealthy fetcher to fetch a group of URLs at the same time, and for each page return a structured output of the result. + Note: This is the only suitable fetcher for high protection levels. Note: If the `css_selector` resolves to more than one element, all the elements will be returned. :param urls: A tuple of the URLs to request. @@ -525,54 +514,52 @@ class ScraplingMCPServer: :param css_selector: CSS selector to extract the content from the page. If main_content_only is True, then it will be executed on the main content of the page. Defaults to None. :param main_content_only: Whether to extract only the main content of the page. Defaults to True. The main content here is the data inside the `` tag. :param headless: Run the browser in headless/hidden (default), or headful/visible mode. - :param block_images: Prevent the loading of images through Firefox preferences. - This can help save your proxy usage but be careful with this option as it makes some websites never finish loading. :param disable_resources: Drop requests of unnecessary resources for a speed boost. It depends, but it made requests ~25% faster in my tests for some websites. Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`. This can help save your proxy usage but be careful with this option as it makes some websites never finish loading. - :param block_webrtc: Blocks WebRTC entirely. + :param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it. :param cookies: Set cookies for the next request. - :param addons: List of Firefox addons to use. Must be paths to extracted addons. - :param humanize: Humanize the cursor movement. Takes either True or the MAX duration in seconds of the cursor movement. The cursor typically takes up to 1.5 seconds to move across the window. :param solve_cloudflare: Solves all types of the Cloudflare's Turnstile/Interstitial challenges before returning the response to you. :param allow_webgl: Enabled by default. Disabling WebGL is not recommended as many WAFs now check if WebGL is enabled. :param network_idle: Wait for the page until there are no network connections for at least 500 ms. - :param disable_ads: Disabled by default, this installs the `uBlock Origin` addon on the browser if enabled. - :param os_randomize: If enabled, Scrapling will randomize the OS fingerprints used. The default is Scrapling matching the fingerprints with the current OS. :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object. :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000 :param wait_selector: Wait for a specific CSS selector to be in a specific state. - :param geoip: Recommended to use with proxies; Automatically use IP's longitude, latitude, timezone, country, locale, and spoof the WebRTC IP address. - It will also calculate and spoof the browser's language based on the distribution of language speakers in the target region. + :param timezone_id: Changes the timezone of the browser. Defaults to the system timezone. + :param locale: Specify user locale, for example, `en-GB`, `de-DE`, etc. Locale will affect navigator.language value, Accept-Language request header value as well as number and date formatting + rules. Defaults to the system default locale. :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`. + :param real_chrome: If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch an instance of your browser and use it. + :param hide_canvas: Add random noise to canvas operations to prevent fingerprinting. + :param block_webrtc: Forces WebRTC to respect proxy settings to prevent local IP address leak. + :param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers through CDP. :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name. :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only. - :param additional_args: Additional arguments to be passed to Camoufox as additional settings, and it takes higher priority than Scrapling's settings. + :param additional_args: Additional arguments to be passed to Playwright's context as additional settings, and it takes higher priority than Scrapling's settings. """ async with AsyncStealthySession( wait=wait, proxy=proxy, - geoip=geoip, - addons=addons, + locale=locale, + cdp_url=cdp_url, timeout=timeout, cookies=cookies, headless=headless, - humanize=humanize, - max_pages=len(urls), + useragent=useragent, + timezone_id=timezone_id, + real_chrome=real_chrome, + hide_canvas=hide_canvas, allow_webgl=allow_webgl, - disable_ads=disable_ads, - block_images=block_images, - block_webrtc=block_webrtc, network_idle=network_idle, - os_randomize=os_randomize, + block_webrtc=block_webrtc, wait_selector=wait_selector, google_search=google_search, extra_headers=extra_headers, + additional_args=additional_args, solve_cloudflare=solve_cloudflare, disable_resources=disable_resources, wait_selector_state=wait_selector_state, - additional_args=additional_args, ) as session: tasks = [session.fetch(url) for url in urls] responses = await gather(*tasks) diff --git a/scrapling/engines/_browsers/_base.py b/scrapling/engines/_browsers/_base.py index fdb298c..bb14b5f 100644 --- a/scrapling/engines/_browsers/_base.py +++ b/scrapling/engines/_browsers/_base.py @@ -1,7 +1,6 @@ from time import time from asyncio import sleep as asyncio_sleep, Lock -from camoufox import DefaultAddons from playwright.sync_api._generated import Page from playwright.sync_api import ( Frame, @@ -17,18 +16,18 @@ from playwright.async_api import ( BrowserContext as AsyncBrowserContext, ) from playwright._impl._errors import Error as PlaywrightError -from camoufox.pkgman import installed_verstr as camoufox_version -from camoufox.utils import launch_options as generate_launch_options from ._page import PageInfo, PagePool from scrapling.parser import Selector -from scrapling.core._types import Any, cast, Dict, List, Optional, Callable, TYPE_CHECKING -from scrapling.engines.toolbelt.fingerprints import get_os_name -from ._validators import validate, PlaywrightConfig, CamoufoxConfig -from ._config_tools import _compiled_stealth_scripts, _launch_kwargs, _context_kwargs +from ._validators import validate, PlaywrightConfig, StealthConfig +from ._config_tools import __default_chrome_useragent__, __default_useragent__ from scrapling.engines.toolbelt.navigation import intercept_route, async_intercept_route - -__ff_version_str__ = camoufox_version().split(".", 1)[0] +from scrapling.core._types import Any, cast, Dict, List, Optional, Callable, TYPE_CHECKING, overload, Tuple +from scrapling.engines.constants import ( + DEFAULT_STEALTH_FLAGS, + HARMFUL_DEFAULT_ARGS, + DEFAULT_FLAGS, +) class SyncSession: @@ -84,10 +83,6 @@ class SyncSession: if disable_resources: page.route("**/*", intercept_route) - if getattr(self, "stealth", False): - for script in _compiled_stealth_scripts(): - page.add_init_script(script=script) - page_info = self.page_pool.add_page(page) page_info.mark_busy() return page_info @@ -202,10 +197,6 @@ class AsyncSession: if disable_resources: await page.route("**/*", async_intercept_route) - if getattr(self, "stealth", False): - for script in _compiled_stealth_scripts(): - await page.add_init_script(script=script) - return self.page_pool.add_page(page) def get_pool_stats(self) -> Dict[str, int]: @@ -251,151 +242,118 @@ class AsyncSession: return handle_response -class DynamicSessionMixin: - def __validate__(self, **params): +class BaseSessionMixin: + @overload + def __validate_routine__(self, params: Dict, model: type[StealthConfig]) -> StealthConfig: ... + + @overload + def __validate_routine__(self, params: Dict, model: type[PlaywrightConfig]) -> PlaywrightConfig: ... + + def __validate_routine__( + self, params: Dict, model: type[PlaywrightConfig] | type[StealthConfig] + ) -> PlaywrightConfig | StealthConfig: + # Dark color scheme bypasses the 'prefersLightColor' check in creepjs + self._context_options: Dict[str, Any] = {"color_scheme": "dark", "device_scale_factor": 2} + self._launch_options: Dict[str, Any] = self._context_options | { + "args": DEFAULT_FLAGS, + "ignore_default_args": HARMFUL_DEFAULT_ARGS, + } if "__max_pages" in params: params["max_pages"] = params.pop("__max_pages") - config = validate(params, model=PlaywrightConfig) + config = validate(params, model=model) + self._headers_keys = ( + {header.lower() for header in config.extra_headers.keys()} if config.extra_headers else set() + ) - self._max_pages = config.max_pages - self._headless = config.headless - self._hide_canvas = config.hide_canvas - self._disable_webgl = config.disable_webgl - self._real_chrome = config.real_chrome - self._stealth = config.stealth - self._google_search = config.google_search - self._wait = config.wait - self._proxy = config.proxy - self._locale = config.locale - self._extra_headers = config.extra_headers - self._useragent = config.useragent - self._timeout = config.timeout - self._cookies = config.cookies - self._disable_resources = config.disable_resources - self._cdp_url = config.cdp_url - self._network_idle = config.network_idle - self._load_dom = config.load_dom - self._wait_selector = config.wait_selector - self._init_script = config.init_script - self._wait_selector_state = config.wait_selector_state - self._extra_flags = config.extra_flags - self._selector_config = config.selector_config - self._timezone_id = config.timezone_id - self._additional_args = config.additional_args - self._page_action = config.page_action - self._user_data_dir = config.user_data_dir - self._headers_keys = {header.lower() for header in self._extra_headers.keys()} if self._extra_headers else set() - self.__initiate_browser_options__() + return config - def __initiate_browser_options__(self): - if TYPE_CHECKING: - assert isinstance(self._proxy, tuple) - - if not self._cdp_url: - # `launch_options` is used with persistent context - self.launch_options = dict( - _launch_kwargs( - self._headless, - self._proxy, - self._locale, - tuple(self._extra_headers.items()) if self._extra_headers else tuple(), - self._useragent, - self._real_chrome, - self._stealth, - self._hide_canvas, - self._disable_webgl, - self._timezone_id, - tuple(self._extra_flags) if self._extra_flags else tuple(), - ) - ) - self.launch_options["extra_http_headers"] = dict(self.launch_options["extra_http_headers"]) - self.launch_options["proxy"] = dict(self.launch_options["proxy"]) or None - self.launch_options["user_data_dir"] = self._user_data_dir - self.launch_options.update(cast(Dict, self._additional_args)) - self.context_options = dict() - else: - # while `context_options` is left to be used when cdp mode is enabled - self.launch_options = dict() - self.context_options = dict( - _context_kwargs( - self._proxy, - self._locale, - tuple(self._extra_headers.items()) if self._extra_headers else tuple(), - self._useragent, - self._stealth, - ) - ) - self.context_options["extra_http_headers"] = dict(self.context_options["extra_http_headers"]) - self.context_options["proxy"] = dict(self.context_options["proxy"]) or None - self.context_options.update(cast(Dict, self._additional_args)) - - -class StealthySessionMixin: - def __validate__(self, **params): - if "__max_pages" in params: - params["max_pages"] = params.pop("__max_pages") - - config: CamoufoxConfig = validate(params, model=CamoufoxConfig) - - self._max_pages = config.max_pages - self._headless = config.headless - self._block_images = config.block_images - self._disable_resources = config.disable_resources - self._block_webrtc = config.block_webrtc - self._allow_webgl = config.allow_webgl - self._network_idle = config.network_idle - self._load_dom = config.load_dom - self._humanize = config.humanize - self._solve_cloudflare = config.solve_cloudflare - self._wait = config.wait - self._timeout = config.timeout - self._page_action = config.page_action - self._wait_selector = config.wait_selector - self._init_script = config.init_script - self._addons = config.addons - self._wait_selector_state = config.wait_selector_state - self._cookies = config.cookies - self._google_search = config.google_search - self._extra_headers = config.extra_headers - self._proxy = config.proxy - self._os_randomize = config.os_randomize - self._disable_ads = config.disable_ads - self._geoip = config.geoip - self._selector_config = config.selector_config - self._additional_args = config.additional_args - self._user_data_dir = config.user_data_dir - self._headers_keys = {header.lower() for header in self._extra_headers.keys()} if self._extra_headers else set() - self.__initiate_browser_options__() - - def __initiate_browser_options__(self): - """Initiate browser options.""" - self.launch_options: Dict[str, Any] = generate_launch_options( - **{ - "geoip": self._geoip, - "proxy": dict(self._proxy) if self._proxy and isinstance(self._proxy, tuple) else self._proxy, - "addons": self._addons, - "exclude_addons": [] if self._disable_ads else [DefaultAddons.UBO], - "headless": self._headless, - "humanize": True if self._solve_cloudflare else self._humanize, - "i_know_what_im_doing": True, # To turn warnings off with the user configurations - "allow_webgl": self._allow_webgl, - "block_webrtc": self._block_webrtc, - "block_images": self._block_images, # Careful! it makes some websites don't finish loading at all like stackoverflow even in headful mode. - "os": None if self._os_randomize else get_os_name(), - "user_data_dir": self._user_data_dir, - "ff_version": __ff_version_str__, - "firefox_user_prefs": { - # This is what enabling `enable_cache` does internally, so we do it from here instead - "browser.sessionhistory.max_entries": 10, - "browser.sessionhistory.max_total_viewers": -1, - "browser.cache.memory.enable": True, - "browser.cache.disk_cache_ssl": True, - "browser.cache.disk.smart_size.enabled": True, - }, - **cast(Dict, self._additional_args), + def __generate_options__(self, extra_flags: Tuple | None = None) -> None: + config = cast(PlaywrightConfig, getattr(self, "_config", None)) + self._context_options.update( + { + "proxy": config.proxy, + "locale": config.locale, + "timezone_id": config.timezone_id, + "extra_http_headers": config.extra_headers, } ) + # The default useragent in the headful is always correct now in the current versions of Playwright + if config.useragent: + self._context_options["user_agent"] = config.useragent + elif not config.useragent and config.headless: + self._context_options["user_agent"] = ( + __default_chrome_useragent__ if config.real_chrome else __default_useragent__ + ) + + if not config.cdp_url: + self._launch_options |= self._context_options + self._context_options = {} + flags = self._launch_options["args"] + if config.extra_flags or extra_flags: + flags = list(set(flags + (config.extra_flags or extra_flags))) + + self._launch_options.update( + { + "args": flags, + "headless": config.headless, + "user_data_dir": config.user_data_dir, + "channel": "chrome" if config.real_chrome else "chromium", + } + ) + + if config.additional_args: + self._launch_options.update(config.additional_args) + else: + # while `context_options` is left to be used when cdp mode is enabled + self._launch_options = dict() + if config.additional_args: + self._context_options.update(config.additional_args) + + +class DynamicSessionMixin(BaseSessionMixin): + def __validate__(self, **params): + self._config = self.__validate_routine__(params, model=PlaywrightConfig) + self.__generate_options__() + + +class StealthySessionMixin(BaseSessionMixin): + def __validate__(self, **params): + self._config: StealthConfig = self.__validate_routine__(params, model=StealthConfig) + self._context_options.update( + { + "is_mobile": False, + "has_touch": False, + # I'm thinking about disabling it to rest from all Service Workers' headache, but let's keep it as it is for now + "service_workers": "allow", + "ignore_https_errors": True, + "screen": {"width": 1920, "height": 1080}, + "viewport": {"width": 1920, "height": 1080}, + "permissions": ["geolocation", "notifications"], + } + ) + self.__generate_stealth_options() + + def __generate_stealth_options(self) -> None: + flags = tuple() + if not self._config.cdp_url: + flags = DEFAULT_FLAGS + DEFAULT_STEALTH_FLAGS + + if self._config.block_webrtc: + flags += ( + "--webrtc-ip-handling-policy=disable_non_proxied_udp", + "--force-webrtc-ip-handling-policy", # Ensures the policy is enforced + ) + if not self._config.allow_webgl: + flags += ( + "--disable-webgl", + "--disable-webgl-image-chromium", + "--disable-webgl2", + ) + if self._config.hide_canvas: + flags += ("--fingerprinting-canvas-image-data-noise",) + + super(StealthySessionMixin, self).__generate_options__(flags) @staticmethod def _detect_cloudflare(page_content: str) -> str | None: diff --git a/scrapling/engines/_browsers/_config_tools.py b/scrapling/engines/_browsers/_config_tools.py index f1f4171..b2501ec 100644 --- a/scrapling/engines/_browsers/_config_tools.py +++ b/scrapling/engines/_browsers/_config_tools.py @@ -58,88 +58,3 @@ def _set_flags(hide_canvas, disable_webgl): # pragma: no cover ) return flags - - -@lru_cache(2, typed=True) -def _launch_kwargs( - headless, - proxy: Tuple, - locale, - extra_headers, - useragent, - real_chrome, - stealth, - hide_canvas, - disable_webgl, - timezone_id, - extra_flags: Tuple, -) -> Tuple: - """Creates the arguments we will use while launching playwright's browser""" - base_args = DEFAULT_FLAGS - if extra_flags: - base_args = base_args + extra_flags - - launch_kwargs = { - "locale": locale, - "timezone_id": timezone_id or None, - "headless": headless, - "args": base_args, - "color_scheme": "dark", # Bypasses the 'prefersLightColor' check in creepjs - "proxy": proxy or tuple(), - "device_scale_factor": 2, - "ignore_default_args": HARMFUL_DEFAULT_ARGS, - "channel": "chrome" if real_chrome else "chromium", - "extra_http_headers": extra_headers or tuple(), - } - # The default useragent in the headful is always correct now in the current versions of Playwright - if useragent: - launch_kwargs["user_agent"] = useragent - elif not useragent and headless: - launch_kwargs["user_agent"] = __default_chrome_useragent__ if real_chrome else __default_useragent__ - - if stealth: - stealth_args = base_args + _set_flags(hide_canvas, disable_webgl) - launch_kwargs.update( - { - "args": stealth_args, - "chromium_sandbox": True, - "is_mobile": False, - "has_touch": False, - # I'm thinking about disabling it to rest from all Service Workers' headache, but let's keep it as it is for now - "service_workers": "allow", - "ignore_https_errors": True, - "screen": {"width": 1920, "height": 1080}, - "viewport": {"width": 1920, "height": 1080}, - "permissions": ["geolocation", "notifications"], - } - ) - - return tuple(launch_kwargs.items()) - - -@lru_cache(2, typed=True) -def _context_kwargs(proxy, locale, extra_headers, useragent, stealth) -> Tuple: - """Creates the arguments for the browser context""" - context_kwargs = { - "proxy": proxy or tuple(), - "locale": locale, - "color_scheme": "dark", # Bypasses the 'prefersLightColor' check in creepjs - "device_scale_factor": 2, - "extra_http_headers": extra_headers or tuple(), - "user_agent": useragent or __default_useragent__, - } - if stealth: - context_kwargs.update( - { - "is_mobile": False, - "has_touch": False, - # I'm thinking about disabling it to rest from all Service Workers' headache, but let's keep it as it is for now - "service_workers": "allow", - "ignore_https_errors": True, - "screen": {"width": 1920, "height": 1080}, - "viewport": {"width": 1920, "height": 1080}, - "permissions": ["geolocation", "notifications"], - } - ) - - return tuple(context_kwargs.items()) diff --git a/scrapling/engines/_browsers/_controllers.py b/scrapling/engines/_browsers/_controllers.py index 9e48675..4474a45 100644 --- a/scrapling/engines/_browsers/_controllers.py +++ b/scrapling/engines/_browsers/_controllers.py @@ -9,8 +9,6 @@ from playwright.async_api import ( Playwright as AsyncPlaywright, BrowserContext as AsyncBrowserContext, ) -from patchright.sync_api import sync_playwright as sync_patchright -from patchright.async_api import async_playwright as async_patchright from scrapling.core.utils import log from scrapling.core._types import Unpack, TYPE_CHECKING @@ -21,44 +19,19 @@ from scrapling.engines.toolbelt.convertor import Response, ResponseFactory from scrapling.engines.toolbelt.fingerprints import generate_convincing_referer -class DynamicSession(DynamicSessionMixin, SyncSession): +class DynamicSession(SyncSession, DynamicSessionMixin): """A Browser session manager with page pooling.""" __slots__ = ( - "_max_pages", - "_headless", - "_hide_canvas", - "_disable_webgl", - "_real_chrome", - "_stealth", - "_google_search", - "_proxy", - "_locale", - "_extra_headers", - "_useragent", - "_timeout", - "_cookies", - "_disable_resources", - "_network_idle", - "_load_dom", - "_wait_selector", - "_init_script", - "_wait_selector_state", - "_wait", - "playwright", - "browser", - "context", + "_config", + "_context_options", + "_launch_options", + "max_pages", "page_pool", + "_max_wait_for_page", + "playwright", + "context", "_closed", - "_selector_config", - "_page_action", - "launch_options", - "context_options", - "_cdp_url", - "_headers_keys", - "_extra_flags", - "_additional_args", - "_user_data_dir", ) def __init__(self, **kwargs: Unpack[PlaywrightSession]): @@ -76,8 +49,9 @@ class DynamicSession(DynamicSessionMixin, SyncSession): :param page_action: Added for automation. A function that takes the `page` object and does the automation you need. :param wait_selector: Wait for a specific CSS selector to be in a specific state. :param init_script: An absolute path to a JavaScript file to be executed on page creation for all pages in this session. - :param locale: Set the locale for the browser if wanted. The default value is `en-US`. - :param timezone_id: Set the timezone for the browser if wanted. + :param locale: Specify user locale, for example, `en-GB`, `de-DE`, etc. Locale will affect navigator.language value, Accept-Language request header value as well as number and date formatting + rules. Defaults to the system default locale. + :param timezone_id: Changes the timezone of the browser. Defaults to the system timezone. :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`. :param stealth: Enables stealth mode, check the documentation to see what stealth mode does currently. :param real_chrome: If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch an instance of your browser and use it. @@ -94,27 +68,24 @@ class DynamicSession(DynamicSessionMixin, SyncSession): :param additional_args: Additional arguments to be passed to Playwright's context as additional settings, and it takes higher priority than Scrapling's settings. """ self.__validate__(**kwargs) - super().__init__(max_pages=self._max_pages) + super().__init__() def start(self): """Create a browser for this instance and context.""" if not self.playwright: - sync_context = sync_patchright if self._stealth else sync_playwright + self.playwright: Playwright = sync_playwright().start() # pyright: ignore [reportAttributeAccessIssue] - self.playwright: Playwright = sync_context().start() # pyright: ignore [reportAttributeAccessIssue] - - if self._cdp_url: # pragma: no cover - self.context = self.playwright.chromium.connect_over_cdp(endpoint_url=self._cdp_url).new_context( - **self.context_options - ) + if self._config.cdp_url: # pragma: no cover + browser = self.playwright.chromium.connect_over_cdp(endpoint_url=self._config.cdp_url) + self.context = browser.new_context(**self._context_options) else: - self.context = self.playwright.chromium.launch_persistent_context(**self.launch_options) + self.context = self.playwright.chromium.launch_persistent_context(**self._launch_options) - if self._init_script: # pragma: no cover - self.context.add_init_script(path=self._init_script) + if self._config.init_script: # pragma: no cover + self.context.add_init_script(path=self._config.init_script) - if self._cookies: # pragma: no cover - self.context.add_cookies(self._cookies) + if self._config.cookies: # pragma: no cover + self.context.add_cookies(self._config.cookies) else: raise RuntimeError("Session has been already started") @@ -139,7 +110,6 @@ class DynamicSession(DynamicSessionMixin, SyncSession): :return: A `Response` object. """ params = _validate(kwargs, self, PlaywrightConfig) - if self._closed: # pragma: no cover raise RuntimeError("Context manager has been closed") @@ -193,7 +163,7 @@ class DynamicSession(DynamicSessionMixin, SyncSession): raise e -class AsyncDynamicSession(DynamicSessionMixin, AsyncSession): +class AsyncDynamicSession(AsyncSession, DynamicSessionMixin): """An async Browser session manager with page pooling, it's using a persistent browser Context by default with a temporary user profile directory.""" def __init__(self, **kwargs: Unpack[PlaywrightSession]): @@ -212,8 +182,9 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession): :param page_action: Added for automation. A function that takes the `page` object and does the automation you need. :param wait_selector: Wait for a specific CSS selector to be in a specific state. :param init_script: An absolute path to a JavaScript file to be executed on page creation for all pages in this session. - :param locale: Set the locale for the browser if wanted. The default value is `en-US`. - :param timezone_id: Set the timezone for the browser if wanted. + :param locale: Specify user locale, for example, `en-GB`, `de-DE`, etc. Locale will affect navigator.language value, Accept-Language request header value as well as number and date formatting + rules. Defaults to the system default locale. + :param timezone_id: Changes the timezone of the browser. Defaults to the system timezone. :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`. :param stealth: Enables stealth mode, check the documentation to see what stealth mode does currently. :param real_chrome: If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch an instance of your browser and use it. @@ -230,28 +201,26 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession): :param additional_args: Additional arguments to be passed to Playwright's context as additional settings, and it takes higher priority than Scrapling's settings. """ self.__validate__(**kwargs) - super().__init__(max_pages=self._max_pages) + super().__init__(max_pages=self._config.max_pages) async def start(self): """Create a browser for this instance and context.""" if not self.playwright: - async_context = async_patchright if self._stealth else async_playwright + self.playwright: AsyncPlaywright = await async_playwright().start() # pyright: ignore [reportAttributeAccessIssue] - self.playwright: AsyncPlaywright = await async_context().start() # pyright: ignore [reportAttributeAccessIssue] - - if self._cdp_url: - browser = await self.playwright.chromium.connect_over_cdp(endpoint_url=self._cdp_url) - self.context: AsyncBrowserContext = await browser.new_context(**self.context_options) + if self._config.cdp_url: + browser = await self.playwright.chromium.connect_over_cdp(endpoint_url=self._config.cdp_url) + self.context: AsyncBrowserContext = await browser.new_context(**self._context_options) else: self.context: AsyncBrowserContext = await self.playwright.chromium.launch_persistent_context( - **self.launch_options + **self._launch_options ) - if self._init_script: # pragma: no cover - await self.context.add_init_script(path=self._init_script) + if self._config.init_script: # pragma: no cover + await self.context.add_init_script(path=self._config.init_script) - if self._cookies: - await self.context.add_cookies(self._cookies) # pyright: ignore + if self._config.cookies: + await self.context.add_cookies(self._config.cookies) # pyright: ignore else: raise RuntimeError("Session has been already started") diff --git a/scrapling/engines/_browsers/_camoufox.py b/scrapling/engines/_browsers/_stealth.py similarity index 75% rename from scrapling/engines/_browsers/_camoufox.py rename to scrapling/engines/_browsers/_stealth.py index e3cbc67..482ae4c 100644 --- a/scrapling/engines/_browsers/_camoufox.py +++ b/scrapling/engines/_browsers/_stealth.py @@ -2,117 +2,102 @@ from random import randint from re import compile as re_compile from playwright.sync_api import ( - Page, Locator, - sync_playwright, + Page, + Playwright, ) from playwright.async_api import ( - async_playwright, Page as async_Page, Locator as AsyncLocator, Playwright as AsyncPlaywright, BrowserContext as AsyncBrowserContext, ) +from patchright.sync_api import sync_playwright +from patchright.async_api import async_playwright from scrapling.core.utils import log -from ._types import CamoufoxSession, CamoufoxFetchParams -from scrapling.core._types import Any, Unpack, TYPE_CHECKING +from scrapling.core._types import Any, Unpack +from ._config_tools import _compiled_stealth_scripts +from ._types import StealthSession, StealthFetchParams from ._base import SyncSession, AsyncSession, StealthySessionMixin -from ._validators import validate_fetch as _validate, CamoufoxConfig +from ._validators import validate_fetch as _validate, StealthConfig from scrapling.engines.toolbelt.convertor import Response, ResponseFactory from scrapling.engines.toolbelt.fingerprints import generate_convincing_referer __CF_PATTERN__ = re_compile("challenges.cloudflare.com/cdn-cgi/challenge-platform/.*") -class StealthySession(StealthySessionMixin, SyncSession): - """A Stealthy session manager with page pooling.""" +class StealthySession(SyncSession, StealthySessionMixin): + """A Stealthy Browser session manager with page pooling.""" __slots__ = ( - "_max_pages", - "_headless", - "_block_images", - "_disable_resources", - "_block_webrtc", - "_allow_webgl", - "_network_idle", - "_load_dom", - "_humanize", - "_solve_cloudflare", - "_wait", - "_timeout", - "_page_action", - "_wait_selector", - "_init_script", - "_addons", - "_wait_selector_state", - "_cookies", - "_google_search", - "_extra_headers", - "_proxy", - "_os_randomize", - "_disable_ads", - "_geoip", - "_selector_config", - "_additional_args", - "playwright", - "browser", - "context", + "_config", + "_context_options", + "_launch_options", + "max_pages", "page_pool", + "_max_wait_for_page", + "playwright", + "context", "_closed", - "launch_options", - "_headers_keys", - "_user_data_dir", ) - def __init__(self, **kwargs: Unpack[CamoufoxSession]): - """A Browser session manager with page pooling + def __init__(self, **kwargs: Unpack[StealthSession]): + """A Browser session manager with page pooling, it's using a persistent browser Context by default with a temporary user profile directory. :param headless: Run the browser in headless/hidden (default), or headful/visible mode. - :param block_images: Prevent the loading of images through Firefox preferences. - This can help save your proxy usage but be careful with this option as it makes some websites never finish loading. :param disable_resources: Drop requests of unnecessary resources for a speed boost. It depends, but it made requests ~25% faster in my tests for some websites. Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`. This can help save your proxy usage but be careful with this option as it makes some websites never finish loading. - :param block_webrtc: Blocks WebRTC entirely. + :param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it. :param cookies: Set cookies for the next request. - :param addons: List of Firefox addons to use. Must be paths to extracted addons. - :param humanize: Humanize the cursor movement. Takes either True or the MAX duration in seconds of the cursor movement. The cursor typically takes up to 1.5 seconds to move across the window. - :param solve_cloudflare: Solves all types of the Cloudflare's Turnstile/Interstitial challenges before returning the response to you. - :param allow_webgl: Enabled by default. Disabling WebGL is not recommended as many WAFs now check if WebGL is enabled. :param network_idle: Wait for the page until there are no network connections for at least 500 ms. - :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute. - :param disable_ads: Disabled by default, this installs the `uBlock Origin` addon on the browser if enabled. - :param os_randomize: If enabled, Scrapling will randomize the OS fingerprints used. The default is Scrapling matching the fingerprints with the current OS. - :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object. :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000 + :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object. :param page_action: Added for automation. A function that takes the `page` object and does the automation you need. :param wait_selector: Wait for a specific CSS selector to be in a specific state. :param init_script: An absolute path to a JavaScript file to be executed on page creation for all pages in this session. - :param geoip: Recommended to use with proxies; Automatically use IP's longitude, latitude, timezone, country, locale, and spoof the WebRTC IP address. - It will also calculate and spoof the browser's language based on the distribution of language speakers in the target region. + :param locale: Specify user locale, for example, `en-GB`, `de-DE`, etc. Locale will affect navigator.language value, Accept-Language request header value as well as number and date formatting + rules. Defaults to the system default locale. + :param timezone_id: Changes the timezone of the browser. Defaults to the system timezone. :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`. + :param solve_cloudflare: Solves all types of the Cloudflare's Turnstile/Interstitial challenges before returning the response to you. + :param real_chrome: If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch an instance of your browser and use it. + :param hide_canvas: Add random noise to canvas operations to prevent fingerprinting. + :param block_webrtc: Forces WebRTC to respect proxy settings to prevent local IP address leak. + :param allow_webgl: Enabled by default. Disabling it disables WebGL and WebGL 2.0 support entirely. Disabling WebGL is not recommended as many WAFs now check if WebGL is enabled. + :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute. + :param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers through CDP. :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name. :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only. :param user_data_dir: Path to a User Data Directory, which stores browser session data like cookies and local storage. The default is to create a temporary directory. + :param extra_flags: A list of additional browser flags to pass to the browser on launch. :param selector_config: The arguments that will be passed in the end while creating the final Selector's class. - :param additional_args: Additional arguments to be passed to Camoufox as additional settings, and it takes higher priority than Scrapling's settings. + :param additional_args: Additional arguments to be passed to Playwright's context as additional settings, and it takes higher priority than Scrapling's settings. """ self.__validate__(**kwargs) - super().__init__(max_pages=self._max_pages) + super().__init__() def start(self): """Create a browser for this instance and context.""" if not self.playwright: - self.playwright = sync_playwright().start() - self.context = self.playwright.firefox.launch_persistent_context(**self.launch_options) + self.playwright: Playwright = sync_playwright().start() # pyright: ignore [reportAttributeAccessIssue] - if self._init_script: # pragma: no cover - self.context.add_init_script(path=self._init_script) + if self._config.cdp_url: # pragma: no cover + browser = self.playwright.chromium.connect_over_cdp(endpoint_url=self._config.cdp_url) + self.context = browser.new_context(**self._context_options) + else: + self.context = self.playwright.chromium.launch_persistent_context(**self._launch_options) - if self._cookies: # pragma: no cover - self.context.add_cookies(self._cookies) + for script in _compiled_stealth_scripts(): + self.context.add_init_script(script=script) + + if self._config.init_script: # pragma: no cover + self.context.add_init_script(path=self._config.init_script) + + if self._config.cookies: # pragma: no cover + self.context.add_cookies(self._config.cookies) else: raise RuntimeError("Session has been already started") @@ -148,22 +133,27 @@ class StealthySession(StealthySessionMixin, SyncSession): outer_box = {} iframe = page.frame(url=__CF_PATTERN__) if iframe is not None: - self._wait_for_page_stability(iframe, True, True) + self._wait_for_page_stability(iframe, True, False) if challenge_type != "embedded": while not iframe.frame_element().is_visible(): # Double-checking that the iframe is loaded page.wait_for_timeout(500) + outer_box: Any = iframe.frame_element().bounding_box() if not iframe or not outer_box: + if "Just a moment..." not in (ResponseFactory._get_page_content(page)): + log.info("Cloudflare captcha is solved") + return + outer_box: Any = page.locator(box_selector).last.bounding_box() # Calculate the Captcha coordinates for any viewport captcha_x, captcha_y = outer_box["x"] + randint(26, 28), outer_box["y"] + randint(25, 27) # Move the mouse to the center of the window, then press and hold the left mouse button - page.mouse.click(captcha_x, captcha_y, delay=60, button="left") + page.mouse.click(captcha_x, captcha_y, delay=randint(100, 200), button="left") self._wait_for_networkidle(page) if iframe is not None: # Wait for the frame to be removed from the page (with 30s timeout = 300 iterations * 100 ms) @@ -182,7 +172,7 @@ class StealthySession(StealthySessionMixin, SyncSession): log.info("Cloudflare captcha is solved") return - def fetch(self, url: str, **kwargs: Unpack[CamoufoxFetchParams]) -> Response: + def fetch(self, url: str, **kwargs: Unpack[StealthFetchParams]) -> Response: """Opens up the browser and do your request based on your chosen options. :param url: The Target url. @@ -203,8 +193,7 @@ class StealthySession(StealthySessionMixin, SyncSession): - selector_config: The arguments that will be passed in the end while creating the final Selector's class. :return: A `Response` object. """ - params = _validate(kwargs, self, CamoufoxConfig) - + params = _validate(kwargs, self, StealthConfig) if self._closed: # pragma: no cover raise RuntimeError("Context manager has been closed") @@ -233,7 +222,7 @@ class StealthySession(StealthySessionMixin, SyncSession): if params.page_action: try: _ = params.page_action(page_info.page) - except Exception as e: + except Exception as e: # pragma: no cover log.error(f"Error executing page_action: {e}") if params.wait_selector: @@ -242,10 +231,12 @@ class StealthySession(StealthySessionMixin, SyncSession): waiter.first.wait_for(state=params.wait_selector_state) # Wait again after waiting for the selector, helpful with protections like Cloudflare self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle) - except Exception as e: + except Exception as e: # pragma: no cover log.error(f"Error waiting for selector {params.wait_selector}: {e}") page_info.page.wait_for_timeout(params.wait) + + # Create response object response = ResponseFactory.from_playwright_response( page_info.page, first_response, final_response[0], params.selector_config ) @@ -256,72 +247,79 @@ class StealthySession(StealthySessionMixin, SyncSession): return response - except Exception as e: # pragma: no cover + except Exception as e: page_info.mark_error() raise e -class AsyncStealthySession(StealthySessionMixin, AsyncSession): - """A Stealthy session manager with page pooling.""" +class AsyncStealthySession(AsyncSession, StealthySessionMixin): + """An async Stealthy Browser session manager with page pooling.""" - def __init__(self, **kwargs: Unpack[CamoufoxSession]): - """A Browser session manager with page pooling + def __init__(self, **kwargs: Unpack[StealthSession]): + """A Browser session manager with page pooling, it's using a persistent browser Context by default with a temporary user profile directory. :param headless: Run the browser in headless/hidden (default), or headful/visible mode. - :param block_images: Prevent the loading of images through Firefox preferences. - This can help save your proxy usage but be careful with this option as it makes some websites never finish loading. :param disable_resources: Drop requests of unnecessary resources for a speed boost. It depends, but it made requests ~25% faster in my tests for some websites. Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`. This can help save your proxy usage but be careful with this option as it makes some websites never finish loading. - :param block_webrtc: Blocks WebRTC entirely. + :param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it. :param cookies: Set cookies for the next request. - :param addons: List of Firefox addons to use. Must be paths to extracted addons. - :param humanize: Humanize the cursor movement. Takes either True or the MAX duration in seconds of the cursor movement. The cursor typically takes up to 1.5 seconds to move across the window. - :param solve_cloudflare: Solves all types of the Cloudflare's Turnstile/Interstitial challenges before returning the response to you. - :param allow_webgl: Enabled by default. Disabling WebGL is not recommended as many WAFs now check if WebGL is enabled. :param network_idle: Wait for the page until there are no network connections for at least 500 ms. - :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute. - :param disable_ads: Disabled by default, this installs the `uBlock Origin` addon on the browser if enabled. - :param os_randomize: If enabled, Scrapling will randomize the OS fingerprints used. The default is Scrapling matching the fingerprints with the current OS. - :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object. :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000 + :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object. :param page_action: Added for automation. A function that takes the `page` object and does the automation you need. :param wait_selector: Wait for a specific CSS selector to be in a specific state. :param init_script: An absolute path to a JavaScript file to be executed on page creation for all pages in this session. - :param geoip: Recommended to use with proxies; Automatically use IP's longitude, latitude, timezone, country, locale, and spoof the WebRTC IP address. - It will also calculate and spoof the browser's language based on the distribution of language speakers in the target region. + :param locale: Specify user locale, for example, `en-GB`, `de-DE`, etc. Locale will affect navigator.language value, Accept-Language request header value as well as number and date formatting + rules. Defaults to the system default locale. + :param timezone_id: Changes the timezone of the browser. Defaults to the system timezone. :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`. + :param solve_cloudflare: Solves all types of the Cloudflare's Turnstile/Interstitial challenges before returning the response to you. + :param real_chrome: If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch an instance of your browser and use it. + :param hide_canvas: Add random noise to canvas operations to prevent fingerprinting. + :param block_webrtc: Forces WebRTC to respect proxy settings to prevent local IP address leak. + :param allow_webgl: Enabled by default. Disabling it disables WebGL and WebGL 2.0 support entirely. Disabling WebGL is not recommended as many WAFs now check if WebGL is enabled. + :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute. + :param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers through CDP. :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name. :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only. - :param max_pages: The maximum number of tabs to be opened at the same time. It will be used in rotation through a PagePool. :param user_data_dir: Path to a User Data Directory, which stores browser session data like cookies and local storage. The default is to create a temporary directory. + :param extra_flags: A list of additional browser flags to pass to the browser on launch. :param selector_config: The arguments that will be passed in the end while creating the final Selector's class. - :param additional_args: Additional arguments to be passed to Camoufox as additional settings, and it takes higher priority than Scrapling's settings. + :param additional_args: Additional arguments to be passed to Playwright's context as additional settings, and it takes higher priority than Scrapling's settings. """ self.__validate__(**kwargs) - super().__init__(max_pages=self._max_pages) + super().__init__(max_pages=self._config.max_pages) async def start(self): """Create a browser for this instance and context.""" if not self.playwright: - self.playwright: AsyncPlaywright = await async_playwright().start() - self.context: AsyncBrowserContext = await self.playwright.firefox.launch_persistent_context( - **self.launch_options - ) + self.playwright: AsyncPlaywright = await async_playwright().start() # pyright: ignore [reportAttributeAccessIssue] - if self._init_script: # pragma: no cover - await self.context.add_init_script(path=self._init_script) + if self._config.cdp_url: + browser = await self.playwright.chromium.connect_over_cdp(endpoint_url=self._config.cdp_url) + self.context: AsyncBrowserContext = await browser.new_context(**self._context_options) + else: + self.context: AsyncBrowserContext = await self.playwright.chromium.launch_persistent_context( + **self._launch_options + ) - if self._cookies: - await self.context.add_cookies(self._cookies) # pyright: ignore [reportArgumentType] + for script in _compiled_stealth_scripts(): + await self.context.add_init_script(script=script) + + if self._config.init_script: # pragma: no cover + await self.context.add_init_script(path=self._config.init_script) + + if self._config.cookies: + await self.context.add_cookies(self._config.cookies) # pyright: ignore else: raise RuntimeError("Session has been already started") - async def _cloudflare_solver(self, page: async_Page): # pragma: no cover - """Solve the cloudflare challenge displayed on the playwright page passed. The async version + async def _cloudflare_solver(self, page: async_Page) -> None: # pragma: no cover + """Solve the cloudflare challenge displayed on the playwright page passed - :param page: The async targeted page + :param page: The targeted page :return: """ await self._wait_for_networkidle(page, timeout=5000) @@ -331,7 +329,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession): return else: log.info(f'The turnstile version discovered is "{challenge_type}"') - if challenge_type == "non-interactive": # pragma: no cover + if challenge_type == "non-interactive": while "Just a moment..." in (await ResponseFactory._get_async_page_content(page)): log.info("Waiting for Cloudflare wait page to disappear.") await page.wait_for_timeout(1000) @@ -350,22 +348,27 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession): outer_box = {} iframe = page.frame(url=__CF_PATTERN__) if iframe is not None: - await self._wait_for_page_stability(iframe, True, True) + await self._wait_for_page_stability(iframe, True, False) if challenge_type != "embedded": while not await (await iframe.frame_element()).is_visible(): # Double-checking that the iframe is loaded await page.wait_for_timeout(500) - outer_box: Any = await (await iframe.frame_element()).bounding_box() + + outer_box: Any = (await iframe.frame_element()).bounding_box() if not iframe or not outer_box: + if "Just a moment..." not in (await ResponseFactory._get_async_page_content(page)): + log.info("Cloudflare captcha is solved") + return + outer_box: Any = await page.locator(box_selector).last.bounding_box() # Calculate the Captcha coordinates for any viewport captcha_x, captcha_y = outer_box["x"] + randint(26, 28), outer_box["y"] + randint(25, 27) # Move the mouse to the center of the window, then press and hold the left mouse button - await page.mouse.click(captcha_x, captcha_y, delay=60, button="left") + await page.mouse.click(captcha_x, captcha_y, delay=randint(100, 200), button="left") await self._wait_for_networkidle(page) if iframe is not None: # Wait for the frame to be removed from the page (with 30s timeout = 300 iterations * 100 ms) @@ -377,14 +380,14 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession): await page.wait_for_timeout(100) attempts += 1 if challenge_type != "embedded": - await page.locator(box_selector).wait_for(state="detached") + await page.locator(box_selector).last.wait_for(state="detached") await page.locator(".zone-name-title").wait_for(state="hidden") await self._wait_for_page_stability(page, True, False) log.info("Cloudflare captcha is solved") return - async def fetch(self, url: str, **kwargs: Unpack[CamoufoxFetchParams]) -> Response: + async def fetch(self, url: str, **kwargs: Unpack[StealthFetchParams]) -> Response: """Opens up the browser and do your request based on your chosen options. :param url: The Target url. @@ -405,7 +408,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession): - selector_config: The arguments that will be passed in the end while creating the final Selector's class. :return: A `Response` object. """ - params = _validate(kwargs, self, CamoufoxConfig) + params = _validate(kwargs, self, StealthConfig) if self._closed: # pragma: no cover raise RuntimeError("Context manager has been closed") @@ -418,10 +421,6 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession): final_response = [None] handle_response = self._create_response_handler(page_info, final_response) - if TYPE_CHECKING: - if not isinstance(page_info.page, async_Page): - raise TypeError - try: # Navigate to URL and wait for a specified state page_info.page.on("response", handle_response) @@ -461,9 +460,8 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession): # Close the page to free up resources await page_info.page.close() self.page_pool.pages.remove(page_info) - return response - except Exception as e: + except Exception as e: # pragma: no cover page_info.mark_error() raise e diff --git a/scrapling/engines/_browsers/_types.py b/scrapling/engines/_browsers/_types.py index d1edca5..1cdbb9a 100644 --- a/scrapling/engines/_browsers/_types.py +++ b/scrapling/engines/_browsers/_types.py @@ -53,7 +53,7 @@ if TYPE_CHECKING: # pragma: no cover json: Optional[Dict | List] # Types for browser session - class BrowserSession(TypedDict, total=False): + class PlaywrightSession(TypedDict, total=False): max_pages: int headless: bool disable_resources: bool @@ -64,6 +64,7 @@ if TYPE_CHECKING: # pragma: no cover cookies: Optional[Iterable[Dict]] google_search: bool wait: int | float + timezone_id: str | None page_action: Optional[Callable] proxy: Optional[str | Dict[str, str] | Tuple] extra_headers: Optional[Dict[str, str]] @@ -72,42 +73,32 @@ if TYPE_CHECKING: # pragma: no cover user_data_dir: str selector_config: Optional[Dict] additional_args: Optional[Dict] - - class PlaywrightSession(BrowserSession, total=False): - cdp_url: Optional[str] - hide_canvas: bool - disable_webgl: bool + locale: Optional[str] real_chrome: bool - stealth: bool - locale: str + cdp_url: Optional[str] useragent: Optional[str] extra_flags: Optional[List[str]] class PlaywrightFetchParams(TypedDict, total=False): + load_dom: bool + wait: int | float + network_idle: bool google_search: bool timeout: int | float - wait: int | float - page_action: Optional[Callable] - extra_headers: Optional[Dict[str, str]] disable_resources: bool wait_selector: Optional[str] - wait_selector_state: SelectorWaitStates - network_idle: bool - load_dom: bool + page_action: Optional[Callable] selector_config: Optional[Dict] + extra_headers: Optional[Dict[str, str]] + wait_selector_state: SelectorWaitStates - class CamoufoxSession(BrowserSession, total=False): - block_images: bool - block_webrtc: bool + class StealthSession(PlaywrightSession, total=False): allow_webgl: bool - humanize: bool | float + hide_canvas: bool + block_webrtc: bool solve_cloudflare: bool - addons: Optional[List[str]] - os_randomize: bool - disable_ads: bool - geoip: bool - class CamoufoxFetchParams(PlaywrightFetchParams, total=False): + class StealthFetchParams(PlaywrightFetchParams, total=False): solve_cloudflare: bool else: # pragma: no cover @@ -116,5 +107,5 @@ else: # pragma: no cover DataRequestParams = TypedDict PlaywrightSession = TypedDict PlaywrightFetchParams = TypedDict - CamoufoxSession = TypedDict - CamoufoxFetchParams = TypedDict + StealthSession = TypedDict + StealthFetchParams = TypedDict diff --git a/scrapling/engines/_browsers/_validators.py b/scrapling/engines/_browsers/_validators.py index f8b9875..cb3f48a 100644 --- a/scrapling/engines/_browsers/_validators.py +++ b/scrapling/engines/_browsers/_validators.py @@ -14,11 +14,13 @@ from scrapling.core._types import ( Optional, Callable, Iterable, - SelectorWaitStates, + Sequence, overload, + SetCookieParam, + SelectorWaitStates, ) from scrapling.engines.toolbelt.navigation import construct_proxy_dict -from scrapling.engines._browsers._types import PlaywrightFetchParams, CamoufoxFetchParams +from scrapling.engines._browsers._types import PlaywrightFetchParams, StealthFetchParams # Custom validators for msgspec @@ -68,26 +70,26 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False, weakref=True): cdp_url: Optional[str] = None headless: bool = True google_search: bool = True - hide_canvas: bool = False - disable_webgl: bool = False + # hide_canvas: bool = False + # disable_webgl: bool = False real_chrome: bool = False - stealth: bool = False + # stealth: bool = False wait: Seconds = 0 page_action: Optional[Callable] = None proxy: Optional[str | Dict[str, str] | Tuple] = None # The default value for proxy in Playwright's source is `None` - locale: str = "en-US" + locale: str | None = None extra_headers: Optional[Dict[str, str]] = None useragent: Optional[str] = None timeout: Seconds = 30000 init_script: Optional[str] = None disable_resources: bool = False wait_selector: Optional[str] = None - cookies: Optional[Iterable[Dict]] = None + cookies: Sequence[SetCookieParam] | None = [] network_idle: bool = False load_dom: bool = True wait_selector_state: SelectorWaitStates = "attached" user_data_dir: str = "" - timezone_id: str = "" + timezone_id: str | None = "" extra_flags: Optional[List[str]] = None selector_config: Optional[Dict] = {} additional_args: Optional[Dict] = {} @@ -118,64 +120,18 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False, weakref=True): raise ValueError(validation_msg) -class CamoufoxConfig(Struct, kw_only=True, frozen=False, weakref=True): - """Configuration struct for validation""" - - max_pages: PagesCount = 1 - headless: bool = True # noqa: F821 - block_images: bool = False - disable_resources: bool = False - block_webrtc: bool = False +class StealthConfig(PlaywrightConfig, kw_only=True, frozen=False, weakref=True): allow_webgl: bool = True - network_idle: bool = False - load_dom: bool = True - humanize: bool | float = True + hide_canvas: bool = False + block_webrtc: bool = False solve_cloudflare: bool = False - wait: Seconds = 0 - timeout: Seconds = 30000 - init_script: Optional[str] = None - page_action: Optional[Callable] = None - wait_selector: Optional[str] = None - addons: Optional[List[str]] = None - wait_selector_state: SelectorWaitStates = "attached" - cookies: Optional[Iterable[Dict]] = None - google_search: bool = True - extra_headers: Optional[Dict[str, str]] = None - proxy: Optional[str | Dict[str, str] | Tuple] = None # The default value for proxy in Playwright's source is `None` - os_randomize: bool = False - disable_ads: bool = False - geoip: bool = False - user_data_dir: str = "" - selector_config: Optional[Dict] = {} - additional_args: Optional[Dict] = {} def __post_init__(self): """Custom validation after msgspec validation""" - if self.page_action and not callable(self.page_action): - raise TypeError(f"page_action must be callable, got {type(self.page_action).__name__}") - if self.proxy: - self.proxy = construct_proxy_dict(self.proxy, as_tuple=True) - - if self.addons: - for addon in self.addons: - _validate_addon_path(addon) - else: - self.addons = [] - - if self.init_script is not None: - validation_msg = _is_invalid_file_path(self.init_script) - if validation_msg: - raise ValueError(validation_msg) - - if not self.cookies: - self.cookies = [] + super(StealthConfig, self).__post_init__() # Cloudflare timeout adjustment if self.solve_cloudflare and self.timeout < 60_000: self.timeout = 60_000 - if not self.selector_config: - self.selector_config = {} - if not self.additional_args: - self.additional_args = {} @dataclass @@ -197,9 +153,9 @@ class _fetch_params: def validate_fetch( - method_kwargs: Dict | PlaywrightFetchParams | CamoufoxFetchParams, + method_kwargs: Dict | PlaywrightFetchParams | StealthFetchParams, session: Any, - model: type[PlaywrightConfig] | type[CamoufoxConfig], + model: type[PlaywrightConfig] | type[StealthConfig], ) -> _fetch_params: # pragma: no cover result = {} overrides = {} @@ -210,21 +166,20 @@ def validate_fetch( for key in fetch_param_fields: if key in method_kwargs: overrides[key] = method_kwargs[key] - else: - # Check for underscore-prefixed attribute (private) - attr_name = f"_{key}" - if hasattr(session, attr_name): - result[key] = getattr(session, attr_name) + elif hasattr(session, "_config") and hasattr(session._config, key): + result[key] = getattr(session._config, key) if overrides: validated_config = validate(overrides, model) - # Extract only the fields that _fetch_params needs from validated_config + # Extract ONLY the fields that were actually overridden (not all fields) + # This prevents validated defaults from overwriting session config values validated_dict = { - f.name: getattr(validated_config, f.name) - for f in fields(_fetch_params) - if hasattr(validated_config, f.name) + field: getattr(validated_config, field) for field in overrides.keys() if hasattr(validated_config, field) } - validated_dict.setdefault("solve_cloudflare", False) + + # Preserve solve_cloudflare if the user explicitly provided it, even if the model doesn't have it + if "solve_cloudflare" in overrides: + validated_dict["solve_cloudflare"] = overrides["solve_cloudflare"] # Start with session defaults, then overwrite with validated overrides result.update(validated_dict) @@ -238,7 +193,7 @@ def validate_fetch( # Cache default values for each model to reduce validation overhead models_default_values = {} -for _model in (CamoufoxConfig, PlaywrightConfig): +for _model in (StealthConfig, PlaywrightConfig): _defaults = {} if hasattr(_model, "__struct_defaults__") and hasattr(_model, "__struct_fields__"): for field_name, default_value in zip(_model.__struct_fields__, _model.__struct_defaults__): # type: ignore @@ -256,14 +211,14 @@ def _filter_defaults(params: Dict, model: str) -> Dict: @overload -def validate(params: Dict, model: type[PlaywrightConfig]) -> PlaywrightConfig: ... +def validate(params: Dict, model: type[StealthConfig]) -> StealthConfig: ... @overload -def validate(params: Dict, model: type[CamoufoxConfig]) -> CamoufoxConfig: ... +def validate(params: Dict, model: type[PlaywrightConfig]) -> PlaywrightConfig: ... -def validate(params: Dict, model: type[PlaywrightConfig] | type[CamoufoxConfig]) -> PlaywrightConfig | CamoufoxConfig: +def validate(params: Dict, model: type[PlaywrightConfig] | type[StealthConfig]) -> PlaywrightConfig | StealthConfig: try: # Filter out params with the default values (no need to validate them) to speed up validation filtered = _filter_defaults(params, model.__name__) diff --git a/scrapling/engines/constants.py b/scrapling/engines/constants.py index df12ee3..774c6c8 100644 --- a/scrapling/engines/constants.py +++ b/scrapling/engines/constants.py @@ -74,7 +74,6 @@ DEFAULT_STEALTH_FLAGS = ( "--disable-domain-reliability", "--disable-threaded-animation", "--disable-threaded-scrolling", - # '--disable-reading-from-canvas', # For Firefox "--enable-simple-cache-backend", "--disable-background-networking", "--enable-surface-synchronization", diff --git a/scrapling/fetchers/__init__.py b/scrapling/fetchers/__init__.py index c135273..e2e5866 100644 --- a/scrapling/fetchers/__init__.py +++ b/scrapling/fetchers/__init__.py @@ -3,7 +3,7 @@ from typing import TYPE_CHECKING, Any if TYPE_CHECKING: from scrapling.fetchers.requests import Fetcher, AsyncFetcher, FetcherSession from scrapling.fetchers.chrome import DynamicFetcher, DynamicSession, AsyncDynamicSession - from scrapling.fetchers.firefox import StealthyFetcher, StealthySession, AsyncStealthySession + from scrapling.fetchers.stealth_chrome import StealthyFetcher, StealthySession, AsyncStealthySession # Lazy import mapping @@ -14,9 +14,9 @@ _LAZY_IMPORTS = { "DynamicFetcher": ("scrapling.fetchers.chrome", "DynamicFetcher"), "DynamicSession": ("scrapling.fetchers.chrome", "DynamicSession"), "AsyncDynamicSession": ("scrapling.fetchers.chrome", "AsyncDynamicSession"), - "StealthyFetcher": ("scrapling.fetchers.firefox", "StealthyFetcher"), - "StealthySession": ("scrapling.fetchers.firefox", "StealthySession"), - "AsyncStealthySession": ("scrapling.fetchers.firefox", "AsyncStealthySession"), + "StealthyFetcher": ("scrapling.fetchers.stealth_chrome", "StealthyFetcher"), + "StealthySession": ("scrapling.fetchers.stealth_chrome", "StealthySession"), + "AsyncStealthySession": ("scrapling.fetchers.stealth_chrome", "AsyncStealthySession"), } __all__ = [ diff --git a/scrapling/fetchers/firefox.py b/scrapling/fetchers/stealth_chrome.py similarity index 52% rename from scrapling/fetchers/firefox.py rename to scrapling/fetchers/stealth_chrome.py index a9361d6..3567388 100644 --- a/scrapling/fetchers/firefox.py +++ b/scrapling/fetchers/stealth_chrome.py @@ -1,48 +1,52 @@ from scrapling.core._types import Unpack -from scrapling.engines._browsers._types import CamoufoxSession +from scrapling.engines._browsers._types import StealthSession from scrapling.engines.toolbelt.custom import BaseFetcher, Response -from scrapling.engines._browsers._camoufox import StealthySession, AsyncStealthySession +from scrapling.engines._browsers._stealth import StealthySession, AsyncStealthySession class StealthyFetcher(BaseFetcher): - """A `Fetcher` class type that is a completely stealthy fetcher that uses a modified version of Firefox. + """A `Fetcher` class type which is a completely stealthy built on top of Chromium. - It works as real browsers passing almost all online tests/protections based on Camoufox. - Other added flavors include setting the faked OS fingerprints to match the user's OS, and the referer of every request is set as if this request came from Google's search of this URL's domain. + It works as real browsers passing almost all online tests/protections with many customization options. """ @classmethod - def fetch(cls, url: str, **kwargs: Unpack[CamoufoxSession]) -> Response: + def fetch(cls, url: str, **kwargs: Unpack[StealthSession]) -> Response: """ Opens up a browser and do your request based on your chosen options below. :param url: Target url. :param kwargs: Browser session configuration options including: - headless: Run the browser in headless/hidden (default), or headful/visible mode. - - block_images: Prevent the loading of images through Firefox preferences. - - disable_resources: Drop requests of unnecessary resources for a speed boost. - - block_webrtc: Blocks WebRTC entirely. - - allow_webgl: Enabled by default. Disabling WebGL is not recommended as many WAFs now check if WebGL is enabled. + - disable_resources: Drop requests of unnecessary resources for a speed boost. It depends, but it made requests ~25% faster in my tests for some websites. + Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`. + This can help save your proxy usage but be careful with this option as it makes some websites never finish loading. + - useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it. + - cookies: Set cookies for the next request. - network_idle: Wait for the page until there are no network connections for at least 500 ms. - - load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute. - - humanize: Humanize the cursor movement. Takes either True or the MAX duration in seconds of the cursor movement. - - solve_cloudflare: Solves all types of the Cloudflare's Turnstile/Interstitial challenges before returning the response to you. - - wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the Response object. - timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000 + - wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object. - page_action: Added for automation. A function that takes the `page` object and does the automation you need. - wait_selector: Wait for a specific CSS selector to be in a specific state. - - init_script: An absolute path to a JavaScript file to be executed on page creation with this request. - - addons: List of Firefox addons to use. Must be paths to extracted addons. + - init_script: An absolute path to a JavaScript file to be executed on page creation for all pages in this session. + - locale: Specify user locale, for example, `en-GB`, `de-DE`, etc. Locale will affect navigator.language value, Accept-Language request header value as well as number and date formatting + rules. Defaults to the system default locale. + - timezone_id: Changes the timezone of the browser. Defaults to the system timezone. - wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`. - - cookies: Set cookies for the next request. + - solve_cloudflare: Solves all types of the Cloudflare's Turnstile/Interstitial challenges before returning the response to you. + - real_chrome: If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch an instance of your browser and use it. + - hide_canvas: Add random noise to canvas operations to prevent fingerprinting. + - block_webrtc: Forces WebRTC to respect proxy settings to prevent local IP address leak. + - allow_webgl: Enabled by default. Disabling it disables WebGL and WebGL 2.0 support entirely. Disabling WebGL is not recommended as many WAFs now check if WebGL is enabled. + - load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute. + - cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers through CDP. - google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name. - - extra_headers: A dictionary of extra headers to add to the request. + - extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ - proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only. - - os_randomize: If enabled, Scrapling will randomize the OS fingerprints used. - - disable_ads: Disabled by default, this installs the `uBlock Origin` addon on the browser if enabled. - - geoip: Recommended to use with proxies; Automatically use IP's longitude, latitude, timezone, country, locale, and spoof the WebRTC IP address. + - user_data_dir: Path to a User Data Directory, which stores browser session data like cookies and local storage. The default is to create a temporary directory. + - extra_flags: A list of additional browser flags to pass to the browser on launch. - selector_config: The arguments that will be passed in the end while creating the final Selector's class. - - additional_args: Additional arguments to be passed to Camoufox as additional settings. + - additional_args: Additional arguments to be passed to Playwright's context as additional settings, and it takes higher priority than Scrapling's settings. :return: A `Response` object. """ selector_config = kwargs.get("selector_config", {}) or kwargs.get( @@ -57,37 +61,42 @@ class StealthyFetcher(BaseFetcher): return engine.fetch(url) @classmethod - async def async_fetch(cls, url: str, **kwargs: Unpack[CamoufoxSession]) -> Response: + async def async_fetch(cls, url: str, **kwargs: Unpack[StealthSession]) -> Response: """ Opens up a browser and do your request based on your chosen options below. :param url: Target url. :param kwargs: Browser session configuration options including: - headless: Run the browser in headless/hidden (default), or headful/visible mode. - - block_images: Prevent the loading of images through Firefox preferences. - - disable_resources: Drop requests of unnecessary resources for a speed boost. - - block_webrtc: Blocks WebRTC entirely. - - allow_webgl: Enabled by default. Disabling WebGL is not recommended as many WAFs now check if WebGL is enabled. + - disable_resources: Drop requests of unnecessary resources for a speed boost. It depends, but it made requests ~25% faster in my tests for some websites. + Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`. + This can help save your proxy usage but be careful with this option as it makes some websites never finish loading. + - useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it. + - cookies: Set cookies for the next request. - network_idle: Wait for the page until there are no network connections for at least 500 ms. - - load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute. - - humanize: Humanize the cursor movement. Takes either True or the MAX duration in seconds of the cursor movement. - - solve_cloudflare: Solves all types of the Cloudflare's Turnstile/Interstitial challenges before returning the response to you. - - wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the Response object. - timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000 + - wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object. - page_action: Added for automation. A function that takes the `page` object and does the automation you need. - wait_selector: Wait for a specific CSS selector to be in a specific state. - - init_script: An absolute path to a JavaScript file to be executed on page creation with this request. - - addons: List of Firefox addons to use. Must be paths to extracted addons. + - init_script: An absolute path to a JavaScript file to be executed on page creation for all pages in this session. + - locale: Specify user locale, for example, `en-GB`, `de-DE`, etc. Locale will affect navigator.language value, Accept-Language request header value as well as number and date formatting + rules. Defaults to the system default locale. + - timezone_id: Changes the timezone of the browser. Defaults to the system timezone. - wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`. - - cookies: Set cookies for the next request. + - solve_cloudflare: Solves all types of the Cloudflare's Turnstile/Interstitial challenges before returning the response to you. + - real_chrome: If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch an instance of your browser and use it. + - hide_canvas: Add random noise to canvas operations to prevent fingerprinting. + - block_webrtc: Forces WebRTC to respect proxy settings to prevent local IP address leak. + - allow_webgl: Enabled by default. Disabling it disables WebGL and WebGL 2.0 support entirely. Disabling WebGL is not recommended as many WAFs now check if WebGL is enabled. + - load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute. + - cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers through CDP. - google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name. - - extra_headers: A dictionary of extra headers to add to the request. + - extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ - proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only. - - os_randomize: If enabled, Scrapling will randomize the OS fingerprints used. - - disable_ads: Disabled by default, this installs the `uBlock Origin` addon on the browser if enabled. - - geoip: Recommended to use with proxies; Automatically use IP's longitude, latitude, timezone, country, locale, and spoof the WebRTC IP address. + - user_data_dir: Path to a User Data Directory, which stores browser session data like cookies and local storage. The default is to create a temporary directory. + - extra_flags: A list of additional browser flags to pass to the browser on launch. - selector_config: The arguments that will be passed in the end while creating the final Selector's class. - - additional_args: Additional arguments to be passed to Camoufox as additional settings. + - additional_args: Additional arguments to be passed to Playwright's context as additional settings, and it takes higher priority than Scrapling's settings. :return: A `Response` object. """ selector_config = kwargs.get("selector_config", {}) or kwargs.get(