refactor(fetchers)!: Replace Camoufox with patchright and many optimizations
- DynamicFetcher became 20% faster - StealthyFetcher became 99% faster - Scrapling size decreased - Code became ~400 lines shorter - Most importantly, scrapling is more stable and reliable now. - Less confusing for new users. - More...
This commit is contained in:
Karim shoair
+79
-92
@@ -213,13 +213,11 @@ class ScraplingMCPServer:
|
||||
main_content_only: bool = True,
|
||||
headless: bool = False,
|
||||
google_search: bool = True,
|
||||
hide_canvas: bool = False,
|
||||
disable_webgl: bool = False,
|
||||
real_chrome: bool = False,
|
||||
stealth: bool = False,
|
||||
wait: int | float = 0,
|
||||
proxy: Optional[str | Dict[str, str]] = None,
|
||||
locale: str = "en-US",
|
||||
timezone_id: str | None = None,
|
||||
locale: str | None = None,
|
||||
extra_headers: Optional[Dict[str, str]] = None,
|
||||
useragent: Optional[str] = None,
|
||||
cdp_url: Optional[str] = None,
|
||||
@@ -251,12 +249,11 @@ class ScraplingMCPServer:
|
||||
:param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
|
||||
:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
|
||||
:param wait_selector: Wait for a specific CSS selector to be in a specific state.
|
||||
:param locale: Set the locale for the browser if wanted. The default value is `en-US`.
|
||||
:param timezone_id: Changes the timezone of the browser. Defaults to the system timezone.
|
||||
:param locale: Specify user locale, for example, `en-GB`, `de-DE`, etc. Locale will affect navigator.language value, Accept-Language request header value as well as number and date formatting
|
||||
rules. Defaults to the system default locale.
|
||||
:param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.
|
||||
:param stealth: Enables stealth mode, check the documentation to see what stealth mode does currently.
|
||||
:param real_chrome: If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch an instance of your browser and use it.
|
||||
:param hide_canvas: Add random noise to canvas operations to prevent fingerprinting.
|
||||
:param disable_webgl: Disables WebGL and WebGL 2.0 support entirely.
|
||||
:param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers through CDP.
|
||||
:param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name.
|
||||
:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
|
||||
@@ -269,15 +266,13 @@ class ScraplingMCPServer:
|
||||
locale=locale,
|
||||
timeout=timeout,
|
||||
cookies=cookies,
|
||||
stealth=stealth,
|
||||
cdp_url=cdp_url,
|
||||
headless=headless,
|
||||
useragent=useragent,
|
||||
hide_canvas=hide_canvas,
|
||||
timezone_id=timezone_id,
|
||||
real_chrome=real_chrome,
|
||||
network_idle=network_idle,
|
||||
wait_selector=wait_selector,
|
||||
disable_webgl=disable_webgl,
|
||||
extra_headers=extra_headers,
|
||||
google_search=google_search,
|
||||
disable_resources=disable_resources,
|
||||
@@ -301,13 +296,11 @@ class ScraplingMCPServer:
|
||||
main_content_only: bool = True,
|
||||
headless: bool = False,
|
||||
google_search: bool = True,
|
||||
hide_canvas: bool = False,
|
||||
disable_webgl: bool = False,
|
||||
real_chrome: bool = False,
|
||||
stealth: bool = False,
|
||||
wait: int | float = 0,
|
||||
proxy: Optional[str | Dict[str, str]] = None,
|
||||
locale: str = "en-US",
|
||||
timezone_id: str | None = None,
|
||||
locale: str | None = None,
|
||||
extra_headers: Optional[Dict[str, str]] = None,
|
||||
useragent: Optional[str] = None,
|
||||
cdp_url: Optional[str] = None,
|
||||
@@ -339,12 +332,11 @@ class ScraplingMCPServer:
|
||||
:param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
|
||||
:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
|
||||
:param wait_selector: Wait for a specific CSS selector to be in a specific state.
|
||||
:param locale: Set the locale for the browser if wanted. The default value is `en-US`.
|
||||
:param timezone_id: Changes the timezone of the browser. Defaults to the system timezone.
|
||||
:param locale: Specify user locale, for example, `en-GB`, `de-DE`, etc. Locale will affect navigator.language value, Accept-Language request header value as well as number and date formatting
|
||||
rules. Defaults to the system default locale.
|
||||
:param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.
|
||||
:param stealth: Enables stealth mode, check the documentation to see what stealth mode does currently.
|
||||
:param real_chrome: If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch an instance of your browser and use it.
|
||||
:param hide_canvas: Add random noise to canvas operations to prevent fingerprinting.
|
||||
:param disable_webgl: Disables WebGL and WebGL 2.0 support entirely.
|
||||
:param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers through CDP.
|
||||
:param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name.
|
||||
:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
|
||||
@@ -356,17 +348,15 @@ class ScraplingMCPServer:
|
||||
locale=locale,
|
||||
timeout=timeout,
|
||||
cookies=cookies,
|
||||
stealth=stealth,
|
||||
cdp_url=cdp_url,
|
||||
headless=headless,
|
||||
max_pages=len(urls),
|
||||
useragent=useragent,
|
||||
hide_canvas=hide_canvas,
|
||||
timezone_id=timezone_id,
|
||||
real_chrome=real_chrome,
|
||||
network_idle=network_idle,
|
||||
wait_selector=wait_selector,
|
||||
google_search=google_search,
|
||||
disable_webgl=disable_webgl,
|
||||
extra_headers=extra_headers,
|
||||
disable_resources=disable_resources,
|
||||
wait_selector_state=wait_selector_state,
|
||||
@@ -393,29 +383,29 @@ class ScraplingMCPServer:
|
||||
css_selector: Optional[str] = None,
|
||||
main_content_only: bool = True,
|
||||
headless: bool = True, # noqa: F821
|
||||
block_images: bool = False,
|
||||
google_search: bool = True,
|
||||
real_chrome: bool = False,
|
||||
wait: int | float = 0,
|
||||
proxy: Optional[str | Dict[str, str]] = None,
|
||||
timezone_id: str | None = None,
|
||||
locale: str | None = None,
|
||||
extra_headers: Optional[Dict[str, str]] = None,
|
||||
useragent: Optional[str] = None,
|
||||
hide_canvas: bool = False,
|
||||
cdp_url: Optional[str] = None,
|
||||
timeout: int | float = 30000,
|
||||
disable_resources: bool = False,
|
||||
wait_selector: Optional[str] = None,
|
||||
cookies: Optional[List[Dict]] = None,
|
||||
network_idle: bool = False,
|
||||
wait_selector_state: SelectorWaitStates = "attached",
|
||||
block_webrtc: bool = False,
|
||||
allow_webgl: bool = True,
|
||||
network_idle: bool = False,
|
||||
humanize: bool | float = True,
|
||||
solve_cloudflare: bool = False,
|
||||
wait: int | float = 0,
|
||||
timeout: int | float = 30000,
|
||||
wait_selector: Optional[str] = None,
|
||||
addons: Optional[List[str]] = None,
|
||||
wait_selector_state: SelectorWaitStates = "attached",
|
||||
cookies: Optional[List[Dict]] = None,
|
||||
google_search: bool = True,
|
||||
extra_headers: Optional[Dict[str, str]] = None,
|
||||
proxy: Optional[str | Dict[str, str]] = None,
|
||||
os_randomize: bool = False,
|
||||
disable_ads: bool = False,
|
||||
geoip: bool = False,
|
||||
additional_args: Optional[Dict] = None,
|
||||
) -> ResponseModel:
|
||||
"""Use Scrapling's version of the Camoufox browser to fetch a URL and return a structured output of the result.
|
||||
Note: This is best suitable for high protection levels. It's slower than the other tools.
|
||||
"""Use the stealthy fetcher to fetch a URL and return a structured output of the result.
|
||||
Note: This is the only suitable fetcher for high protection levels.
|
||||
Note: If the `css_selector` resolves to more than one element, all the elements will be returned.
|
||||
|
||||
:param url: The URL to request.
|
||||
@@ -426,54 +416,53 @@ class ScraplingMCPServer:
|
||||
:param css_selector: CSS selector to extract the content from the page. If main_content_only is True, then it will be executed on the main content of the page. Defaults to None.
|
||||
:param main_content_only: Whether to extract only the main content of the page. Defaults to True. The main content here is the data inside the `<body>` tag.
|
||||
:param headless: Run the browser in headless/hidden (default), or headful/visible mode.
|
||||
:param block_images: Prevent the loading of images through Firefox preferences.
|
||||
This can help save your proxy usage but be careful with this option as it makes some websites never finish loading.
|
||||
:param disable_resources: Drop requests of unnecessary resources for a speed boost. It depends, but it made requests ~25% faster in my tests for some websites.
|
||||
Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
|
||||
This can help save your proxy usage but be careful with this option as it makes some websites never finish loading.
|
||||
:param block_webrtc: Blocks WebRTC entirely.
|
||||
:param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it.
|
||||
:param cookies: Set cookies for the next request.
|
||||
:param addons: List of Firefox addons to use. Must be paths to extracted addons.
|
||||
:param humanize: Humanize the cursor movement. Takes either True or the MAX duration in seconds of the cursor movement. The cursor typically takes up to 1.5 seconds to move across the window.
|
||||
:param solve_cloudflare: Solves all types of the Cloudflare's Turnstile/Interstitial challenges before returning the response to you.
|
||||
:param allow_webgl: Enabled by default. Disabling WebGL is not recommended as many WAFs now check if WebGL is enabled.
|
||||
:param network_idle: Wait for the page until there are no network connections for at least 500 ms.
|
||||
:param disable_ads: Disabled by default, this installs the `uBlock Origin` addon on the browser if enabled.
|
||||
:param os_randomize: If enabled, Scrapling will randomize the OS fingerprints used. The default is Scrapling matching the fingerprints with the current OS.
|
||||
:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
|
||||
:param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
|
||||
:param wait_selector: Wait for a specific CSS selector to be in a specific state.
|
||||
:param geoip: Recommended to use with proxies; Automatically use IP's longitude, latitude, timezone, country, locale, and spoof the WebRTC IP address.
|
||||
It will also calculate and spoof the browser's language based on the distribution of language speakers in the target region.
|
||||
:param timezone_id: Changes the timezone of the browser. Defaults to the system timezone.
|
||||
:param locale: Specify user locale, for example, `en-GB`, `de-DE`, etc. Locale will affect navigator.language value, Accept-Language request header value as well as number and date formatting
|
||||
rules. Defaults to the system default locale.
|
||||
:param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.
|
||||
:param real_chrome: If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch an instance of your browser and use it.
|
||||
:param hide_canvas: Add random noise to canvas operations to prevent fingerprinting.
|
||||
:param block_webrtc: Forces WebRTC to respect proxy settings to prevent local IP address leak.
|
||||
:param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers through CDP.
|
||||
:param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name.
|
||||
:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
|
||||
:param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.
|
||||
:param additional_args: Additional arguments to be passed to Camoufox as additional settings, and it takes higher priority than Scrapling's settings.
|
||||
:param additional_args: Additional arguments to be passed to Playwright's context as additional settings, and it takes higher priority than Scrapling's settings.
|
||||
"""
|
||||
page = await StealthyFetcher.async_fetch(
|
||||
url,
|
||||
wait=wait,
|
||||
proxy=proxy,
|
||||
geoip=geoip,
|
||||
addons=addons,
|
||||
locale=locale,
|
||||
cdp_url=cdp_url,
|
||||
timeout=timeout,
|
||||
cookies=cookies,
|
||||
headless=headless,
|
||||
humanize=humanize,
|
||||
useragent=useragent,
|
||||
timezone_id=timezone_id,
|
||||
real_chrome=real_chrome,
|
||||
hide_canvas=hide_canvas,
|
||||
allow_webgl=allow_webgl,
|
||||
disable_ads=disable_ads,
|
||||
network_idle=network_idle,
|
||||
block_images=block_images,
|
||||
block_webrtc=block_webrtc,
|
||||
os_randomize=os_randomize,
|
||||
wait_selector=wait_selector,
|
||||
google_search=google_search,
|
||||
extra_headers=extra_headers,
|
||||
additional_args=additional_args,
|
||||
solve_cloudflare=solve_cloudflare,
|
||||
disable_resources=disable_resources,
|
||||
wait_selector_state=wait_selector_state,
|
||||
additional_args=additional_args,
|
||||
)
|
||||
return _ContentTranslator(
|
||||
Convertor._extract_content(
|
||||
@@ -492,29 +481,29 @@ class ScraplingMCPServer:
|
||||
css_selector: Optional[str] = None,
|
||||
main_content_only: bool = True,
|
||||
headless: bool = True, # noqa: F821
|
||||
block_images: bool = False,
|
||||
google_search: bool = True,
|
||||
real_chrome: bool = False,
|
||||
wait: int | float = 0,
|
||||
proxy: Optional[str | Dict[str, str]] = None,
|
||||
timezone_id: str | None = None,
|
||||
locale: str | None = None,
|
||||
extra_headers: Optional[Dict[str, str]] = None,
|
||||
useragent: Optional[str] = None,
|
||||
hide_canvas: bool = False,
|
||||
cdp_url: Optional[str] = None,
|
||||
timeout: int | float = 30000,
|
||||
disable_resources: bool = False,
|
||||
wait_selector: Optional[str] = None,
|
||||
cookies: Optional[List[Dict]] = None,
|
||||
network_idle: bool = False,
|
||||
wait_selector_state: SelectorWaitStates = "attached",
|
||||
block_webrtc: bool = False,
|
||||
allow_webgl: bool = True,
|
||||
network_idle: bool = False,
|
||||
humanize: bool | float = True,
|
||||
solve_cloudflare: bool = False,
|
||||
wait: int | float = 0,
|
||||
timeout: int | float = 30000,
|
||||
wait_selector: Optional[str] = None,
|
||||
addons: Optional[List[str]] = None,
|
||||
wait_selector_state: SelectorWaitStates = "attached",
|
||||
cookies: Optional[List[Dict]] = None,
|
||||
google_search: bool = True,
|
||||
extra_headers: Optional[Dict[str, str]] = None,
|
||||
proxy: Optional[str | Dict[str, str]] = None,
|
||||
os_randomize: bool = False,
|
||||
disable_ads: bool = False,
|
||||
geoip: bool = False,
|
||||
additional_args: Optional[Dict] = None,
|
||||
) -> List[ResponseModel]:
|
||||
"""Use Scrapling's version of the Camoufox browser to fetch a group of URLs at the same time, and for each page return a structured output of the result.
|
||||
Note: This is best suitable for high protection levels. It's slower than the other tools.
|
||||
"""Use the stealthy fetcher to fetch a group of URLs at the same time, and for each page return a structured output of the result.
|
||||
Note: This is the only suitable fetcher for high protection levels.
|
||||
Note: If the `css_selector` resolves to more than one element, all the elements will be returned.
|
||||
|
||||
:param urls: A tuple of the URLs to request.
|
||||
@@ -525,54 +514,52 @@ class ScraplingMCPServer:
|
||||
:param css_selector: CSS selector to extract the content from the page. If main_content_only is True, then it will be executed on the main content of the page. Defaults to None.
|
||||
:param main_content_only: Whether to extract only the main content of the page. Defaults to True. The main content here is the data inside the `<body>` tag.
|
||||
:param headless: Run the browser in headless/hidden (default), or headful/visible mode.
|
||||
:param block_images: Prevent the loading of images through Firefox preferences.
|
||||
This can help save your proxy usage but be careful with this option as it makes some websites never finish loading.
|
||||
:param disable_resources: Drop requests of unnecessary resources for a speed boost. It depends, but it made requests ~25% faster in my tests for some websites.
|
||||
Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
|
||||
This can help save your proxy usage but be careful with this option as it makes some websites never finish loading.
|
||||
:param block_webrtc: Blocks WebRTC entirely.
|
||||
:param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it.
|
||||
:param cookies: Set cookies for the next request.
|
||||
:param addons: List of Firefox addons to use. Must be paths to extracted addons.
|
||||
:param humanize: Humanize the cursor movement. Takes either True or the MAX duration in seconds of the cursor movement. The cursor typically takes up to 1.5 seconds to move across the window.
|
||||
:param solve_cloudflare: Solves all types of the Cloudflare's Turnstile/Interstitial challenges before returning the response to you.
|
||||
:param allow_webgl: Enabled by default. Disabling WebGL is not recommended as many WAFs now check if WebGL is enabled.
|
||||
:param network_idle: Wait for the page until there are no network connections for at least 500 ms.
|
||||
:param disable_ads: Disabled by default, this installs the `uBlock Origin` addon on the browser if enabled.
|
||||
:param os_randomize: If enabled, Scrapling will randomize the OS fingerprints used. The default is Scrapling matching the fingerprints with the current OS.
|
||||
:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
|
||||
:param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
|
||||
:param wait_selector: Wait for a specific CSS selector to be in a specific state.
|
||||
:param geoip: Recommended to use with proxies; Automatically use IP's longitude, latitude, timezone, country, locale, and spoof the WebRTC IP address.
|
||||
It will also calculate and spoof the browser's language based on the distribution of language speakers in the target region.
|
||||
:param timezone_id: Changes the timezone of the browser. Defaults to the system timezone.
|
||||
:param locale: Specify user locale, for example, `en-GB`, `de-DE`, etc. Locale will affect navigator.language value, Accept-Language request header value as well as number and date formatting
|
||||
rules. Defaults to the system default locale.
|
||||
:param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.
|
||||
:param real_chrome: If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch an instance of your browser and use it.
|
||||
:param hide_canvas: Add random noise to canvas operations to prevent fingerprinting.
|
||||
:param block_webrtc: Forces WebRTC to respect proxy settings to prevent local IP address leak.
|
||||
:param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers through CDP.
|
||||
:param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name.
|
||||
:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
|
||||
:param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.
|
||||
:param additional_args: Additional arguments to be passed to Camoufox as additional settings, and it takes higher priority than Scrapling's settings.
|
||||
:param additional_args: Additional arguments to be passed to Playwright's context as additional settings, and it takes higher priority than Scrapling's settings.
|
||||
"""
|
||||
async with AsyncStealthySession(
|
||||
wait=wait,
|
||||
proxy=proxy,
|
||||
geoip=geoip,
|
||||
addons=addons,
|
||||
locale=locale,
|
||||
cdp_url=cdp_url,
|
||||
timeout=timeout,
|
||||
cookies=cookies,
|
||||
headless=headless,
|
||||
humanize=humanize,
|
||||
max_pages=len(urls),
|
||||
useragent=useragent,
|
||||
timezone_id=timezone_id,
|
||||
real_chrome=real_chrome,
|
||||
hide_canvas=hide_canvas,
|
||||
allow_webgl=allow_webgl,
|
||||
disable_ads=disable_ads,
|
||||
block_images=block_images,
|
||||
block_webrtc=block_webrtc,
|
||||
network_idle=network_idle,
|
||||
os_randomize=os_randomize,
|
||||
block_webrtc=block_webrtc,
|
||||
wait_selector=wait_selector,
|
||||
google_search=google_search,
|
||||
extra_headers=extra_headers,
|
||||
additional_args=additional_args,
|
||||
solve_cloudflare=solve_cloudflare,
|
||||
disable_resources=disable_resources,
|
||||
wait_selector_state=wait_selector_state,
|
||||
additional_args=additional_args,
|
||||
) as session:
|
||||
tasks = [session.fetch(url) for url in urls]
|
||||
responses = await gather(*tasks)
|
||||
|
||||
Reference in New Issue
Block a user