freedak/Scrapling
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(browser fetchers): A new option to control page JS

Browse Source
This commit is contained in:
Karim shoair
2025-09-13 03:59:12 +03:00
parent aaec00563c
commit 8c4d6158d6
5 changed files with 63 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scrapling/fetchers.py
+13 -1
View File
@@ -56,6 +56,7 @@ class StealthyFetcher(BaseFetcher):
block_webrtc: bool = False,
allow_webgl: bool = True,
network_idle: bool = False,
load_dom: bool = True,
humanize: bool | float = True,
solve_cloudflare: bool = False,
wait: int | float = 0,
@@ -92,6 +93,7 @@ class StealthyFetcher(BaseFetcher):
:param solve_cloudflare: Solves all 3 types of the Cloudflare's Turnstile wait page before returning the response to you.
:param allow_webgl: Enabled by default. Disabling WebGL is not recommended as many WAFs now check if WebGL is enabled.
:param network_idle: Wait for the page until there are no network connections for at least 500 ms.
:param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute.
:param disable_ads: Disabled by default, this installs the `uBlock Origin` addon on the browser if enabled.
:param os_randomize: If enabled, Scrapling will randomize the OS fingerprints used. The default is Scrapling matching the fingerprints with the current OS.
:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
@@ -123,6 +125,7 @@ class StealthyFetcher(BaseFetcher):
cookies=cookies,
headless=headless,
humanize=humanize,
load_dom=load_dom,
disable_ads=disable_ads,
allow_webgl=allow_webgl,
page_action=page_action,
@@ -152,6 +155,7 @@ class StealthyFetcher(BaseFetcher):
block_webrtc: bool = False,
allow_webgl: bool = True,
network_idle: bool = False,
load_dom: bool = True,
humanize: bool | float = True,
solve_cloudflare: bool = False,
wait: int | float = 0,
@@ -188,6 +192,7 @@ class StealthyFetcher(BaseFetcher):
:param solve_cloudflare: Solves all 3 types of the Cloudflare's Turnstile wait page before returning the response to you.
:param allow_webgl: Enabled by default. Disabling WebGL is not recommended as many WAFs now check if WebGL is enabled.
:param network_idle: Wait for the page until there are no network connections for at least 500 ms.
:param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute.
:param disable_ads: Disabled by default, this installs the `uBlock Origin` addon on the browser if enabled.
:param os_randomize: If enabled, Scrapling will randomize the OS fingerprints used. The default is Scrapling matching the fingerprints with the current OS.
:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
@@ -220,6 +225,7 @@ class StealthyFetcher(BaseFetcher):
cookies=cookies,
headless=headless,
humanize=humanize,
load_dom=load_dom,
disable_ads=disable_ads,
allow_webgl=allow_webgl,
page_action=page_action,
@@ -280,6 +286,7 @@ class DynamicFetcher(BaseFetcher):
init_script: Optional[str] = None,
cookies: Optional[Iterable[Dict]] = None,
network_idle: bool = False,
load_dom: bool = True,
wait_selector_state: SelectorWaitStates = "attached",
custom_config: Optional[Dict] = None,
) -> Response:
@@ -293,6 +300,7 @@ class DynamicFetcher(BaseFetcher):
:param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it.
:param cookies: Set cookies for the next request.
:param network_idle: Wait for the page until there are no network connections for at least 500 ms.
:param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute.
:param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
:param page_action: Added for automation. A function that takes the `page` object and does the automation you need.
@@ -325,6 +333,7 @@ class DynamicFetcher(BaseFetcher):
cdp_url=cdp_url,
cookies=cookies,
headless=headless,
load_dom=load_dom,
useragent=useragent,
real_chrome=real_chrome,
page_action=page_action,
@@ -364,6 +373,7 @@ class DynamicFetcher(BaseFetcher):
init_script: Optional[str] = None,
cookies: Optional[Iterable[Dict]] = None,
network_idle: bool = False,
load_dom: bool = True,
wait_selector_state: SelectorWaitStates = "attached",
custom_config: Optional[Dict] = None,
) -> Response:
@@ -377,6 +387,7 @@ class DynamicFetcher(BaseFetcher):
:param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it.
:param cookies: Set cookies for the next request.
:param network_idle: Wait for the page until there are no network connections for at least 500 ms.
:param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute.
:param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
:param page_action: Added for automation. A function that takes the `page` object and does the automation you need.
@@ -402,6 +413,7 @@ class DynamicFetcher(BaseFetcher):
async with AsyncDynamicSession(
wait=wait,
max_pages=1,
proxy=proxy,
locale=locale,
timeout=timeout,
@@ -409,8 +421,8 @@ class DynamicFetcher(BaseFetcher):
cdp_url=cdp_url,
cookies=cookies,
headless=headless,
load_dom=load_dom,
useragent=useragent,
max_pages=1,
real_chrome=real_chrome,
page_action=page_action,
hide_canvas=hide_canvas,