freedak/Scrapling
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

PlaywrightFetcher - The option to drop unnecessary resources requests for speed boost

Browse Source
This commit is contained in:
Karim shoair
2024-11-03 16:27:55 +02:00
parent b32e1203f2
commit 711de81106
1 changed files with 5 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scrapling/engines/pw.py
+5 -1
View File
@@ -7,6 +7,7 @@ from scrapling.engines.toolbelt import (
Response, Response,
do_nothing, do_nothing,
js_bypass_path, js_bypass_path,
intercept_route,
generate_headers, generate_headers,
check_type_validity, check_type_validity,
construct_websocket_url, construct_websocket_url,
@@ -17,7 +18,7 @@ from scrapling.engines.toolbelt import (
class PlaywrightEngine: class PlaywrightEngine:
def __init__( def __init__(
self, headless: Union[bool, str] = True, self, headless: Union[bool, str] = True,
disable_resources: Optional[List] = None, disable_resources: Optional[bool] = False,
useragent: Optional[str] = None, useragent: Optional[str] = None,
network_idle: Optional[bool] = False, network_idle: Optional[bool] = False,
timeout: Optional[float] = 30000, timeout: Optional[float] = 30000,
@@ -134,6 +135,9 @@ class PlaywrightEngine:
page = context.new_page() page = context.new_page()
page.set_default_navigation_timeout(self.timeout) page.set_default_navigation_timeout(self.timeout)
page.set_default_timeout(self.timeout) page.set_default_timeout(self.timeout)
if self.disable_resources:
page.route("**/*", intercept_route)
if self.stealth: if self.stealth:
# Basic bypasses nothing fancy as I'm still working on it # Basic bypasses nothing fancy as I'm still working on it
# But with adding these bypasses to the above config, it bypasses many online tests like # But with adding these bypasses to the above config, it bypasses many online tests like