From 711de8110637cce88d7832b2f1b5bbf526dcb1a0 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Sun, 3 Nov 2024 16:27:55 +0200 Subject: [PATCH] PlaywrightFetcher - The option to drop unnecessary resources requests for speed boost --- scrapling/engines/pw.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/scrapling/engines/pw.py b/scrapling/engines/pw.py index 4d628c3..4ce0600 100644 --- a/scrapling/engines/pw.py +++ b/scrapling/engines/pw.py @@ -7,6 +7,7 @@ from scrapling.engines.toolbelt import ( Response, do_nothing, js_bypass_path, + intercept_route, generate_headers, check_type_validity, construct_websocket_url, @@ -17,7 +18,7 @@ from scrapling.engines.toolbelt import ( class PlaywrightEngine: def __init__( self, headless: Union[bool, str] = True, - disable_resources: Optional[List] = None, + disable_resources: Optional[bool] = False, useragent: Optional[str] = None, network_idle: Optional[bool] = False, timeout: Optional[float] = 30000, @@ -134,6 +135,9 @@ class PlaywrightEngine: page = context.new_page() page.set_default_navigation_timeout(self.timeout) page.set_default_timeout(self.timeout) + if self.disable_resources: + page.route("**/*", intercept_route) + if self.stealth: # Basic bypasses nothing fancy as I'm still working on it # But with adding these bypasses to the above config, it bypasses many online tests like