From 43045e3eb2fdd01e4de7a731f1d367aae03fc5ab Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Sun, 10 Nov 2024 15:54:44 +0200 Subject: [PATCH] Get page content after JS Execution Thanks for @AbdullahY36 --- scrapling/engines/camo.py | 4 +--- scrapling/engines/pw.py | 4 +--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/scrapling/engines/camo.py b/scrapling/engines/camo.py index f56ad69..61731fc 100644 --- a/scrapling/engines/camo.py +++ b/scrapling/engines/camo.py @@ -87,9 +87,7 @@ class CamoufoxEngine: if self.extra_headers: page.set_extra_http_headers(self.extra_headers) - res = page.goto(url, referer=generate_convincing_referer(url) if self.google_search else None) - page.wait_for_load_state(state="load") - page.wait_for_load_state(state="domcontentloaded") + res = page.goto(url, referer=generate_convincing_referer(url) if self.google_search else None, wait_until="domcontentloaded") if self.network_idle: page.wait_for_load_state('networkidle') diff --git a/scrapling/engines/pw.py b/scrapling/engines/pw.py index 4f453fe..72f571a 100644 --- a/scrapling/engines/pw.py +++ b/scrapling/engines/pw.py @@ -199,9 +199,7 @@ class PlaywrightEngine: page.add_init_script(path=js_bypass_path('screen_props.js')) page.add_init_script(path=js_bypass_path('playwright_fingerprint.js')) - res = page.goto(url, referer=generate_convincing_referer(url) if self.google_search else None) - page.wait_for_load_state(state="load") - page.wait_for_load_state(state="domcontentloaded") + res = page.goto(url, referer=generate_convincing_referer(url) if self.google_search else None, wait_until="domcontentloaded") if self.network_idle: page.wait_for_load_state('networkidle')