freedak/Scrapling
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Get page content after JS Execution

Browse Source 
Thanks for @AbdullahY36
This commit is contained in:
Karim shoair
2024-11-10 15:54:44 +02:00
parent 7d2f55bd4c
commit 43045e3eb2
2 changed files with 2 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scrapling/engines/camo.py
+1 -3
View File
@@ -87,9 +87,7 @@ class CamoufoxEngine:
if self.extra_headers:
page.set_extra_http_headers(self.extra_headers)
res = page.goto(url, referer=generate_convincing_referer(url) if self.google_search else None)
page.wait_for_load_state(state="load")
page.wait_for_load_state(state="domcontentloaded")
res = page.goto(url, referer=generate_convincing_referer(url) if self.google_search else None, wait_until="domcontentloaded")
if self.network_idle:
page.wait_for_load_state('networkidle')
scrapling/engines/pw.py
+1 -3
View File
@@ -199,9 +199,7 @@ class PlaywrightEngine:
page.add_init_script(path=js_bypass_path('screen_props.js'))
page.add_init_script(path=js_bypass_path('playwright_fingerprint.js'))
res = page.goto(url, referer=generate_convincing_referer(url) if self.google_search else None)
page.wait_for_load_state(state="load")
page.wait_for_load_state(state="domcontentloaded")
res = page.goto(url, referer=generate_convincing_referer(url) if self.google_search else None, wait_until="domcontentloaded")
if self.network_idle:
page.wait_for_load_state('networkidle')