freedak/Scrapling
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fetchers - Wait for page to be stable again if the user wanted to wait for a selector

Browse Source
This commit is contained in:
Karim shoair
2024-11-26 20:26:54 +02:00
parent d6a59774ea
commit 1cf7cee1a2
2 changed files with 10 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scrapling/engines/camo.py
+5
View File
@@ -111,6 +111,11 @@ class CamoufoxEngine:
if self.wait_selector and type(self.wait_selector) is str: if self.wait_selector and type(self.wait_selector) is str:
waiter = page.locator(self.wait_selector) waiter = page.locator(self.wait_selector)
waiter.first.wait_for(state=self.wait_selector_state) waiter.first.wait_for(state=self.wait_selector_state)
# Wait again after waiting for the selector, helpful with protections like Cloudflare
page.wait_for_load_state(state="load")
page.wait_for_load_state(state="domcontentloaded")
if self.network_idle:
page.wait_for_load_state('networkidle')
# This will be parsed inside `Response` # This will be parsed inside `Response`
encoding = res.headers.get('content-type', '') or 'utf-8' # default encoding encoding = res.headers.get('content-type', '') or 'utf-8' # default encoding
scrapling/engines/pw.py
+5
View File
@@ -226,6 +226,11 @@ class PlaywrightEngine:
if self.wait_selector and type(self.wait_selector) is str: if self.wait_selector and type(self.wait_selector) is str:
waiter = page.locator(self.wait_selector) waiter = page.locator(self.wait_selector)
waiter.first.wait_for(state=self.wait_selector_state) waiter.first.wait_for(state=self.wait_selector_state)
# Wait again after waiting for the selector, helpful with protections like Cloudflare
page.wait_for_load_state(state="load")
page.wait_for_load_state(state="domcontentloaded")
if self.network_idle:
page.wait_for_load_state('networkidle')
# This will be parsed inside `Response` # This will be parsed inside `Response`
encoding = res.headers.get('content-type', '') or 'utf-8' # default encoding encoding = res.headers.get('content-type', '') or 'utf-8' # default encoding