From 1cf7cee1a20d2769b5d8c9d99a54c52515b6b12f Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Tue, 26 Nov 2024 20:26:54 +0200 Subject: [PATCH] Fetchers - Wait for page to be stable again if the user wanted to wait for a selector --- scrapling/engines/camo.py | 5 +++++ scrapling/engines/pw.py | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/scrapling/engines/camo.py b/scrapling/engines/camo.py index 5853a18..90d3378 100644 --- a/scrapling/engines/camo.py +++ b/scrapling/engines/camo.py @@ -111,6 +111,11 @@ class CamoufoxEngine: if self.wait_selector and type(self.wait_selector) is str: waiter = page.locator(self.wait_selector) waiter.first.wait_for(state=self.wait_selector_state) + # Wait again after waiting for the selector, helpful with protections like Cloudflare + page.wait_for_load_state(state="load") + page.wait_for_load_state(state="domcontentloaded") + if self.network_idle: + page.wait_for_load_state('networkidle') # This will be parsed inside `Response` encoding = res.headers.get('content-type', '') or 'utf-8' # default encoding diff --git a/scrapling/engines/pw.py b/scrapling/engines/pw.py index d4dc5ea..3f6a187 100644 --- a/scrapling/engines/pw.py +++ b/scrapling/engines/pw.py @@ -226,6 +226,11 @@ class PlaywrightEngine: if self.wait_selector and type(self.wait_selector) is str: waiter = page.locator(self.wait_selector) waiter.first.wait_for(state=self.wait_selector_state) + # Wait again after waiting for the selector, helpful with protections like Cloudflare + page.wait_for_load_state(state="load") + page.wait_for_load_state(state="domcontentloaded") + if self.network_idle: + page.wait_for_load_state('networkidle') # This will be parsed inside `Response` encoding = res.headers.get('content-type', '') or 'utf-8' # default encoding