feat(cf solver): Make solver able to handle Turnstile and interstitial

scrapling/engines/_browsers/_base.py

@@ -12,17 +12,13 @@ from camoufox.utils import (
     installed_verstr as camoufox_version,
 )
 
-from scrapling.engines.toolbelt.navigation import intercept_route, async_intercept_route
-from scrapling.core._types import (
-    Any,
-    Dict,
-    Optional,
-)
 from ._page import PageInfo, PagePool
-from ._config_tools import _compiled_stealth_scripts
+from scrapling.parser import Selector
-from ._config_tools import _launch_kwargs, _context_kwargs
+from scrapling.core._types import Dict, Optional
 from scrapling.engines.toolbelt.fingerprints import get_os_name
 from ._validators import validate, PlaywrightConfig, CamoufoxConfig
+from ._config_tools import _compiled_stealth_scripts, _launch_kwargs, _context_kwargs
+from scrapling.engines.toolbelt.navigation import intercept_route, async_intercept_route
 
 __ff_version_str__ = camoufox_version().split(".", 1)[0]
 
@@ -268,4 +264,9 @@ class StealthySessionMixin:
             if f"cType: '{ctype}'" in page_content:
                 return ctype
 
+        # Check if turnstile captcha is embedded inside the page (Usually inside a closed Shadow iframe)
+        selector = Selector(content=page_content)
+        if selector.css('script[src*="challenges.cloudflare.com/turnstile/v"]'):
+            return "embedded"
+
         return None

scrapling/engines/_browsers/_camoufox.py

@@ -237,26 +237,33 @@ class StealthySession(StealthySessionMixin, SyncSession):
                 return
 
             else:
-                while "Verifying you are human." in self._get_page_content(page):
+                box_selector = "#cf_turnstile div, #cf-turnstile div, .turnstile>div>div"
-                    # Waiting for the verify spinner to disappear, checking every 1s if it disappeared
+                if challenge_type != "embedded":
-                    page.wait_for_timeout(500)
+                    box_selector = ".main-content p+div>div>div"
+                    while "Verifying you are human." in self._get_page_content(page):
+                        # Waiting for the verify spinner to disappear, checking every 1s if it disappeared
+                        page.wait_for_timeout(500)
 
                 iframe = page.frame(url=__CF_PATTERN__)
                 if iframe is None:
-                    log.info("Didn't find Cloudflare iframe!")
+                    log.error("Didn't find Cloudflare iframe!")
                     return
 
-                while not iframe.frame_element().is_visible():
+                if challenge_type != "embedded":
-                    # Double-checking that the iframe is loaded
+                    while not iframe.frame_element().is_visible():
-                    page.wait_for_timeout(500)
+                        # Double-checking that the iframe is loaded
+                        page.wait_for_timeout(500)
 
+                iframe.wait_for_load_state(state="domcontentloaded")
+                iframe.wait_for_load_state("networkidle")
                 # Calculate the Captcha coordinates for any viewport
-                outer_box = page.locator(".main-content p+div>div>div").bounding_box()
+                outer_box = page.locator(box_selector).last.bounding_box()
                 captcha_x, captcha_y = outer_box["x"] + 26, outer_box["y"] + 25
 
                 # Move the mouse to the center of the window, then press and hold the left mouse button
                 page.mouse.click(captcha_x, captcha_y, delay=60, button="left")
-                page.locator(".zone-name-title").wait_for(state="hidden")
+                if challenge_type != "embedded":
+                    page.locator(".zone-name-title").wait_for(state="hidden")
                 page.wait_for_load_state(state="domcontentloaded")
 
                 log.info("Cloudflare captcha is solved")
@@ -556,26 +563,33 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):
                 return
 
             else:
-                while "Verifying you are human." in (await self._get_page_content(page)):
+                box_selector = "#cf_turnstile div, #cf-turnstile div, .turnstile>div>div"
-                    # Waiting for the verify spinner to disappear, checking every 1s if it disappeared
+                if challenge_type != "embedded":
-                    await page.wait_for_timeout(500)
+                    box_selector = ".main-content p+div>div>div"
+                    while "Verifying you are human." in (await self._get_page_content(page)):
+                        # Waiting for the verify spinner to disappear, checking every 1s if it disappeared
+                        await page.wait_for_timeout(500)
 
                 iframe = page.frame(url=__CF_PATTERN__)
                 if iframe is None:
-                    log.info("Didn't find Cloudflare iframe!")
+                    log.error("Didn't find Cloudflare iframe!")
                     return
 
-                while not await (await iframe.frame_element()).is_visible():
+                if challenge_type != "embedded":
-                    # Double-checking that the iframe is loaded
+                    while not await (await iframe.frame_element()).is_visible():
-                    await page.wait_for_timeout(500)
+                        # Double-checking that the iframe is loaded
+                        await page.wait_for_timeout(500)
 
+                await iframe.wait_for_load_state(state="domcontentloaded")
+                await iframe.wait_for_load_state("networkidle")
                 # Calculate the Captcha coordinates for any viewport
-                outer_box = await page.locator(".main-content p+div>div>div").bounding_box()
+                outer_box = await page.locator(box_selector).last.bounding_box()
                 captcha_x, captcha_y = outer_box["x"] + 26, outer_box["y"] + 25
 
                 # Move the mouse to the center of the window, then press and hold the left mouse button
                 await page.mouse.click(captcha_x, captcha_y, delay=60, button="left")
-                await page.locator(".zone-name-title").wait_for(state="hidden")
+                if challenge_type != "embedded":
+                    await page.locator(".zone-name-title").wait_for(state="hidden")
                 await page.wait_for_load_state(state="domcontentloaded")
 
                 log.info("Cloudflare captcha is solved")