diff --git a/scrapling/engines/_browsers/_base.py b/scrapling/engines/_browsers/_base.py index 69681ee..d1a872d 100644 --- a/scrapling/engines/_browsers/_base.py +++ b/scrapling/engines/_browsers/_base.py @@ -12,17 +12,13 @@ from camoufox.utils import ( installed_verstr as camoufox_version, ) -from scrapling.engines.toolbelt.navigation import intercept_route, async_intercept_route -from scrapling.core._types import ( - Any, - Dict, - Optional, -) from ._page import PageInfo, PagePool -from ._config_tools import _compiled_stealth_scripts -from ._config_tools import _launch_kwargs, _context_kwargs +from scrapling.parser import Selector +from scrapling.core._types import Dict, Optional from scrapling.engines.toolbelt.fingerprints import get_os_name from ._validators import validate, PlaywrightConfig, CamoufoxConfig +from ._config_tools import _compiled_stealth_scripts, _launch_kwargs, _context_kwargs +from scrapling.engines.toolbelt.navigation import intercept_route, async_intercept_route __ff_version_str__ = camoufox_version().split(".", 1)[0] @@ -268,4 +264,9 @@ class StealthySessionMixin: if f"cType: '{ctype}'" in page_content: return ctype + # Check if turnstile captcha is embedded inside the page (Usually inside a closed Shadow iframe) + selector = Selector(content=page_content) + if selector.css('script[src*="challenges.cloudflare.com/turnstile/v"]'): + return "embedded" + return None diff --git a/scrapling/engines/_browsers/_camoufox.py b/scrapling/engines/_browsers/_camoufox.py index 5f0e795..207be90 100644 --- a/scrapling/engines/_browsers/_camoufox.py +++ b/scrapling/engines/_browsers/_camoufox.py @@ -237,26 +237,33 @@ class StealthySession(StealthySessionMixin, SyncSession): return else: - while "Verifying you are human." in self._get_page_content(page): - # Waiting for the verify spinner to disappear, checking every 1s if it disappeared - page.wait_for_timeout(500) + box_selector = "#cf_turnstile div, #cf-turnstile div, .turnstile>div>div" + if challenge_type != "embedded": + box_selector = ".main-content p+div>div>div" + while "Verifying you are human." in self._get_page_content(page): + # Waiting for the verify spinner to disappear, checking every 1s if it disappeared + page.wait_for_timeout(500) iframe = page.frame(url=__CF_PATTERN__) if iframe is None: - log.info("Didn't find Cloudflare iframe!") + log.error("Didn't find Cloudflare iframe!") return - while not iframe.frame_element().is_visible(): - # Double-checking that the iframe is loaded - page.wait_for_timeout(500) + if challenge_type != "embedded": + while not iframe.frame_element().is_visible(): + # Double-checking that the iframe is loaded + page.wait_for_timeout(500) + iframe.wait_for_load_state(state="domcontentloaded") + iframe.wait_for_load_state("networkidle") # Calculate the Captcha coordinates for any viewport - outer_box = page.locator(".main-content p+div>div>div").bounding_box() + outer_box = page.locator(box_selector).last.bounding_box() captcha_x, captcha_y = outer_box["x"] + 26, outer_box["y"] + 25 # Move the mouse to the center of the window, then press and hold the left mouse button page.mouse.click(captcha_x, captcha_y, delay=60, button="left") - page.locator(".zone-name-title").wait_for(state="hidden") + if challenge_type != "embedded": + page.locator(".zone-name-title").wait_for(state="hidden") page.wait_for_load_state(state="domcontentloaded") log.info("Cloudflare captcha is solved") @@ -556,26 +563,33 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession): return else: - while "Verifying you are human." in (await self._get_page_content(page)): - # Waiting for the verify spinner to disappear, checking every 1s if it disappeared - await page.wait_for_timeout(500) + box_selector = "#cf_turnstile div, #cf-turnstile div, .turnstile>div>div" + if challenge_type != "embedded": + box_selector = ".main-content p+div>div>div" + while "Verifying you are human." in (await self._get_page_content(page)): + # Waiting for the verify spinner to disappear, checking every 1s if it disappeared + await page.wait_for_timeout(500) iframe = page.frame(url=__CF_PATTERN__) if iframe is None: - log.info("Didn't find Cloudflare iframe!") + log.error("Didn't find Cloudflare iframe!") return - while not await (await iframe.frame_element()).is_visible(): - # Double-checking that the iframe is loaded - await page.wait_for_timeout(500) + if challenge_type != "embedded": + while not await (await iframe.frame_element()).is_visible(): + # Double-checking that the iframe is loaded + await page.wait_for_timeout(500) + await iframe.wait_for_load_state(state="domcontentloaded") + await iframe.wait_for_load_state("networkidle") # Calculate the Captcha coordinates for any viewport - outer_box = await page.locator(".main-content p+div>div>div").bounding_box() + outer_box = await page.locator(box_selector).last.bounding_box() captcha_x, captcha_y = outer_box["x"] + 26, outer_box["y"] + 25 # Move the mouse to the center of the window, then press and hold the left mouse button await page.mouse.click(captcha_x, captcha_y, delay=60, button="left") - await page.locator(".zone-name-title").wait_for(state="hidden") + if challenge_type != "embedded": + await page.locator(".zone-name-title").wait_for(state="hidden") await page.wait_for_load_state(state="domcontentloaded") log.info("Cloudflare captcha is solved")