freedak/Scrapling
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(solver): Solve CF faster and handle websites that show captcha twice

Browse Source
This commit is contained in:
Karim shoair
2026-02-27 02:32:21 +02:00
parent ef29568cd0
commit d750fec468
2 changed files with 41 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+3
View File
@@ -1,4 +1,7 @@
# local files
site/* site/*
local_tests/*
.mcpregistry_*
# AI related files # AI related files
.claude/* .claude/*
scrapling/engines/_browsers/_stealth.py
+36 -24
View File
@@ -127,7 +127,7 @@ class StealthySession(SyncSession, StealthySessionMixin):
challenge_type = self._detect_cloudflare(ResponseFactory._get_page_content(page)) challenge_type = self._detect_cloudflare(ResponseFactory._get_page_content(page))
if not challenge_type: if not challenge_type:
log.error("No Cloudflare challenge found.") log.error("No Cloudflare challenge found.")
return return None
else: else:
log.info(f'The turnstile version discovered is "{challenge_type}"') log.info(f'The turnstile version discovered is "{challenge_type}"')
if challenge_type == "non-interactive": if challenge_type == "non-interactive":
@@ -136,7 +136,7 @@ class StealthySession(SyncSession, StealthySessionMixin):
page.wait_for_timeout(1000) page.wait_for_timeout(1000)
page.wait_for_load_state() page.wait_for_load_state()
log.info("Cloudflare captcha is solved") log.info("Cloudflare captcha is solved")
return return None
else: else:
box_selector = "#cf_turnstile div, #cf-turnstile div, .turnstile>div>div" box_selector = "#cf_turnstile div, #cf-turnstile div, .turnstile>div>div"
@@ -161,7 +161,7 @@ class StealthySession(SyncSession, StealthySessionMixin):
if not iframe or not outer_box: if not iframe or not outer_box:
if "<title>Just a moment...</title>" not in (ResponseFactory._get_page_content(page)): if "<title>Just a moment...</title>" not in (ResponseFactory._get_page_content(page)):
log.info("Cloudflare captcha is solved") log.info("Cloudflare captcha is solved")
return return None
outer_box = page.locator(box_selector).last.bounding_box() outer_box = page.locator(box_selector).last.bounding_box()
@@ -171,22 +171,28 @@ class StealthySession(SyncSession, StealthySessionMixin):
# Move the mouse to the center of the window, then press and hold the left mouse button # Move the mouse to the center of the window, then press and hold the left mouse button
page.mouse.click(captcha_x, captcha_y, delay=randint(100, 200), button="left") page.mouse.click(captcha_x, captcha_y, delay=randint(100, 200), button="left")
self._wait_for_networkidle(page) self._wait_for_networkidle(page)
if iframe is not None:
# Wait for the frame to be removed from the page (with 30s timeout = 300 iterations * 100 ms) if challenge_type != "embedded":
attempts = 0 attempts = 0
while iframe in page.frames: while "<title>Just a moment...</title>" in ResponseFactory._get_page_content(page):
if attempts >= 300: # Wait for the page
log.info("Cloudflare iframe didn't disappear after 30s, continuing...") if attempts >= 100:
log.info("Cloudflare page didn't disappear after 10s, continuing...")
break break
page.wait_for_timeout(100) page.wait_for_timeout(100)
attempts += 1 attempts += 1
if challenge_type != "embedded":
page.locator(box_selector).last.wait_for(state="detached") # page.locator(box_selector).last.wait_for(state="detached")
page.locator(".zone-name-title").wait_for(state="hidden") # page.locator(".zone-name-title").wait_for(state="hidden")
self._wait_for_page_stability(page, True, False) self._wait_for_page_stability(page, True, False)
if "<title>Just a moment...</title>" not in (ResponseFactory._get_page_content(page)):
log.info("Cloudflare captcha is solved") log.info("Cloudflare captcha is solved")
return return None
else:
log.info("Looks like Cloudflare captcha is still present, solving again")
return self._cloudflare_solver(page)
def fetch(self, url: str, **kwargs: Unpack[StealthFetchParams]) -> Response: def fetch(self, url: str, **kwargs: Unpack[StealthFetchParams]) -> Response:
"""Opens up the browser and do your request based on your chosen options. """Opens up the browser and do your request based on your chosen options.
@@ -384,7 +390,7 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin):
challenge_type = self._detect_cloudflare(await ResponseFactory._get_async_page_content(page)) challenge_type = self._detect_cloudflare(await ResponseFactory._get_async_page_content(page))
if not challenge_type: if not challenge_type:
log.error("No Cloudflare challenge found.") log.error("No Cloudflare challenge found.")
return return None
else: else:
log.info(f'The turnstile version discovered is "{challenge_type}"') log.info(f'The turnstile version discovered is "{challenge_type}"')
if challenge_type == "non-interactive": if challenge_type == "non-interactive":
@@ -393,7 +399,7 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin):
await page.wait_for_timeout(1000) await page.wait_for_timeout(1000)
await page.wait_for_load_state() await page.wait_for_load_state()
log.info("Cloudflare captcha is solved") log.info("Cloudflare captcha is solved")
return return None
else: else:
box_selector = "#cf_turnstile div, #cf-turnstile div, .turnstile>div>div" box_selector = "#cf_turnstile div, #cf-turnstile div, .turnstile>div>div"
@@ -418,7 +424,7 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin):
if not iframe or not outer_box: if not iframe or not outer_box:
if "<title>Just a moment...</title>" not in (await ResponseFactory._get_async_page_content(page)): if "<title>Just a moment...</title>" not in (await ResponseFactory._get_async_page_content(page)):
log.info("Cloudflare captcha is solved") log.info("Cloudflare captcha is solved")
return return None
outer_box = await page.locator(box_selector).last.bounding_box() outer_box = await page.locator(box_selector).last.bounding_box()
@@ -428,22 +434,28 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin):
# Move the mouse to the center of the window, then press and hold the left mouse button # Move the mouse to the center of the window, then press and hold the left mouse button
await page.mouse.click(captcha_x, captcha_y, delay=randint(100, 200), button="left") await page.mouse.click(captcha_x, captcha_y, delay=randint(100, 200), button="left")
await self._wait_for_networkidle(page) await self._wait_for_networkidle(page)
if iframe is not None:
# Wait for the frame to be removed from the page (with 30s timeout = 300 iterations * 100 ms) if challenge_type != "embedded":
attempts = 0 attempts = 0
while iframe in page.frames: while "<title>Just a moment...</title>" in (await ResponseFactory._get_async_page_content(page)):
if attempts >= 300: # Wait for the page
log.info("Cloudflare iframe didn't disappear after 30s, continuing...") if attempts >= 100:
log.info("Cloudflare page didn't disappear after 10s, continuing...")
break break
await page.wait_for_timeout(100) await page.wait_for_timeout(100)
attempts += 1 attempts += 1
if challenge_type != "embedded":
await page.locator(box_selector).last.wait_for(state="detached") # await page.locator(box_selector).last.wait_for(state="detached")
await page.locator(".zone-name-title").wait_for(state="hidden") # await page.locator(".zone-name-title").wait_for(state="hidden")
await self._wait_for_page_stability(page, True, False) await self._wait_for_page_stability(page, True, False)
if "<title>Just a moment...</title>" not in (await ResponseFactory._get_async_page_content(page)):
log.info("Cloudflare captcha is solved") log.info("Cloudflare captcha is solved")
return return None
else:
log.info("Looks like Cloudflare captcha is still present, solving again")
return await self._cloudflare_solver(page)
async def fetch(self, url: str, **kwargs: Unpack[StealthFetchParams]) -> Response: async def fetch(self, url: str, **kwargs: Unpack[StealthFetchParams]) -> Response:
"""Opens up the browser and do your request based on your chosen options. """Opens up the browser and do your request based on your chosen options.