freedak/Scrapling
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(StealthyFetcher): Wait for the real response status code

Browse Source 
This will return the real status response after redirection for websites that have Cloudflare instead of Playwright API's default behavior.
This commit is contained in:
Karim shoair
2024-12-19 13:08:41 +02:00
parent 844cb0c6a2
commit 94745b1555
1 changed files with 42 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scrapling/engines/camo.py
+42 -16
View File
@@ -84,6 +84,14 @@ class CamoufoxEngine:
:return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
"""
addons = [] if self.disable_ads else [DefaultAddons.UBO]
# Store the final response
final_response = None
def handle_response(finished_response):
nonlocal final_response
if finished_response.request.resource_type == "document":
final_response = finished_response
with Camoufox(
geoip=self.geoip,
proxy=self.proxy,
@@ -100,13 +108,15 @@ class CamoufoxEngine:
page = browser.new_page()
page.set_default_navigation_timeout(self.timeout)
page.set_default_timeout(self.timeout)
# Listen for all responses
page.on("response", handle_response)
if self.disable_resources:
page.route("**/*", intercept_route)
if self.extra_headers:
page.set_extra_http_headers(self.extra_headers)
res = page.goto(url, referer=generate_convincing_referer(url) if self.google_search else None)
first_response = page.goto(url, referer=generate_convincing_referer(url) if self.google_search else None)
page.wait_for_load_state(state="domcontentloaded")
if self.network_idle:
page.wait_for_load_state('networkidle')
@@ -123,21 +133,24 @@ class CamoufoxEngine:
if self.network_idle:
page.wait_for_load_state('networkidle')
response_bytes = final_response.body() if final_response else page.content().encode('utf-8')
# In case we didn't catch a document type somehow
final_response = final_response if final_response else first_response
# This will be parsed inside `Response`
encoding = res.headers.get('content-type', '') or 'utf-8' # default encoding
encoding = final_response.headers.get('content-type', '') or 'utf-8' # default encoding
# PlayWright API sometimes give empty status text for some reason!
status_text = res.status_text or StatusText.get(res.status)
status_text = final_response.status_text or StatusText.get(final_response.status)
response = Response(
url=res.url,
url=final_response.url,
text=page.content(),
body=page.content().encode('utf-8'),
status=res.status,
body=response_bytes,
status=final_response.status,
reason=status_text,
encoding=encoding,
cookies={cookie['name']: cookie['value'] for cookie in page.context.cookies()},
headers=res.all_headers(),
request_headers=res.request.all_headers(),
headers=final_response.all_headers(),
request_headers=final_response.request.all_headers(),
**self.adaptor_arguments
)
page.close()
@@ -151,6 +164,14 @@ class CamoufoxEngine:
:return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
"""
addons = [] if self.disable_ads else [DefaultAddons.UBO]
# Store the final response
final_response = None
async def handle_response(finished_response):
nonlocal final_response
if finished_response.request.resource_type == "document":
final_response = finished_response
async with AsyncCamoufox(
geoip=self.geoip,
proxy=self.proxy,
@@ -167,13 +188,15 @@ class CamoufoxEngine:
page = await browser.new_page()
page.set_default_navigation_timeout(self.timeout)
page.set_default_timeout(self.timeout)
# Listen for all responses
page.on("response", handle_response)
if self.disable_resources:
await page.route("**/*", async_intercept_route)
if self.extra_headers:
await page.set_extra_http_headers(self.extra_headers)
res = await page.goto(url, referer=generate_convincing_referer(url) if self.google_search else None)
first_response = await page.goto(url, referer=generate_convincing_referer(url) if self.google_search else None)
await page.wait_for_load_state(state="domcontentloaded")
if self.network_idle:
await page.wait_for_load_state('networkidle')
@@ -190,21 +213,24 @@ class CamoufoxEngine:
if self.network_idle:
await page.wait_for_load_state('networkidle')
response_bytes = await final_response.body() if final_response else (await page.content()).encode('utf-8')
# In case we didn't catch a document type somehow
final_response = final_response if final_response else first_response
# This will be parsed inside `Response`
encoding = res.headers.get('content-type', '') or 'utf-8' # default encoding
encoding = final_response.headers.get('content-type', '') or 'utf-8' # default encoding
# PlayWright API sometimes give empty status text for some reason!
status_text = res.status_text or StatusText.get(res.status)
status_text = final_response.status_text or StatusText.get(final_response.status)
response = Response(
url=res.url,
url=final_response.url,
text=await page.content(),
body=(await page.content()).encode('utf-8'),
status=res.status,
body=response_bytes,
status=final_response.status,
reason=status_text,
encoding=encoding,
cookies={cookie['name']: cookie['value'] for cookie in await page.context.cookies()},
headers=await res.all_headers(),
request_headers=await res.request.all_headers(),
headers=await final_response.all_headers(),
request_headers=await final_response.request.all_headers(),
**self.adaptor_arguments
)
await page.close()