freedak/Scrapling
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(StealthyFetcher): Use more dependable response data

Browse Source
This commit is contained in:
Karim shoair
2024-12-26 00:29:42 +02:00
parent f9b85cfbac
commit 2006be2173
1 changed files with 10 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scrapling/engines/camo.py
+10 -12
View File
@@ -89,7 +89,7 @@ class CamoufoxEngine:
def handle_response(finished_response):
nonlocal final_response
if finished_response.request.resource_type == "document":
if finished_response.request.resource_type == "document" and finished_response.request.is_navigation_request():
final_response = finished_response
with Camoufox(
@@ -133,7 +133,6 @@ class CamoufoxEngine:
if self.network_idle:
page.wait_for_load_state('networkidle')
response_bytes = final_response.body() if final_response else page.content().encode('utf-8')
# In case we didn't catch a document type somehow
final_response = final_response if final_response else first_response
# This will be parsed inside `Response`
@@ -142,15 +141,15 @@ class CamoufoxEngine:
status_text = final_response.status_text or StatusText.get(final_response.status)
response = Response(
url=final_response.url,
url=page.url,
text=page.content(),
body=response_bytes,
body=page.content().encode('utf-8'),
status=final_response.status,
reason=status_text,
encoding=encoding,
cookies={cookie['name']: cookie['value'] for cookie in page.context.cookies()},
headers=final_response.all_headers(),
request_headers=final_response.request.all_headers(),
headers=first_response.all_headers(),
request_headers=first_response.request.all_headers(),
**self.adaptor_arguments
)
page.close()
@@ -169,7 +168,7 @@ class CamoufoxEngine:
async def handle_response(finished_response):
nonlocal final_response
if finished_response.request.resource_type == "document":
if finished_response.request.resource_type == "document" and finished_response.request.is_navigation_request():
final_response = finished_response
async with AsyncCamoufox(
@@ -213,7 +212,6 @@ class CamoufoxEngine:
if self.network_idle:
await page.wait_for_load_state('networkidle')
response_bytes = await final_response.body() if final_response else (await page.content()).encode('utf-8')
# In case we didn't catch a document type somehow
final_response = final_response if final_response else first_response
# This will be parsed inside `Response`
@@ -222,15 +220,15 @@ class CamoufoxEngine:
status_text = final_response.status_text or StatusText.get(final_response.status)
response = Response(
url=final_response.url,
url=page.url,
text=await page.content(),
body=response_bytes,
body=(await page.content()).encode('utf-8'),
status=final_response.status,
reason=status_text,
encoding=encoding,
cookies={cookie['name']: cookie['value'] for cookie in await page.context.cookies()},
headers=await final_response.all_headers(),
request_headers=await final_response.request.all_headers(),
headers=await first_response.all_headers(),
request_headers=await first_response.request.all_headers(),
**self.adaptor_arguments
)
await page.close()