diff --git a/scrapling/engines/camo.py b/scrapling/engines/camo.py index 1eb9976..2c0ff3d 100644 --- a/scrapling/engines/camo.py +++ b/scrapling/engines/camo.py @@ -84,6 +84,14 @@ class CamoufoxEngine: :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers` """ addons = [] if self.disable_ads else [DefaultAddons.UBO] + # Store the final response + final_response = None + + def handle_response(finished_response): + nonlocal final_response + if finished_response.request.resource_type == "document": + final_response = finished_response + with Camoufox( geoip=self.geoip, proxy=self.proxy, @@ -100,13 +108,15 @@ class CamoufoxEngine: page = browser.new_page() page.set_default_navigation_timeout(self.timeout) page.set_default_timeout(self.timeout) + # Listen for all responses + page.on("response", handle_response) if self.disable_resources: page.route("**/*", intercept_route) if self.extra_headers: page.set_extra_http_headers(self.extra_headers) - res = page.goto(url, referer=generate_convincing_referer(url) if self.google_search else None) + first_response = page.goto(url, referer=generate_convincing_referer(url) if self.google_search else None) page.wait_for_load_state(state="domcontentloaded") if self.network_idle: page.wait_for_load_state('networkidle') @@ -123,21 +133,24 @@ class CamoufoxEngine: if self.network_idle: page.wait_for_load_state('networkidle') + response_bytes = final_response.body() if final_response else page.content().encode('utf-8') + # In case we didn't catch a document type somehow + final_response = final_response if final_response else first_response # This will be parsed inside `Response` - encoding = res.headers.get('content-type', '') or 'utf-8' # default encoding + encoding = final_response.headers.get('content-type', '') or 'utf-8' # default encoding # PlayWright API sometimes give empty status text for some reason! - status_text = res.status_text or StatusText.get(res.status) + status_text = final_response.status_text or StatusText.get(final_response.status) response = Response( - url=res.url, + url=final_response.url, text=page.content(), - body=page.content().encode('utf-8'), - status=res.status, + body=response_bytes, + status=final_response.status, reason=status_text, encoding=encoding, cookies={cookie['name']: cookie['value'] for cookie in page.context.cookies()}, - headers=res.all_headers(), - request_headers=res.request.all_headers(), + headers=final_response.all_headers(), + request_headers=final_response.request.all_headers(), **self.adaptor_arguments ) page.close() @@ -151,6 +164,14 @@ class CamoufoxEngine: :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers` """ addons = [] if self.disable_ads else [DefaultAddons.UBO] + # Store the final response + final_response = None + + async def handle_response(finished_response): + nonlocal final_response + if finished_response.request.resource_type == "document": + final_response = finished_response + async with AsyncCamoufox( geoip=self.geoip, proxy=self.proxy, @@ -167,13 +188,15 @@ class CamoufoxEngine: page = await browser.new_page() page.set_default_navigation_timeout(self.timeout) page.set_default_timeout(self.timeout) + # Listen for all responses + page.on("response", handle_response) if self.disable_resources: await page.route("**/*", async_intercept_route) if self.extra_headers: await page.set_extra_http_headers(self.extra_headers) - res = await page.goto(url, referer=generate_convincing_referer(url) if self.google_search else None) + first_response = await page.goto(url, referer=generate_convincing_referer(url) if self.google_search else None) await page.wait_for_load_state(state="domcontentloaded") if self.network_idle: await page.wait_for_load_state('networkidle') @@ -190,21 +213,24 @@ class CamoufoxEngine: if self.network_idle: await page.wait_for_load_state('networkidle') + response_bytes = await final_response.body() if final_response else (await page.content()).encode('utf-8') + # In case we didn't catch a document type somehow + final_response = final_response if final_response else first_response # This will be parsed inside `Response` - encoding = res.headers.get('content-type', '') or 'utf-8' # default encoding + encoding = final_response.headers.get('content-type', '') or 'utf-8' # default encoding # PlayWright API sometimes give empty status text for some reason! - status_text = res.status_text or StatusText.get(res.status) + status_text = final_response.status_text or StatusText.get(final_response.status) response = Response( - url=res.url, + url=final_response.url, text=await page.content(), - body=(await page.content()).encode('utf-8'), - status=res.status, + body=response_bytes, + status=final_response.status, reason=status_text, encoding=encoding, cookies={cookie['name']: cookie['value'] for cookie in await page.context.cookies()}, - headers=await res.all_headers(), - request_headers=await res.request.all_headers(), + headers=await final_response.all_headers(), + request_headers=await final_response.request.all_headers(), **self.adaptor_arguments ) await page.close()