From 0c31fb7c0e6caee347767a8ff9dea83eb266a8af Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Sat, 15 Mar 2025 01:45:12 +0200 Subject: [PATCH] refactor/fix(Playwright engine): Improvements and fixes --- scrapling/engines/pw.py | 171 +++++++++++++++++++++++----------------- 1 file changed, 98 insertions(+), 73 deletions(-) diff --git a/scrapling/engines/pw.py b/scrapling/engines/pw.py index 6b8003c..ce60a51 100644 --- a/scrapling/engines/pw.py +++ b/scrapling/engines/pw.py @@ -19,20 +19,20 @@ class PlaywrightEngine: self, headless: Union[bool, str] = True, disable_resources: bool = False, useragent: Optional[str] = None, - network_idle: Optional[bool] = False, + network_idle: bool = False, timeout: Optional[float] = 30000, page_action: Callable = None, wait_selector: Optional[str] = None, locale: Optional[str] = 'en-US', wait_selector_state: SelectorWaitStates = 'attached', - stealth: Optional[bool] = False, - real_chrome: Optional[bool] = False, - hide_canvas: Optional[bool] = False, - disable_webgl: Optional[bool] = False, + stealth: bool = False, + real_chrome: bool = False, + hide_canvas: bool = False, + disable_webgl: bool = False, cdp_url: Optional[str] = None, - nstbrowser_mode: Optional[bool] = False, + nstbrowser_mode: bool = False, nstbrowser_config: Optional[Dict] = None, - google_search: Optional[bool] = True, + google_search: bool = True, extra_headers: Optional[Dict[str, str]] = None, proxy: Optional[Union[str, Dict[str, str]]] = None, adaptor_arguments: Dict = None @@ -188,6 +188,38 @@ class PlaywrightEngine: ) ) + def _process_response_history(self, first_response): + """Process response history to build a list of Response objects""" + history = [] + current_request = first_response.request.redirected_from + + try: + while current_request: + try: + current_response = current_request.response() + history.insert(0, Response( + url=current_request.url, + # using current_response.text() will trigger "Error: Response.text: Response body is unavailable for redirect responses" + text='', + body=b'', + status=current_response.status if current_response else 301, + reason=(current_response.status_text or StatusText.get(current_response.status)) if current_response else StatusText.get(301), + encoding=current_response.headers.get('content-type', '') or 'utf-8', + cookies={}, + headers=current_response.all_headers() if current_response else {}, + request_headers=current_request.all_headers(), + **self.adaptor_arguments + )) + except Exception as e: + log.error(f"Error processing redirect: {e}") + break + + current_request = current_request.redirected_from + except Exception as e: + log.error(f"Error processing response history: {e}") + + return history + def fetch(self, url: str) -> Response: """Opens up the browser and do your request based on your chosen options. @@ -201,8 +233,8 @@ class PlaywrightEngine: else: from rebrowser_playwright.sync_api import sync_playwright - # Store the final response final_response = None + referer = generate_convincing_referer(url) if self.google_search else None def handle_response(finished_response: PlaywrightResponse): nonlocal final_response @@ -218,11 +250,9 @@ class PlaywrightEngine: browser = p.chromium.launch(**self.__launch_kwargs()) context = browser.new_context(**self.__context_kwargs()) - # Finally we are in business page = context.new_page() page.set_default_navigation_timeout(self.timeout) page.set_default_timeout(self.timeout) - # Listen for all responses page.on("response", handle_response) if self.extra_headers: @@ -235,54 +265,51 @@ class PlaywrightEngine: for script in self.__stealth_scripts(): page.add_init_script(path=script) - first_response = page.goto(url, referer=generate_convincing_referer(url) if self.google_search else None) + first_response = page.goto(url, referer=referer) page.wait_for_load_state(state="domcontentloaded") + if self.network_idle: page.wait_for_load_state('networkidle') if self.page_action is not None: - page = self.page_action(page) + try: + page = self.page_action(page) + except Exception as e: + log.error(f"Error executing page_action: {e}") if self.wait_selector and type(self.wait_selector) is str: - waiter = page.locator(self.wait_selector) - waiter.first.wait_for(state=self.wait_selector_state) - # Wait again after waiting for the selector, helpful with protections like Cloudflare - page.wait_for_load_state(state="load") - page.wait_for_load_state(state="domcontentloaded") - if self.network_idle: - page.wait_for_load_state('networkidle') + try: + waiter = page.locator(self.wait_selector) + waiter.first.wait_for(state=self.wait_selector_state) + # Wait again after waiting for the selector, helpful with protections like Cloudflare + page.wait_for_load_state(state="load") + page.wait_for_load_state(state="domcontentloaded") + if self.network_idle: + page.wait_for_load_state('networkidle') + except Exception as e: + log.error(f"Error waiting for selector {self.wait_selector}: {e}") # In case we didn't catch a document type somehow final_response = final_response if final_response else first_response + if not final_response: + raise ValueError("Failed to get a response from the page") + # This will be parsed inside `Response` encoding = final_response.headers.get('content-type', '') or 'utf-8' # default encoding # PlayWright API sometimes give empty status text for some reason! status_text = final_response.status_text or StatusText.get(final_response.status) - history = [] - current_request = first_response.request.redirected_from - while current_request: - current_response = current_request.response() - - history.insert(0, Response( - url=current_request.url, - # using current_response.text() will trigger "Error: Response.text: Response body is unavailable for redirect responses" - text='', - body=b'', - status=current_response.status if current_response else 301, - reason=(current_response.status_text or StatusText.get(current_response.status)) if current_response else StatusText.get(301), - encoding=current_response.headers.get('content-type', '') or 'utf-8', - cookies={}, - headers=current_response.all_headers() if current_response else {}, - request_headers=current_request.all_headers(), - **self.adaptor_arguments - )) - current_request = current_request.redirected_from + history = self._process_response_history(first_response) + try: + page_content = page.content() + except Exception as e: + log.error(f"Error getting page content: {e}") + page_content = "" response = Response( url=page.url, - text=page.content(), - body=page.content().encode('utf-8'), + text=page_content, + body=page_content.encode('utf-8'), status=final_response.status, reason=status_text, encoding=encoding, @@ -293,6 +320,7 @@ class PlaywrightEngine: **self.adaptor_arguments ) page.close() + context.close() return response async def async_fetch(self, url: str) -> Response: @@ -308,8 +336,8 @@ class PlaywrightEngine: else: from rebrowser_playwright.async_api import async_playwright - # Store the final response final_response = None + referer = generate_convincing_referer(url) if self.google_search else None async def handle_response(finished_response: PlaywrightResponse): nonlocal final_response @@ -325,11 +353,9 @@ class PlaywrightEngine: browser = await p.chromium.launch(**self.__launch_kwargs()) context = await browser.new_context(**self.__context_kwargs()) - # Finally we are in business page = await context.new_page() page.set_default_navigation_timeout(self.timeout) page.set_default_timeout(self.timeout) - # Listen for all responses page.on("response", handle_response) if self.extra_headers: @@ -342,54 +368,51 @@ class PlaywrightEngine: for script in self.__stealth_scripts(): await page.add_init_script(path=script) - first_response = await page.goto(url, referer=generate_convincing_referer(url) if self.google_search else None) + first_response = await page.goto(url, referer=referer) await page.wait_for_load_state(state="domcontentloaded") + if self.network_idle: await page.wait_for_load_state('networkidle') if self.page_action is not None: - page = await self.page_action(page) + try: + page = await self.page_action(page) + except Exception as e: + log.error(f"Error executing async page_action: {e}") if self.wait_selector and type(self.wait_selector) is str: - waiter = page.locator(self.wait_selector) - await waiter.first.wait_for(state=self.wait_selector_state) - # Wait again after waiting for the selector, helpful with protections like Cloudflare - await page.wait_for_load_state(state="load") - await page.wait_for_load_state(state="domcontentloaded") - if self.network_idle: - await page.wait_for_load_state('networkidle') + try: + waiter = page.locator(self.wait_selector) + await waiter.first.wait_for(state=self.wait_selector_state) + # Wait again after waiting for the selector, helpful with protections like Cloudflare + await page.wait_for_load_state(state="load") + await page.wait_for_load_state(state="domcontentloaded") + if self.network_idle: + await page.wait_for_load_state('networkidle') + except Exception as e: + log.error(f"Error waiting for selector {self.wait_selector}: {e}") # In case we didn't catch a document type somehow final_response = final_response if final_response else first_response + if not final_response: + raise ValueError("Failed to get a response from the page") + # This will be parsed inside `Response` encoding = final_response.headers.get('content-type', '') or 'utf-8' # default encoding # PlayWright API sometimes give empty status text for some reason! status_text = final_response.status_text or StatusText.get(final_response.status) - history = [] - current_request = first_response.request.redirected_from - while current_request: - current_response = await current_request.response() - - history.insert(0, Response( - url=current_request.url, - # using current_response.text() will trigger "Error: Response.text: Response body is unavailable for redirect responses" - text='', - body=b'', - status=current_response.status if current_response else 301, - reason=(current_response.status_text or StatusText.get(current_response.status)) if current_response else StatusText.get(301), - encoding=current_response.headers.get('content-type', '') or 'utf-8', - cookies={}, - headers=await current_response.all_headers() if current_response else {}, - request_headers=await current_request.all_headers(), - **self.adaptor_arguments - )) - current_request = current_request.redirected_from + history = self._process_response_history(first_response) + try: + page_content = await page.content() + except Exception as e: + log.error(f"Error getting page content in async: {e}") + page_content = "" response = Response( url=page.url, - text=await page.content(), - body=(await page.content()).encode('utf-8'), + text=page_content, + body=page_content.encode('utf-8'), status=final_response.status, reason=status_text, encoding=encoding, @@ -400,4 +423,6 @@ class PlaywrightEngine: **self.adaptor_arguments ) await page.close() + await context.close() + return response