From 128f7b9d6f350cecc24c0b32c22593ddb533b6ec Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Tue, 8 Apr 2025 02:54:46 +0200 Subject: [PATCH] feat(PlayWrightFetcher): adding the option to `sleep` after fetch before closing the page --- scrapling/engines/pw.py | 5 +++++ scrapling/fetchers.py | 12 ++++++++---- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/scrapling/engines/pw.py b/scrapling/engines/pw.py index 0c71fec..5b55a95 100644 --- a/scrapling/engines/pw.py +++ b/scrapling/engines/pw.py @@ -21,6 +21,7 @@ class PlaywrightEngine: useragent: Optional[str] = None, network_idle: bool = False, timeout: Optional[float] = 30000, + wait: Optional[int] = 0, page_action: Callable = None, wait_selector: Optional[str] = None, locale: Optional[str] = 'en-US', @@ -46,6 +47,7 @@ class PlaywrightEngine: :param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it. :param network_idle: Wait for the page until there are no network connections for at least 500 ms. :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30000 + :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning `Response` object. :param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again. :param wait_selector: Wait for a specific css selector to be in a specific state. :param locale: Set the locale for the browser if wanted. The default value is `en-US`. @@ -76,6 +78,7 @@ class PlaywrightEngine: self.cdp_url = cdp_url self.useragent = useragent self.timeout = check_type_validity(timeout, [int, float], 30000) + self.wait = check_type_validity(wait, [int, float], 0) if page_action is not None: if callable(page_action): self.page_action = page_action @@ -289,6 +292,7 @@ class PlaywrightEngine: except Exception as e: log.error(f"Error waiting for selector {self.wait_selector}: {e}") + page.wait_for_timeout(self.wait) # In case we didn't catch a document type somehow final_response = final_response if final_response else first_response if not final_response: @@ -392,6 +396,7 @@ class PlaywrightEngine: except Exception as e: log.error(f"Error waiting for selector {self.wait_selector}: {e}") + await page.wait_for_timeout(self.wait) # In case we didn't catch a document type somehow final_response = final_response if final_response else first_response if not final_response: diff --git a/scrapling/fetchers.py b/scrapling/fetchers.py index 3848ca6..84596cd 100644 --- a/scrapling/fetchers.py +++ b/scrapling/fetchers.py @@ -384,7 +384,7 @@ class PlayWrightFetcher(BaseFetcher): @classmethod def fetch( cls, url: str, headless: Union[bool, str] = True, disable_resources: bool = None, - useragent: Optional[str] = None, network_idle: bool = False, timeout: Optional[float] = 30000, + useragent: Optional[str] = None, network_idle: bool = False, timeout: Optional[float] = 30000, wait: Optional[int] = 0, page_action: Optional[Callable] = None, wait_selector: Optional[str] = None, wait_selector_state: SelectorWaitStates = 'attached', hide_canvas: bool = False, disable_webgl: bool = False, extra_headers: Optional[Dict[str, str]] = None, google_search: bool = True, proxy: Optional[Union[str, Dict[str, str]]] = None, locale: Optional[str] = 'en-US', @@ -402,7 +402,8 @@ class PlayWrightFetcher(BaseFetcher): This can help save your proxy usage but be careful with this option as it makes some websites never finish loading. :param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it. :param network_idle: Wait for the page until there are no network connections for at least 500 ms. - :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30000 + :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30000. + :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning `Response` object. :param locale: Set the locale for the browser if wanted. The default value is `en-US`. :param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again. :param wait_selector: Wait for a specific css selector to be in a specific state. @@ -426,6 +427,7 @@ class PlayWrightFetcher(BaseFetcher): ValueError(f"The custom parser config must be of type dictionary, got {cls.__class__}") engine = PlaywrightEngine( + wait=wait, proxy=proxy, locale=locale, timeout=timeout, @@ -452,7 +454,7 @@ class PlayWrightFetcher(BaseFetcher): @classmethod async def async_fetch( cls, url: str, headless: Union[bool, str] = True, disable_resources: bool = None, - useragent: Optional[str] = None, network_idle: bool = False, timeout: Optional[float] = 30000, + useragent: Optional[str] = None, network_idle: bool = False, timeout: Optional[float] = 30000, wait: Optional[int] = 0, page_action: Optional[Callable] = None, wait_selector: Optional[str] = None, wait_selector_state: SelectorWaitStates = 'attached', hide_canvas: bool = False, disable_webgl: bool = False, extra_headers: Optional[Dict[str, str]] = None, google_search: bool = True, proxy: Optional[Union[str, Dict[str, str]]] = None, locale: Optional[str] = 'en-US', @@ -470,7 +472,8 @@ class PlayWrightFetcher(BaseFetcher): This can help save your proxy usage but be careful with this option as it makes some websites never finish loading. :param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it. :param network_idle: Wait for the page until there are no network connections for at least 500 ms. - :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30000 + :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30000. + :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning `Response` object. :param locale: Set the locale for the browser if wanted. The default value is `en-US`. :param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again. :param wait_selector: Wait for a specific css selector to be in a specific state. @@ -494,6 +497,7 @@ class PlayWrightFetcher(BaseFetcher): ValueError(f"The custom parser config must be of type dictionary, got {cls.__class__}") engine = PlaywrightEngine( + wait=wait, proxy=proxy, locale=locale, timeout=timeout,