freedak/Scrapling
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(PlayWrightFetcher): adding the option to sleep after fetch before closing the page

Browse Source
This commit is contained in:
Karim shoair
2025-04-08 02:54:46 +02:00
parent eb4ca2c6a4
commit 128f7b9d6f
2 changed files with 13 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scrapling/engines/pw.py
+5
View File
@@ -21,6 +21,7 @@ class PlaywrightEngine:
useragent: Optional[str] = None, useragent: Optional[str] = None,
network_idle: bool = False, network_idle: bool = False,
timeout: Optional[float] = 30000, timeout: Optional[float] = 30000,
wait: Optional[int] = 0,
page_action: Callable = None, page_action: Callable = None,
wait_selector: Optional[str] = None, wait_selector: Optional[str] = None,
locale: Optional[str] = 'en-US', locale: Optional[str] = 'en-US',
@@ -46,6 +47,7 @@ class PlaywrightEngine:
:param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it. :param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it.
:param network_idle: Wait for the page until there are no network connections for at least 500 ms. :param network_idle: Wait for the page until there are no network connections for at least 500 ms.
:param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30000 :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30000
:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning `Response` object.
:param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again. :param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again.
:param wait_selector: Wait for a specific css selector to be in a specific state. :param wait_selector: Wait for a specific css selector to be in a specific state.
:param locale: Set the locale for the browser if wanted. The default value is `en-US`. :param locale: Set the locale for the browser if wanted. The default value is `en-US`.
@@ -76,6 +78,7 @@ class PlaywrightEngine:
self.cdp_url = cdp_url self.cdp_url = cdp_url
self.useragent = useragent self.useragent = useragent
self.timeout = check_type_validity(timeout, [int, float], 30000) self.timeout = check_type_validity(timeout, [int, float], 30000)
self.wait = check_type_validity(wait, [int, float], 0)
if page_action is not None: if page_action is not None:
if callable(page_action): if callable(page_action):
self.page_action = page_action self.page_action = page_action
@@ -289,6 +292,7 @@ class PlaywrightEngine:
except Exception as e: except Exception as e:
log.error(f"Error waiting for selector {self.wait_selector}: {e}") log.error(f"Error waiting for selector {self.wait_selector}: {e}")
page.wait_for_timeout(self.wait)
# In case we didn't catch a document type somehow # In case we didn't catch a document type somehow
final_response = final_response if final_response else first_response final_response = final_response if final_response else first_response
if not final_response: if not final_response:
@@ -392,6 +396,7 @@ class PlaywrightEngine:
except Exception as e: except Exception as e:
log.error(f"Error waiting for selector {self.wait_selector}: {e}") log.error(f"Error waiting for selector {self.wait_selector}: {e}")
await page.wait_for_timeout(self.wait)
# In case we didn't catch a document type somehow # In case we didn't catch a document type somehow
final_response = final_response if final_response else first_response final_response = final_response if final_response else first_response
if not final_response: if not final_response:
scrapling/fetchers.py
+8 -4
View File
@@ -384,7 +384,7 @@ class PlayWrightFetcher(BaseFetcher):
@classmethod @classmethod
def fetch( def fetch(
cls, url: str, headless: Union[bool, str] = True, disable_resources: bool = None, cls, url: str, headless: Union[bool, str] = True, disable_resources: bool = None,
useragent: Optional[str] = None, network_idle: bool = False, timeout: Optional[float] = 30000, useragent: Optional[str] = None, network_idle: bool = False, timeout: Optional[float] = 30000, wait: Optional[int] = 0,
page_action: Optional[Callable] = None, wait_selector: Optional[str] = None, wait_selector_state: SelectorWaitStates = 'attached', page_action: Optional[Callable] = None, wait_selector: Optional[str] = None, wait_selector_state: SelectorWaitStates = 'attached',
hide_canvas: bool = False, disable_webgl: bool = False, extra_headers: Optional[Dict[str, str]] = None, google_search: bool = True, hide_canvas: bool = False, disable_webgl: bool = False, extra_headers: Optional[Dict[str, str]] = None, google_search: bool = True,
proxy: Optional[Union[str, Dict[str, str]]] = None, locale: Optional[str] = 'en-US', proxy: Optional[Union[str, Dict[str, str]]] = None, locale: Optional[str] = 'en-US',
@@ -402,7 +402,8 @@ class PlayWrightFetcher(BaseFetcher):
This can help save your proxy usage but be careful with this option as it makes some websites never finish loading. This can help save your proxy usage but be careful with this option as it makes some websites never finish loading.
:param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it. :param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it.
:param network_idle: Wait for the page until there are no network connections for at least 500 ms. :param network_idle: Wait for the page until there are no network connections for at least 500 ms.
:param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30000 :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30000.
:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning `Response` object.
:param locale: Set the locale for the browser if wanted. The default value is `en-US`. :param locale: Set the locale for the browser if wanted. The default value is `en-US`.
:param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again. :param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again.
:param wait_selector: Wait for a specific css selector to be in a specific state. :param wait_selector: Wait for a specific css selector to be in a specific state.
@@ -426,6 +427,7 @@ class PlayWrightFetcher(BaseFetcher):
ValueError(f"The custom parser config must be of type dictionary, got {cls.__class__}") ValueError(f"The custom parser config must be of type dictionary, got {cls.__class__}")
engine = PlaywrightEngine( engine = PlaywrightEngine(
wait=wait,
proxy=proxy, proxy=proxy,
locale=locale, locale=locale,
timeout=timeout, timeout=timeout,
@@ -452,7 +454,7 @@ class PlayWrightFetcher(BaseFetcher):
@classmethod @classmethod
async def async_fetch( async def async_fetch(
cls, url: str, headless: Union[bool, str] = True, disable_resources: bool = None, cls, url: str, headless: Union[bool, str] = True, disable_resources: bool = None,
useragent: Optional[str] = None, network_idle: bool = False, timeout: Optional[float] = 30000, useragent: Optional[str] = None, network_idle: bool = False, timeout: Optional[float] = 30000, wait: Optional[int] = 0,
page_action: Optional[Callable] = None, wait_selector: Optional[str] = None, wait_selector_state: SelectorWaitStates = 'attached', page_action: Optional[Callable] = None, wait_selector: Optional[str] = None, wait_selector_state: SelectorWaitStates = 'attached',
hide_canvas: bool = False, disable_webgl: bool = False, extra_headers: Optional[Dict[str, str]] = None, google_search: bool = True, hide_canvas: bool = False, disable_webgl: bool = False, extra_headers: Optional[Dict[str, str]] = None, google_search: bool = True,
proxy: Optional[Union[str, Dict[str, str]]] = None, locale: Optional[str] = 'en-US', proxy: Optional[Union[str, Dict[str, str]]] = None, locale: Optional[str] = 'en-US',
@@ -470,7 +472,8 @@ class PlayWrightFetcher(BaseFetcher):
This can help save your proxy usage but be careful with this option as it makes some websites never finish loading. This can help save your proxy usage but be careful with this option as it makes some websites never finish loading.
:param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it. :param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it.
:param network_idle: Wait for the page until there are no network connections for at least 500 ms. :param network_idle: Wait for the page until there are no network connections for at least 500 ms.
:param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30000 :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30000.
:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning `Response` object.
:param locale: Set the locale for the browser if wanted. The default value is `en-US`. :param locale: Set the locale for the browser if wanted. The default value is `en-US`.
:param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again. :param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again.
:param wait_selector: Wait for a specific css selector to be in a specific state. :param wait_selector: Wait for a specific css selector to be in a specific state.
@@ -494,6 +497,7 @@ class PlayWrightFetcher(BaseFetcher):
ValueError(f"The custom parser config must be of type dictionary, got {cls.__class__}") ValueError(f"The custom parser config must be of type dictionary, got {cls.__class__}")
engine = PlaywrightEngine( engine = PlaywrightEngine(
wait=wait,
proxy=proxy, proxy=proxy,
locale=locale, locale=locale,
timeout=timeout, timeout=timeout,