freedak/Scrapling
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(StealthyFetcher): adding the wait option

Browse Source 
This will make fetcher sleep after fetch before closing the page
This commit is contained in:
Karim shoair
2025-04-08 03:08:48 +02:00
parent 8f5d66f678
commit a79bf3cc9d
2 changed files with 12 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scrapling/engines/camo.py
+5 -1
View File
@@ -16,7 +16,7 @@ from scrapling.engines.toolbelt import (Response, StatusText,
class CamoufoxEngine: class CamoufoxEngine:
def __init__( def __init__(
self, headless: Union[bool, Literal['virtual']] = True, block_images: bool = False, disable_resources: bool = False, self, headless: Union[bool, Literal['virtual']] = True, block_images: bool = False, disable_resources: bool = False,
block_webrtc: bool = False, allow_webgl: bool = True, network_idle: bool = False, humanize: Union[bool, float] = True, block_webrtc: bool = False, allow_webgl: bool = True, network_idle: bool = False, humanize: Union[bool, float] = True, wait: Optional[int] = 0,
timeout: Optional[float] = 30000, page_action: Callable = None, wait_selector: Optional[str] = None, addons: Optional[List[str]] = None, timeout: Optional[float] = 30000, page_action: Callable = None, wait_selector: Optional[str] = None, addons: Optional[List[str]] = None,
wait_selector_state: SelectorWaitStates = 'attached', google_search: bool = True, extra_headers: Optional[Dict[str, str]] = None, wait_selector_state: SelectorWaitStates = 'attached', google_search: bool = True, extra_headers: Optional[Dict[str, str]] = None,
proxy: Optional[Union[str, Dict[str, str]]] = None, os_randomize: bool = False, disable_ads: bool = False, proxy: Optional[Union[str, Dict[str, str]]] = None, os_randomize: bool = False, disable_ads: bool = False,
@@ -39,6 +39,7 @@ class CamoufoxEngine:
:param network_idle: Wait for the page until there are no network connections for at least 500 ms. :param network_idle: Wait for the page until there are no network connections for at least 500 ms.
:param disable_ads: Disabled by default, this installs `uBlock Origin` addon on the browser if enabled. :param disable_ads: Disabled by default, this installs `uBlock Origin` addon on the browser if enabled.
:param os_randomize: If enabled, Scrapling will randomize the OS fingerprints used. The default is Scrapling matching the fingerprints with the current OS. :param os_randomize: If enabled, Scrapling will randomize the OS fingerprints used. The default is Scrapling matching the fingerprints with the current OS.
:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning `Response` object.
:param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30000 :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30000
:param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again. :param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again.
:param wait_selector: Wait for a specific css selector to be in a specific state. :param wait_selector: Wait for a specific css selector to be in a specific state.
@@ -67,6 +68,7 @@ class CamoufoxEngine:
self.addons = addons or [] self.addons = addons or []
self.humanize = humanize self.humanize = humanize
self.timeout = check_type_validity(timeout, [int, float], 30000) self.timeout = check_type_validity(timeout, [int, float], 30000)
self.wait = check_type_validity(wait, [int, float], 0)
# Page action callable validation # Page action callable validation
self.page_action = None self.page_action = None
@@ -213,6 +215,7 @@ class CamoufoxEngine:
except Exception as e: except Exception as e:
log.error(f"Error waiting for selector {self.wait_selector}: {e}") log.error(f"Error waiting for selector {self.wait_selector}: {e}")
page.wait_for_timeout(self.wait)
# In case we didn't catch a document type somehow # In case we didn't catch a document type somehow
final_response = final_response if final_response else first_response final_response = final_response if final_response else first_response
if not final_response: if not final_response:
@@ -299,6 +302,7 @@ class CamoufoxEngine:
except Exception as e: except Exception as e:
log.error(f"Error waiting for selector {self.wait_selector}: {e}") log.error(f"Error waiting for selector {self.wait_selector}: {e}")
await page.wait_for_timeout(self.wait)
# In case we didn't catch a document type somehow # In case we didn't catch a document type somehow
final_response = final_response if final_response else first_response final_response = final_response if final_response else first_response
if not final_response: if not final_response:
scrapling/fetchers.py
+7 -3
View File
@@ -231,7 +231,7 @@ class StealthyFetcher(BaseFetcher):
@classmethod @classmethod
def fetch( def fetch(
cls, url: str, headless: Union[bool, Literal['virtual']] = True, block_images: bool = False, disable_resources: bool = False, cls, url: str, headless: Union[bool, Literal['virtual']] = True, block_images: bool = False, disable_resources: bool = False,
block_webrtc: bool = False, allow_webgl: bool = True, network_idle: bool = False, addons: Optional[List[str]] = None, block_webrtc: bool = False, allow_webgl: bool = True, network_idle: bool = False, addons: Optional[List[str]] = None, wait: Optional[int] = 0,
timeout: Optional[float] = 30000, page_action: Callable = None, wait_selector: Optional[str] = None, humanize: Optional[Union[bool, float]] = True, timeout: Optional[float] = 30000, page_action: Callable = None, wait_selector: Optional[str] = None, humanize: Optional[Union[bool, float]] = True,
wait_selector_state: SelectorWaitStates = 'attached', google_search: bool = True, extra_headers: Optional[Dict[str, str]] = None, wait_selector_state: SelectorWaitStates = 'attached', google_search: bool = True, extra_headers: Optional[Dict[str, str]] = None,
proxy: Optional[Union[str, Dict[str, str]]] = None, os_randomize: bool = False, disable_ads: bool = False, geoip: bool = False, proxy: Optional[Union[str, Dict[str, str]]] = None, os_randomize: bool = False, disable_ads: bool = False, geoip: bool = False,
@@ -256,7 +256,8 @@ class StealthyFetcher(BaseFetcher):
It will also calculate and spoof the browser's language based on the distribution of language speakers in the target region. It will also calculate and spoof the browser's language based on the distribution of language speakers in the target region.
:param network_idle: Wait for the page until there are no network connections for at least 500 ms. :param network_idle: Wait for the page until there are no network connections for at least 500 ms.
:param os_randomize: If enabled, Scrapling will randomize the OS fingerprints used. The default is Scrapling matching the fingerprints with the current OS. :param os_randomize: If enabled, Scrapling will randomize the OS fingerprints used. The default is Scrapling matching the fingerprints with the current OS.
:param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30000 :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30000.
:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning `Response` object.
:param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again. :param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again.
:param wait_selector: Wait for a specific css selector to be in a specific state. :param wait_selector: Wait for a specific css selector to be in a specific state.
:param wait_selector_state: The state to wait for the selector given with `wait_selector`. Default state is `attached`. :param wait_selector_state: The state to wait for the selector given with `wait_selector`. Default state is `attached`.
@@ -273,6 +274,7 @@ class StealthyFetcher(BaseFetcher):
ValueError(f"The custom parser config must be of type dictionary, got {cls.__class__}") ValueError(f"The custom parser config must be of type dictionary, got {cls.__class__}")
engine = CamoufoxEngine( engine = CamoufoxEngine(
wait=wait,
proxy=proxy, proxy=proxy,
geoip=geoip, geoip=geoip,
addons=addons, addons=addons,
@@ -299,7 +301,7 @@ class StealthyFetcher(BaseFetcher):
@classmethod @classmethod
async def async_fetch( async def async_fetch(
cls, url: str, headless: Union[bool, Literal['virtual']] = True, block_images: bool = False, disable_resources: bool = False, cls, url: str, headless: Union[bool, Literal['virtual']] = True, block_images: bool = False, disable_resources: bool = False,
block_webrtc: bool = False, allow_webgl: bool = True, network_idle: bool = False, addons: Optional[List[str]] = None, block_webrtc: bool = False, allow_webgl: bool = True, network_idle: bool = False, addons: Optional[List[str]] = None, wait: Optional[int] = 0,
timeout: Optional[float] = 30000, page_action: Callable = None, wait_selector: Optional[str] = None, humanize: Optional[Union[bool, float]] = True, timeout: Optional[float] = 30000, page_action: Callable = None, wait_selector: Optional[str] = None, humanize: Optional[Union[bool, float]] = True,
wait_selector_state: SelectorWaitStates = 'attached', google_search: bool = True, extra_headers: Optional[Dict[str, str]] = None, wait_selector_state: SelectorWaitStates = 'attached', google_search: bool = True, extra_headers: Optional[Dict[str, str]] = None,
proxy: Optional[Union[str, Dict[str, str]]] = None, os_randomize: bool = False, disable_ads: bool = False, geoip: bool = False, proxy: Optional[Union[str, Dict[str, str]]] = None, os_randomize: bool = False, disable_ads: bool = False, geoip: bool = False,
@@ -325,6 +327,7 @@ class StealthyFetcher(BaseFetcher):
:param network_idle: Wait for the page until there are no network connections for at least 500 ms. :param network_idle: Wait for the page until there are no network connections for at least 500 ms.
:param os_randomize: If enabled, Scrapling will randomize the OS fingerprints used. The default is Scrapling matching the fingerprints with the current OS. :param os_randomize: If enabled, Scrapling will randomize the OS fingerprints used. The default is Scrapling matching the fingerprints with the current OS.
:param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30000 :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30000
:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning `Response` object.
:param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again. :param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again.
:param wait_selector: Wait for a specific css selector to be in a specific state. :param wait_selector: Wait for a specific css selector to be in a specific state.
:param wait_selector_state: The state to wait for the selector given with `wait_selector`. Default state is `attached`. :param wait_selector_state: The state to wait for the selector given with `wait_selector`. Default state is `attached`.
@@ -341,6 +344,7 @@ class StealthyFetcher(BaseFetcher):
ValueError(f"The custom parser config must be of type dictionary, got {cls.__class__}") ValueError(f"The custom parser config must be of type dictionary, got {cls.__class__}")
engine = CamoufoxEngine( engine = CamoufoxEngine(
wait=wait,
proxy=proxy, proxy=proxy,
geoip=geoip, geoip=geoip,
addons=addons, addons=addons,