freedak/Scrapling
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(DynamicFetcher): add the ability to add init_script to context (#56)

Browse Source
This commit is contained in:
Karim shoair
2025-09-02 16:25:13 +03:00
parent 9543f171aa
commit 6ad1bab2e4
3 changed files with 30 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scrapling/engines/_browsers/_controllers.py
+14
View File
@@ -60,6 +60,7 @@ class DynamicSession:
"disable_resources",
"network_idle",
"wait_selector",
"init_script",
"wait_selector_state",
"wait",
"playwright",
@@ -94,6 +95,7 @@ class DynamicSession:
timeout: int | float = 30000,
disable_resources: bool = False,
wait_selector: Optional[str] = None,
init_script: Optional[str] = None,
cookies: Optional[List[Dict]] = None,
network_idle: bool = False,
wait_selector_state: SelectorWaitStates = "attached",
@@ -112,6 +114,7 @@ class DynamicSession:
:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
:param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again.
:param wait_selector: Wait for a specific CSS selector to be in a specific state.
:param init_script: An absolute path to a JavaScript file to be executed on page creation for all pages in this session.
:param locale: Set the locale for the browser if wanted. The default value is `en-US`.
:param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.
:param stealth: Enables stealth mode, check the documentation to see what stealth mode does currently.
@@ -143,6 +146,7 @@ class DynamicSession:
"selector_config": selector_config,
"disable_resources": disable_resources,
"wait_selector": wait_selector,
"init_script": init_script,
"cookies": cookies,
"network_idle": network_idle,
"wait_selector_state": wait_selector_state,
@@ -168,6 +172,7 @@ class DynamicSession:
self.cdp_url = config.cdp_url
self.network_idle = config.network_idle
self.wait_selector = config.wait_selector
self.init_script = config.init_script
self.wait_selector_state = config.wait_selector_state
self.playwright: Optional[Playwright] = None
@@ -243,6 +248,9 @@ class DynamicSession:
user_data_dir="", **self.launch_options
)
if self.init_script: # pragma: no cover
self.context.add_init_script(path=self.init_script)
if self.cookies: # pragma: no cover
self.context.add_cookies(self.cookies)
@@ -409,6 +417,7 @@ class AsyncDynamicSession(DynamicSession):
timeout: int | float = 30000,
disable_resources: bool = False,
wait_selector: Optional[str] = None,
init_script: Optional[str] = None,
cookies: Optional[List[Dict]] = None,
network_idle: bool = False,
wait_selector_state: SelectorWaitStates = "attached",
@@ -427,6 +436,7 @@ class AsyncDynamicSession(DynamicSession):
:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
:param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again.
:param wait_selector: Wait for a specific CSS selector to be in a specific state.
:param init_script: An absolute path to a JavaScript file to be executed on page creation for all pages in this session.
:param locale: Set the locale for the browser if wanted. The default value is `en-US`.
:param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.
:param stealth: Enables stealth mode, check the documentation to see what stealth mode does currently.
@@ -459,6 +469,7 @@ class AsyncDynamicSession(DynamicSession):
timeout,
disable_resources,
wait_selector,
init_script,
cookies,
network_idle,
wait_selector_state,
@@ -494,6 +505,9 @@ class AsyncDynamicSession(DynamicSession):
)
)
if self.init_script: # pragma: no cover
await self.context.add_init_script(path=self.init_script)
if self.cookies:
await self.context.add_cookies(self.cookies)
scrapling/engines/_browsers/_validators.py
+10
View File
@@ -32,6 +32,7 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False):
extra_headers: Optional[Dict[str, str]] = None
useragent: Optional[str] = None
timeout: int | float = 30000
init_script: Optional[str] = None
disable_resources: bool = False
wait_selector: Optional[str] = None
cookies: Optional[List[Dict]] = None
@@ -58,6 +59,15 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False):
if not self.selector_config:
self.selector_config = {}
if self.init_script is not None:
script_path = Path(self.init_script)
if not script_path.exists():
raise ValueError("Init script path not found")
elif not script_path.is_file():
raise ValueError("Init script is not a file")
elif not script_path.is_absolute():
raise ValueError("Init script is not a absolute path")
@staticmethod
def __validate_cdp(cdp_url):
try:
scrapling/fetchers.py
+6
View File
@@ -282,6 +282,7 @@ class DynamicFetcher(BaseFetcher):
timeout: int | float = 30000,
disable_resources: bool = False,
wait_selector: Optional[str] = None,
init_script: Optional[str] = None,
cookies: Optional[Iterable[Dict]] = None,
network_idle: bool = False,
wait_selector_state: SelectorWaitStates = "attached",
@@ -301,6 +302,7 @@ class DynamicFetcher(BaseFetcher):
:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
:param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again.
:param wait_selector: Wait for a specific CSS selector to be in a specific state.
:param init_script: An absolute path to a JavaScript file to be executed on page creation with this request.
:param locale: Set the locale for the browser if wanted. The default value is `en-US`.
:param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.
:param stealth: Enables stealth mode, check the documentation to see what stealth mode does currently.
@@ -334,6 +336,7 @@ class DynamicFetcher(BaseFetcher):
real_chrome=real_chrome,
page_action=page_action,
hide_canvas=hide_canvas,
init_script=init_script,
network_idle=network_idle,
google_search=google_search,
extra_headers=extra_headers,
@@ -365,6 +368,7 @@ class DynamicFetcher(BaseFetcher):
timeout: int | float = 30000,
disable_resources: bool = False,
wait_selector: Optional[str] = None,
init_script: Optional[str] = None,
cookies: Optional[Iterable[Dict]] = None,
network_idle: bool = False,
wait_selector_state: SelectorWaitStates = "attached",
@@ -384,6 +388,7 @@ class DynamicFetcher(BaseFetcher):
:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
:param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again.
:param wait_selector: Wait for a specific CSS selector to be in a specific state.
:param init_script: An absolute path to a JavaScript file to be executed on page creation with this request.
:param locale: Set the locale for the browser if wanted. The default value is `en-US`.
:param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.
:param stealth: Enables stealth mode, check the documentation to see what stealth mode does currently.
@@ -418,6 +423,7 @@ class DynamicFetcher(BaseFetcher):
real_chrome=real_chrome,
page_action=page_action,
hide_canvas=hide_canvas,
init_script=init_script,
network_idle=network_idle,
google_search=google_search,
extra_headers=extra_headers,