diff --git a/docs/fetching/dynamic.md b/docs/fetching/dynamic.md index 5d572ae..e6566c5 100644 --- a/docs/fetching/dynamic.md +++ b/docs/fetching/dynamic.md @@ -79,6 +79,7 @@ Scrapling provides many options with this fetcher. To make it as simple as possi | wait | The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the `Response` object. | ✔️ | | page_action | Added for automation. Pass a function that takes the `page` object and does the necessary automation, then returns `page` again. | ✔️ | | wait_selector | Wait for a specific css selector to be in a specific state. | ✔️ | +| init_script | An absolute path to a JavaScript file to be executed on page creation for all pages in this session. | ✔️ | | wait_selector_state | Scrapling will wait for the given state to be fulfilled for the selector given with `wait_selector`. _Default state is `attached`._ | ✔️ | | google_search | Enabled by default, Scrapling will set the referer header as if this request came from a Google search of this website's domain name. | ✔️ | | extra_headers | A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ | ✔️ | diff --git a/docs/fetching/stealthy.md b/docs/fetching/stealthy.md index 489fd4f..667f43e 100644 --- a/docs/fetching/stealthy.md +++ b/docs/fetching/stealthy.md @@ -43,6 +43,7 @@ Before jumping to [examples](#examples), here's the full list of arguments | timeout | The timeout used in all operations and waits through the page. It's in milliseconds, and the default is 30000. | ✔️ | | wait | The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the `Response` object. | ✔️ | | wait_selector | Wait for a specific css selector to be in a specific state. | ✔️ | +| init_script | An absolute path to a JavaScript file to be executed on page creation for all pages in this session. | ✔️ | | wait_selector_state | Scrapling will wait for the given state to be fulfilled for the selector given with `wait_selector`. _Default state is `attached`._ | ✔️ | | proxy | The proxy to be used with requests. It can be a string or a dictionary with the keys 'server', 'username', and 'password' only. | ✔️ | | additional_args | Additional arguments to be passed to Camoufox as additional settings, and they take higher priority than Scrapling's settings. | ✔️ | diff --git a/scrapling/__init__.py b/scrapling/__init__.py index 56e62db..f08e652 100644 --- a/scrapling/__init__.py +++ b/scrapling/__init__.py @@ -1,5 +1,5 @@ __author__ = "Karim Shoair (karim.shoair@pm.me)" -__version__ = "0.3" +__version__ = "0.3.1" __copyright__ = "Copyright (c) 2024 Karim Shoair" diff --git a/scrapling/core/shell.py b/scrapling/core/shell.py index 0125ee0..24e504c 100644 --- a/scrapling/core/shell.py +++ b/scrapling/core/shell.py @@ -20,7 +20,6 @@ from logging import ( getLevelName, ) -from IPython.terminal.embed import InteractiveShellEmbed from orjson import loads as json_loads, JSONDecodeError from scrapling import __version__ @@ -394,8 +393,7 @@ class CurlParser: else: # pragma: no cover log.error("Input must be a valid curl command string or a Request object.") - - return None + return None def show_page_in_browser(page: Selector): # pragma: no cover @@ -544,6 +542,8 @@ Type 'exit' or press Ctrl+D to exit. def start(self): # pragma: no cover """Start the interactive shell""" + from IPython.terminal.embed import InteractiveShellEmbed + # Get our namespace with application objects namespace = self.get_namespace() ipython_shell = InteractiveShellEmbed( diff --git a/scrapling/engines/_browsers/_camoufox.py b/scrapling/engines/_browsers/_camoufox.py index 55f185d..d6150e4 100644 --- a/scrapling/engines/_browsers/_camoufox.py +++ b/scrapling/engines/_browsers/_camoufox.py @@ -60,6 +60,7 @@ class StealthySession: "timeout", "page_action", "wait_selector", + "init_script", "addons", "wait_selector_state", "cookies", @@ -95,6 +96,7 @@ class StealthySession: timeout: int | float = 30000, page_action: Optional[Callable] = None, wait_selector: Optional[str] = None, + init_script: Optional[str] = None, addons: Optional[List[str]] = None, wait_selector_state: SelectorWaitStates = "attached", cookies: Optional[List[Dict]] = None, @@ -128,6 +130,7 @@ class StealthySession: :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000 :param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again. :param wait_selector: Wait for a specific CSS selector to be in a specific state. + :param init_script: An absolute path to a JavaScript file to be executed on page creation for all pages in this session. :param geoip: Recommended to use with proxies; Automatically use IP's longitude, latitude, timezone, country, locale, and spoof the WebRTC IP address. It will also calculate and spoof the browser's language based on the distribution of language speakers in the target region. :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`. @@ -153,6 +156,7 @@ class StealthySession: "timeout": timeout, "page_action": page_action, "wait_selector": wait_selector, + "init_script": init_script, "addons": addons, "wait_selector_state": wait_selector_state, "cookies": cookies, @@ -180,6 +184,7 @@ class StealthySession: self.timeout = config.timeout self.page_action = config.page_action self.wait_selector = config.wait_selector + self.init_script = config.init_script self.addons = config.addons self.wait_selector_state = config.wait_selector_state self.cookies = config.cookies @@ -234,6 +239,9 @@ class StealthySession: **self.launch_options ) ) + if self.init_script: # pragma: no cover + self.context.add_init_script(path=self.init_script) + if self.cookies: # pragma: no cover self.context.add_cookies(self.cookies) @@ -474,6 +482,7 @@ class AsyncStealthySession(StealthySession): timeout: int | float = 30000, page_action: Optional[Callable] = None, wait_selector: Optional[str] = None, + init_script: Optional[str] = None, addons: Optional[List[str]] = None, wait_selector_state: SelectorWaitStates = "attached", cookies: Optional[List[Dict]] = None, @@ -507,6 +516,7 @@ class AsyncStealthySession(StealthySession): :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000 :param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again. :param wait_selector: Wait for a specific CSS selector to be in a specific state. + :param init_script: An absolute path to a JavaScript file to be executed on page creation for all pages in this session. :param geoip: Recommended to use with proxies; Automatically use IP's longitude, latitude, timezone, country, locale, and spoof the WebRTC IP address. It will also calculate and spoof the browser's language based on the distribution of language speakers in the target region. :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`. @@ -531,6 +541,7 @@ class AsyncStealthySession(StealthySession): timeout, page_action, wait_selector, + init_script, addons, wait_selector_state, cookies, @@ -557,6 +568,9 @@ class AsyncStealthySession(StealthySession): **self.launch_options ) ) + if self.init_script: # pragma: no cover + await self.context.add_init_script(path=self.init_script) + if self.cookies: await self.context.add_cookies(self.cookies) diff --git a/scrapling/engines/_browsers/_controllers.py b/scrapling/engines/_browsers/_controllers.py index 285553f..2747ed9 100644 --- a/scrapling/engines/_browsers/_controllers.py +++ b/scrapling/engines/_browsers/_controllers.py @@ -60,6 +60,7 @@ class DynamicSession: "disable_resources", "network_idle", "wait_selector", + "init_script", "wait_selector_state", "wait", "playwright", @@ -94,6 +95,7 @@ class DynamicSession: timeout: int | float = 30000, disable_resources: bool = False, wait_selector: Optional[str] = None, + init_script: Optional[str] = None, cookies: Optional[List[Dict]] = None, network_idle: bool = False, wait_selector_state: SelectorWaitStates = "attached", @@ -112,6 +114,7 @@ class DynamicSession: :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object. :param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again. :param wait_selector: Wait for a specific CSS selector to be in a specific state. + :param init_script: An absolute path to a JavaScript file to be executed on page creation for all pages in this session. :param locale: Set the locale for the browser if wanted. The default value is `en-US`. :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`. :param stealth: Enables stealth mode, check the documentation to see what stealth mode does currently. @@ -143,6 +146,7 @@ class DynamicSession: "selector_config": selector_config, "disable_resources": disable_resources, "wait_selector": wait_selector, + "init_script": init_script, "cookies": cookies, "network_idle": network_idle, "wait_selector_state": wait_selector_state, @@ -168,6 +172,7 @@ class DynamicSession: self.cdp_url = config.cdp_url self.network_idle = config.network_idle self.wait_selector = config.wait_selector + self.init_script = config.init_script self.wait_selector_state = config.wait_selector_state self.playwright: Optional[Playwright] = None @@ -243,6 +248,9 @@ class DynamicSession: user_data_dir="", **self.launch_options ) + if self.init_script: # pragma: no cover + self.context.add_init_script(path=self.init_script) + if self.cookies: # pragma: no cover self.context.add_cookies(self.cookies) @@ -409,6 +417,7 @@ class AsyncDynamicSession(DynamicSession): timeout: int | float = 30000, disable_resources: bool = False, wait_selector: Optional[str] = None, + init_script: Optional[str] = None, cookies: Optional[List[Dict]] = None, network_idle: bool = False, wait_selector_state: SelectorWaitStates = "attached", @@ -427,6 +436,7 @@ class AsyncDynamicSession(DynamicSession): :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object. :param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again. :param wait_selector: Wait for a specific CSS selector to be in a specific state. + :param init_script: An absolute path to a JavaScript file to be executed on page creation for all pages in this session. :param locale: Set the locale for the browser if wanted. The default value is `en-US`. :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`. :param stealth: Enables stealth mode, check the documentation to see what stealth mode does currently. @@ -459,6 +469,7 @@ class AsyncDynamicSession(DynamicSession): timeout, disable_resources, wait_selector, + init_script, cookies, network_idle, wait_selector_state, @@ -494,6 +505,9 @@ class AsyncDynamicSession(DynamicSession): ) ) + if self.init_script: # pragma: no cover + await self.context.add_init_script(path=self.init_script) + if self.cookies: await self.context.add_cookies(self.cookies) diff --git a/scrapling/engines/_browsers/_validators.py b/scrapling/engines/_browsers/_validators.py index 24bd314..6363df7 100644 --- a/scrapling/engines/_browsers/_validators.py +++ b/scrapling/engines/_browsers/_validators.py @@ -32,6 +32,7 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False): extra_headers: Optional[Dict[str, str]] = None useragent: Optional[str] = None timeout: int | float = 30000 + init_script: Optional[str] = None disable_resources: bool = False wait_selector: Optional[str] = None cookies: Optional[List[Dict]] = None @@ -58,6 +59,15 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False): if not self.selector_config: self.selector_config = {} + if self.init_script is not None: + script_path = Path(self.init_script) + if not script_path.exists(): + raise ValueError("Init script path not found") + elif not script_path.is_file(): + raise ValueError("Init script is not a file") + elif not script_path.is_absolute(): + raise ValueError("Init script is not a absolute path") + @staticmethod def __validate_cdp(cdp_url): try: @@ -90,6 +100,7 @@ class CamoufoxConfig(Struct, kw_only=True, frozen=False): solve_cloudflare: bool = False wait: int | float = 0 timeout: int | float = 30000 + init_script: Optional[str] = None page_action: Optional[Callable] = None wait_selector: Optional[str] = None addons: Optional[List[str]] = None @@ -131,6 +142,15 @@ class CamoufoxConfig(Struct, kw_only=True, frozen=False): f"Addon's path is not a folder, you need to pass a folder of the extracted addon: {addon}" ) + if self.init_script is not None: + script_path = Path(self.init_script) + if not script_path.exists(): + raise ValueError("Init script path not found") + elif not script_path.is_file(): + raise ValueError("Init script is not a file") + elif not script_path.is_absolute(): + raise ValueError("Init script is not a absolute path") + if not self.cookies: self.cookies = [] if self.solve_cloudflare and self.timeout < 60_000: diff --git a/scrapling/fetchers.py b/scrapling/fetchers.py index 055ea33..c1ce8f7 100644 --- a/scrapling/fetchers.py +++ b/scrapling/fetchers.py @@ -62,6 +62,7 @@ class StealthyFetcher(BaseFetcher): timeout: int | float = 30000, page_action: Optional[Callable] = None, wait_selector: Optional[str] = None, + init_script: Optional[str] = None, addons: Optional[List[str]] = None, wait_selector_state: SelectorWaitStates = "attached", cookies: Optional[List[Dict]] = None, @@ -97,6 +98,7 @@ class StealthyFetcher(BaseFetcher): :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000 :param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again. :param wait_selector: Wait for a specific CSS selector to be in a specific state. + :param init_script: An absolute path to a JavaScript file to be executed on page creation with this request. :param geoip: Recommended to use with proxies; Automatically use IP's longitude, latitude, timezone, country, locale, and spoof the WebRTC IP address. It will also calculate and spoof the browser's language based on the distribution of language speakers in the target region. :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`. @@ -127,6 +129,7 @@ class StealthyFetcher(BaseFetcher): disable_ads=disable_ads, allow_webgl=allow_webgl, page_action=page_action, + init_script=init_script, network_idle=network_idle, block_images=block_images, block_webrtc=block_webrtc, @@ -158,6 +161,7 @@ class StealthyFetcher(BaseFetcher): timeout: int | float = 30000, page_action: Optional[Callable] = None, wait_selector: Optional[str] = None, + init_script: Optional[str] = None, addons: Optional[List[str]] = None, wait_selector_state: SelectorWaitStates = "attached", cookies: Optional[List[Dict]] = None, @@ -193,6 +197,7 @@ class StealthyFetcher(BaseFetcher): :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000 :param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again. :param wait_selector: Wait for a specific CSS selector to be in a specific state. + :param init_script: An absolute path to a JavaScript file to be executed on page creation with this request. :param geoip: Recommended to use with proxies; Automatically use IP's longitude, latitude, timezone, country, locale, and spoof the WebRTC IP address. It will also calculate and spoof the browser's language based on the distribution of language speakers in the target region. :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`. @@ -223,6 +228,7 @@ class StealthyFetcher(BaseFetcher): disable_ads=disable_ads, allow_webgl=allow_webgl, page_action=page_action, + init_script=init_script, network_idle=network_idle, block_images=block_images, block_webrtc=block_webrtc, @@ -276,6 +282,7 @@ class DynamicFetcher(BaseFetcher): timeout: int | float = 30000, disable_resources: bool = False, wait_selector: Optional[str] = None, + init_script: Optional[str] = None, cookies: Optional[Iterable[Dict]] = None, network_idle: bool = False, wait_selector_state: SelectorWaitStates = "attached", @@ -295,6 +302,7 @@ class DynamicFetcher(BaseFetcher): :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object. :param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again. :param wait_selector: Wait for a specific CSS selector to be in a specific state. + :param init_script: An absolute path to a JavaScript file to be executed on page creation with this request. :param locale: Set the locale for the browser if wanted. The default value is `en-US`. :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`. :param stealth: Enables stealth mode, check the documentation to see what stealth mode does currently. @@ -328,6 +336,7 @@ class DynamicFetcher(BaseFetcher): real_chrome=real_chrome, page_action=page_action, hide_canvas=hide_canvas, + init_script=init_script, network_idle=network_idle, google_search=google_search, extra_headers=extra_headers, @@ -359,6 +368,7 @@ class DynamicFetcher(BaseFetcher): timeout: int | float = 30000, disable_resources: bool = False, wait_selector: Optional[str] = None, + init_script: Optional[str] = None, cookies: Optional[Iterable[Dict]] = None, network_idle: bool = False, wait_selector_state: SelectorWaitStates = "attached", @@ -378,6 +388,7 @@ class DynamicFetcher(BaseFetcher): :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object. :param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again. :param wait_selector: Wait for a specific CSS selector to be in a specific state. + :param init_script: An absolute path to a JavaScript file to be executed on page creation with this request. :param locale: Set the locale for the browser if wanted. The default value is `en-US`. :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`. :param stealth: Enables stealth mode, check the documentation to see what stealth mode does currently. @@ -412,6 +423,7 @@ class DynamicFetcher(BaseFetcher): real_chrome=real_chrome, page_action=page_action, hide_canvas=hide_canvas, + init_script=init_script, network_idle=network_idle, google_search=google_search, extra_headers=extra_headers, diff --git a/setup.cfg b/setup.cfg index d20f7e7..ae28f61 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = scrapling -version = 0.3 +version = 0.3.1 author = Karim Shoair author_email = karim.shoair@pm.me description = Scrapling is an undetectable, powerful, flexible, high-performance Python library that makes Web Scraping easy and effortless as it should be! diff --git a/tests/cli/test_shell_functionality.py b/tests/cli/test_shell_functionality.py index 817ef57..d23fa22 100644 --- a/tests/cli/test_shell_functionality.py +++ b/tests/cli/test_shell_functionality.py @@ -178,23 +178,21 @@ class TestCustomShell: def test_shell_initialization(self): """Test shell initialization""" - with patch('scrapling.core.shell.InteractiveShellEmbed'): - shell = CustomShell(code="", log_level="debug") + shell = CustomShell(code="", log_level="debug") - assert shell.log_level == 10 # DEBUG level - assert shell.page is None - assert len(shell.pages) == 0 + assert shell.log_level == 10 # DEBUG level + assert shell.page is None + assert len(shell.pages) == 0 def test_shell_namespace(self): """Test shell namespace creation""" - with patch('scrapling.core.shell.InteractiveShellEmbed'): - shell = CustomShell(code="") - namespace = shell.get_namespace() + shell = CustomShell(code="") + namespace = shell.get_namespace() - # Check all expected functions/classes are available - assert 'get' in namespace - assert 'post' in namespace - assert 'Fetcher' in namespace - assert 'DynamicFetcher' in namespace - assert 'view' in namespace - assert 'uncurl' in namespace + # Check all expected functions/classes are available + assert 'get' in namespace + assert 'post' in namespace + assert 'Fetcher' in namespace + assert 'DynamicFetcher' in namespace + assert 'view' in namespace + assert 'uncurl' in namespace