From f6c122b87f8221b8d71aa2b1fe6554b1318bde43 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Mon, 29 Sep 2025 05:09:23 +0300 Subject: [PATCH] style: Removing dead code/docstrings --- scrapling/core/ai.py | 4 ++-- scrapling/engines/_browsers/_controllers.py | 4 ++-- scrapling/engines/constants.py | 15 --------------- scrapling/fetchers.py | 5 ++--- tests/fetchers/async/test_dynamic.py | 2 +- tests/fetchers/sync/test_dynamic.py | 2 +- 6 files changed, 8 insertions(+), 24 deletions(-) diff --git a/scrapling/core/ai.py b/scrapling/core/ai.py index 242eded..283f183 100644 --- a/scrapling/core/ai.py +++ b/scrapling/core/ai.py @@ -258,7 +258,7 @@ class ScraplingMCPServer: :param real_chrome: If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch an instance of your browser and use it. :param hide_canvas: Add random noise to canvas operations to prevent fingerprinting. :param disable_webgl: Disables WebGL and WebGL 2.0 support entirely. - :param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers/NSTBrowser through CDP. + :param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers through CDP. :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name. :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only. @@ -346,7 +346,7 @@ class ScraplingMCPServer: :param real_chrome: If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch an instance of your browser and use it. :param hide_canvas: Add random noise to canvas operations to prevent fingerprinting. :param disable_webgl: Disables WebGL and WebGL 2.0 support entirely. - :param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers/NSTBrowser through CDP. + :param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers through CDP. :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name. :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only. diff --git a/scrapling/engines/_browsers/_controllers.py b/scrapling/engines/_browsers/_controllers.py index b4b6276..b895f9a 100644 --- a/scrapling/engines/_browsers/_controllers.py +++ b/scrapling/engines/_browsers/_controllers.py @@ -117,7 +117,7 @@ class DynamicSession(DynamicSessionMixin, SyncSession): :param hide_canvas: Add random noise to canvas operations to prevent fingerprinting. :param disable_webgl: Disables WebGL and WebGL 2.0 support entirely. :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute. - :param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers/NSTBrowser through CDP. + :param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers through CDP. :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name. :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only. @@ -360,7 +360,7 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession): :param real_chrome: If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch an instance of your browser and use it. :param hide_canvas: Add random noise to canvas operations to prevent fingerprinting. :param disable_webgl: Disables WebGL and WebGL 2.0 support entirely. - :param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers/NSTBrowser through CDP. + :param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers through CDP. :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name. :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only. diff --git a/scrapling/engines/constants.py b/scrapling/engines/constants.py index 03a678e..df12ee3 100644 --- a/scrapling/engines/constants.py +++ b/scrapling/engines/constants.py @@ -101,18 +101,3 @@ DEFAULT_STEALTH_FLAGS = ( "--blink-settings=primaryHoverType=2,availableHoverTypes=2,primaryPointerType=4,availablePointerTypes=4", "--disable-features=AudioServiceOutOfProcess,IsolateOrigins,site-per-process,TranslateUI,BlinkGenPropertyTrees", ) - -# Defaulting to the docker mode, token doesn't matter in it as it's passed for the container -NSTBROWSER_DEFAULT_QUERY = { - "once": True, - "headless": True, - "autoClose": True, - "fingerprint": { - "flags": {"timezone": "BasedOnIp", "screen": "Custom"}, - "platform": "linux", # support: windows, mac, linux - "kernel": "chromium", # only support: chromium - "kernelMilestone": "128", - "hardwareConcurrency": 8, - "deviceMemory": 8, - }, -} diff --git a/scrapling/fetchers.py b/scrapling/fetchers.py index 81389e6..ea99ed5 100644 --- a/scrapling/fetchers.py +++ b/scrapling/fetchers.py @@ -260,7 +260,6 @@ class DynamicFetcher(BaseFetcher): 3) Using custom flags on launch to hide Playwright even more and make it faster. 4) Generates real browser's headers of the same type and same user OS, then append it to the request. - Real browsers by passing the `real_chrome` argument or the CDP URL of your browser to be controlled by the Fetcher, and most of the options can be enabled on it. - - NSTBrowser's docker browserless option by passing the CDP URL and enabling `nstbrowser_mode` option. > Note that these are the main options with PlayWright, but it can be mixed. """ @@ -314,7 +313,7 @@ class DynamicFetcher(BaseFetcher): :param real_chrome: If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch an instance of your browser and use it. :param hide_canvas: Add random noise to canvas operations to prevent fingerprinting. :param disable_webgl: Disables WebGL and WebGL 2.0 support entirely. - :param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers/NSTBrowser through CDP. + :param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers through CDP. :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name. :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only. @@ -401,7 +400,7 @@ class DynamicFetcher(BaseFetcher): :param real_chrome: If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch an instance of your browser and use it. :param hide_canvas: Add random noise to canvas operations to prevent fingerprinting. :param disable_webgl: Disables WebGL and WebGL 2.0 support entirely. - :param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers/NSTBrowser through CDP. + :param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers through CDP. :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name. :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only. diff --git a/tests/fetchers/async/test_dynamic.py b/tests/fetchers/async/test_dynamic.py index 5174c9f..3f5f7b5 100644 --- a/tests/fetchers/async/test_dynamic.py +++ b/tests/fetchers/async/test_dynamic.py @@ -86,7 +86,7 @@ class TestDynamicFetcherAsync: with pytest.raises(TypeError): await fetcher.async_fetch( - urls["html_url"], cdp_url="blahblah", nstbrowser_mode=True + urls["html_url"], cdp_url="blahblah" ) with pytest.raises(Exception): diff --git a/tests/fetchers/sync/test_dynamic.py b/tests/fetchers/sync/test_dynamic.py index a140076..a60d9d8 100644 --- a/tests/fetchers/sync/test_dynamic.py +++ b/tests/fetchers/sync/test_dynamic.py @@ -81,7 +81,7 @@ class TestDynamicFetcher: fetcher.fetch(self.html_url, cdp_url="blahblah") with pytest.raises(TypeError): - fetcher.fetch(self.html_url, cdp_url="blahblah", nstbrowser_mode=True) + fetcher.fetch(self.html_url, cdp_url="blahblah") with pytest.raises(Exception): fetcher.fetch(self.html_url, cdp_url="ws://blahblah")