From e34651d8873ed287b216fdd061268a5bcc5445fd Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Fri, 27 Feb 2026 03:50:14 +0200 Subject: [PATCH] fix(stealth): improve stealth mode by removing unnecessary scripts It doesn't add anything to the table as before (it might turn out I was wrong and add them again lol) --- MANIFEST.in | 1 - scrapling/engines/_browsers/_config_tools.py | 34 --- scrapling/engines/_browsers/_stealth.py | 17 -- .../toolbelt/bypasses/navigator_plugins.js | 40 ---- .../bypasses/notification_permission.js | 5 - .../bypasses/playwright_fingerprint.js | 3 - .../engines/toolbelt/bypasses/screen_props.js | 27 --- .../toolbelt/bypasses/webdriver_fully.js | 27 --- .../toolbelt/bypasses/window_chrome.js | 213 ------------------ scrapling/engines/toolbelt/navigation.py | 14 -- tests/fetchers/test_utils.py | 21 -- 11 files changed, 402 deletions(-) delete mode 100644 scrapling/engines/toolbelt/bypasses/navigator_plugins.js delete mode 100644 scrapling/engines/toolbelt/bypasses/notification_permission.js delete mode 100644 scrapling/engines/toolbelt/bypasses/playwright_fingerprint.js delete mode 100644 scrapling/engines/toolbelt/bypasses/screen_props.js delete mode 100644 scrapling/engines/toolbelt/bypasses/webdriver_fully.js delete mode 100644 scrapling/engines/toolbelt/bypasses/window_chrome.js diff --git a/MANIFEST.in b/MANIFEST.in index 8c168dd..aa9bf23 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,7 +1,6 @@ include LICENSE include *.db include *.js -include scrapling/engines/toolbelt/bypasses/*.js include scrapling/*.db include scrapling/*.db* include scrapling/*.db-* diff --git a/scrapling/engines/_browsers/_config_tools.py b/scrapling/engines/_browsers/_config_tools.py index 2d7691b..5b9ee2e 100644 --- a/scrapling/engines/_browsers/_config_tools.py +++ b/scrapling/engines/_browsers/_config_tools.py @@ -1,38 +1,4 @@ -from functools import lru_cache - -from scrapling.engines.toolbelt.navigation import js_bypass_path from scrapling.engines.toolbelt.fingerprints import generate_headers __default_useragent__ = generate_headers(browser_mode=True).get("User-Agent") __default_chrome_useragent__ = generate_headers(browser_mode="chrome").get("User-Agent") - - -@lru_cache(1) -def _compiled_stealth_scripts(): - """Pre-read and compile stealth scripts""" - # Basic bypasses nothing fancy as I'm still working on it - # But with adding these bypasses to the above config, it bypasses many online tests like - # https://bot.sannysoft.com/ - # https://kaliiiiiiiiii.github.io/brotector/ - # https://pixelscan.net/ - # https://iphey.com/ - # https://www.browserscan.net/bot-detection <== this one also checks for the CDP runtime fingerprint - # https://arh.antoinevastel.com/bots/areyouheadless/ - # https://prescience-data.github.io/execution-monitor.html - stealth_scripts_paths = tuple( - js_bypass_path(script) - for script in ( - # Order is important - "webdriver_fully.js", - "window_chrome.js", - "navigator_plugins.js", - "notification_permission.js", - "screen_props.js", - "playwright_fingerprint.js", - ) - ) - scripts = [] - for script_path in stealth_scripts_paths: - with open(script_path, "r") as f: - scripts.append(f.read()) - return tuple(scripts) diff --git a/scrapling/engines/_browsers/_stealth.py b/scrapling/engines/_browsers/_stealth.py index b375198..797a0c1 100644 --- a/scrapling/engines/_browsers/_stealth.py +++ b/scrapling/engines/_browsers/_stealth.py @@ -17,7 +17,6 @@ from scrapling.core._types import Any, Optional, ProxyType, Unpack from scrapling.engines.toolbelt.proxy_rotation import is_proxy_error from scrapling.engines.toolbelt.convertor import Response, ResponseFactory from scrapling.engines.toolbelt.fingerprints import generate_convincing_referer -from scrapling.engines._browsers._config_tools import _compiled_stealth_scripts from scrapling.engines._browsers._types import StealthSession, StealthFetchParams from scrapling.engines._browsers._base import SyncSession, AsyncSession, StealthySessionMixin from scrapling.engines._browsers._validators import validate_fetch as _validate, StealthConfig @@ -109,14 +108,6 @@ class StealthySession(SyncSession, StealthySessionMixin): else: raise RuntimeError("Session has been already started") - def _initialize_context(self, config, ctx: BrowserContext) -> BrowserContext: - """Initialize the browser context.""" - for script in _compiled_stealth_scripts(): - ctx.add_init_script(script=script) - - ctx = super()._initialize_context(config, ctx) - return ctx - def _cloudflare_solver(self, page: Page) -> None: # pragma: no cover """Solve the cloudflare challenge displayed on the playwright page passed @@ -372,14 +363,6 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin): else: raise RuntimeError("Session has been already started") - async def _initialize_context(self, config: Any, ctx: AsyncBrowserContext) -> AsyncBrowserContext: - """Initialize the browser context.""" - for script in _compiled_stealth_scripts(): - await ctx.add_init_script(script=script) - - ctx = await super()._initialize_context(config, ctx) - return ctx - async def _cloudflare_solver(self, page: async_Page) -> None: # pragma: no cover """Solve the cloudflare challenge displayed on the playwright page passed diff --git a/scrapling/engines/toolbelt/bypasses/navigator_plugins.js b/scrapling/engines/toolbelt/bypasses/navigator_plugins.js deleted file mode 100644 index 653fa5f..0000000 --- a/scrapling/engines/toolbelt/bypasses/navigator_plugins.js +++ /dev/null @@ -1,40 +0,0 @@ -if(navigator.plugins.length == 0){ - Object.defineProperty(navigator, 'plugins', { - get: () => { - const PDFViewerPlugin = Object.create(Plugin.prototype, { - description: { value: 'Portable Document Format', enumerable: false }, - filename: { value: 'internal-pdf-viewer', enumerable: false }, - name: { value: 'PDF Viewer', enumerable: false }, - }); - const ChromePDFViewer = Object.create(Plugin.prototype, { - description: { value: 'Portable Document Format', enumerable: false }, - filename: { value: 'internal-pdf-viewer', enumerable: false }, - name: { value: 'Chrome PDF Viewer', enumerable: false }, - }); - const ChromiumPDFViewer = Object.create(Plugin.prototype, { - description: { value: 'Portable Document Format', enumerable: false }, - filename: { value: 'internal-pdf-viewer', enumerable: false }, - name: { value: 'Chromium PDF Viewer', enumerable: false }, - }); - const EdgePDFViewer = Object.create(Plugin.prototype, { - description: { value: 'Portable Document Format', enumerable: false }, - filename: { value: 'internal-pdf-viewer', enumerable: false }, - name: { value: 'Microsoft Edge PDF Viewer', enumerable: false }, - }); - const WebKitPDFPlugin = Object.create(Plugin.prototype, { - description: { value: 'Portable Document Format', enumerable: false }, - filename: { value: 'internal-pdf-viewer', enumerable: false }, - name: { value: 'WebKit built-in PDF', enumerable: false }, - }); - - return Object.create(PluginArray.prototype, { - length: { value: 5 }, - 0: { value: PDFViewerPlugin }, - 1: { value: ChromePDFViewer }, - 2: { value: ChromiumPDFViewer }, - 3: { value: EdgePDFViewer }, - 4: { value: WebKitPDFPlugin }, - }); - }, - }); -} \ No newline at end of file diff --git a/scrapling/engines/toolbelt/bypasses/notification_permission.js b/scrapling/engines/toolbelt/bypasses/notification_permission.js deleted file mode 100644 index 0c9c676..0000000 --- a/scrapling/engines/toolbelt/bypasses/notification_permission.js +++ /dev/null @@ -1,5 +0,0 @@ -// Bypasses `notificationIsDenied` test in creepsjs's 'Like Headless' sections -const isSecure = document.location.protocol.startsWith('https') -if (isSecure){ - Object.defineProperty(Notification, 'permission', {get: () => 'default'}) -} \ No newline at end of file diff --git a/scrapling/engines/toolbelt/bypasses/playwright_fingerprint.js b/scrapling/engines/toolbelt/bypasses/playwright_fingerprint.js deleted file mode 100644 index e1b959d..0000000 --- a/scrapling/engines/toolbelt/bypasses/playwright_fingerprint.js +++ /dev/null @@ -1,3 +0,0 @@ -// Remove playwright fingerprint => https://github.com/microsoft/playwright/commit/c9e673c6dca746384338ab6bb0cf63c7e7caa9b2#diff-087773eea292da9db5a3f27de8f1a2940cdb895383ad750c3cd8e01772a35b40R915 -delete window.__pwInitScripts; -delete window.__playwright__binding__; \ No newline at end of file diff --git a/scrapling/engines/toolbelt/bypasses/screen_props.js b/scrapling/engines/toolbelt/bypasses/screen_props.js deleted file mode 100644 index 5056b4b..0000000 --- a/scrapling/engines/toolbelt/bypasses/screen_props.js +++ /dev/null @@ -1,27 +0,0 @@ -const windowScreenProps = { - // Dimensions - innerHeight: 0, - innerWidth: 0, - outerHeight: 754, - outerWidth: 1313, - - // Position - screenX: 19, - pageXOffset: 0, - pageYOffset: 0, - - // Display - devicePixelRatio: 2 -}; - -try { - for (const [prop, value] of Object.entries(windowScreenProps)) { - if (value > 0) { - // The 0 values are introduced by collecting in the hidden iframe. - // They are document sizes anyway so no need to test them or inject them. - window[prop] = value; - } - } -} catch (e) { - console.warn(e); -}; \ No newline at end of file diff --git a/scrapling/engines/toolbelt/bypasses/webdriver_fully.js b/scrapling/engines/toolbelt/bypasses/webdriver_fully.js deleted file mode 100644 index 4bda260..0000000 --- a/scrapling/engines/toolbelt/bypasses/webdriver_fully.js +++ /dev/null @@ -1,27 +0,0 @@ -// Create a function that looks like a native getter -const nativeGetter = function get webdriver() { - return false; -}; - -// Copy over native function properties -Object.defineProperties(nativeGetter, { - name: { value: 'get webdriver', configurable: true }, - length: { value: 0, configurable: true }, - toString: { - value: function() { - return `function get webdriver() { [native code] }`; - }, - configurable: true - } -}); - -// Make it look native -Object.setPrototypeOf(nativeGetter, Function.prototype); - -// Apply the modified descriptor -Object.defineProperty(Navigator.prototype, 'webdriver', { - get: nativeGetter, - set: undefined, - enumerable: true, - configurable: true -}); \ No newline at end of file diff --git a/scrapling/engines/toolbelt/bypasses/window_chrome.js b/scrapling/engines/toolbelt/bypasses/window_chrome.js deleted file mode 100644 index ba63a9c..0000000 --- a/scrapling/engines/toolbelt/bypasses/window_chrome.js +++ /dev/null @@ -1,213 +0,0 @@ -// To escape `HEADCHR_CHROME_OBJ` test in headless mode => https://github.com/antoinevastel/fp-collect/blob/master/src/fpCollect.js#L322 -// Faking window.chrome fully - -if (!window.chrome) { - // First, save all existing properties - const originalKeys = Object.getOwnPropertyNames(window); - const tempObj = {}; - - // Recreate all properties in original order - for (const key of originalKeys) { - const descriptor = Object.getOwnPropertyDescriptor(window, key); - const value = window[key]; - // delete window[key]; - Object.defineProperty(tempObj, key, descriptor); - } - - // Use the exact property descriptor found in headful Chrome - // fetch it via `Object.getOwnPropertyDescriptor(window, 'chrome')` - const mockChrome = { - loadTimes: {}, - csi: {}, - app: { - isInstalled: false - }, - // Add other Chrome-specific properties - }; - - Object.defineProperty(tempObj, 'chrome', { - writable: true, - enumerable: true, - configurable: false, - value: mockChrome - }); - for (const key of Object.getOwnPropertyNames(tempObj)) { - try { - Object.defineProperty(window, key, - Object.getOwnPropertyDescriptor(tempObj, key)); - } catch (e) {} - }; - // todo: solve this - // Using line below bypasses the hasHighChromeIndex test in creepjs ==> https://github.com/abrahamjuliot/creepjs/blob/master/src/headless/index.ts#L121 - // Chrome object have to be in the end of the window properties - // Object.assign(window, tempObj); - // But makes window.chrome unreadable on 'https://bot.sannysoft.com/' -} - -// That means we're running headful and don't need to mock anything -if ('app' in window.chrome) { - return; // Nothing to do here -} -const makeError = { - ErrorInInvocation: fn => { - const err = new TypeError(`Error in invocation of app.${fn}()`); - return utils.stripErrorWithAnchor( - err, - `at ${fn} (eval at `, - ); - }, -}; -// check with: `JSON.stringify(window.chrome['app'])` -const STATIC_DATA = JSON.parse( - ` -{ - "isInstalled": false, - "InstallState": { - "DISABLED": "disabled", - "INSTALLED": "installed", - "NOT_INSTALLED": "not_installed" - }, - "RunningState": { - "CANNOT_RUN": "cannot_run", - "READY_TO_RUN": "ready_to_run", - "RUNNING": "running" - } -} - `.trim(), - ); -window.chrome.app = { - ...STATIC_DATA, - - get isInstalled() { - return false; - }, - - getDetails: function getDetails() { - if (arguments.length) { - throw makeError.ErrorInInvocation(`getDetails`); - } - return null; - }, - getIsInstalled: function getDetails() { - if (arguments.length) { - throw makeError.ErrorInInvocation(`getIsInstalled`); - } - return false; - }, - runningState: function getDetails() { - if (arguments.length) { - throw makeError.ErrorInInvocation(`runningState`); - } - return 'cannot_run'; - }, -}; -// Check that the Navigation Timing API v1 is available, we need that -if (!window.performance || !window.performance.timing) { - return; -} -const {timing} = window.performance; -window.chrome.csi = function () { - return { - onloadT: timing.domContentLoadedEventEnd, - startE: timing.navigationStart, - pageT: Date.now() - timing.navigationStart, - tran: 15, // Transition type or something - }; -}; -if (!window.PerformancePaintTiming){ - return; -} -const {performance} = window; -// Some stuff is not available on about:blank as it requires a navigation to occur, -// let's harden the code to not fail then: -const ntEntryFallback = { - nextHopProtocol: 'h2', - type: 'other', -}; - -// The API exposes some funky info regarding the connection -const protocolInfo = { - get connectionInfo() { - const ntEntry = - performance.getEntriesByType('navigation')[0] || ntEntryFallback; - return ntEntry.nextHopProtocol; - }, - get npnNegotiatedProtocol() { - // NPN is deprecated in favor of ALPN, but this implementation returns the - // HTTP/2 or HTTP2+QUIC/39 requests negotiated via ALPN. - const ntEntry = - performance.getEntriesByType('navigation')[0] || ntEntryFallback; - return ['h2', 'hq'].includes(ntEntry.nextHopProtocol) - ? ntEntry.nextHopProtocol - : 'unknown'; - }, - get navigationType() { - const ntEntry = - performance.getEntriesByType('navigation')[0] || ntEntryFallback; - return ntEntry.type; - }, - get wasAlternateProtocolAvailable() { - // The Alternate-Protocol header is deprecated in favor of Alt-Svc - // (https://www.mnot.net/blog/2016/03/09/alt-svc), so technically this - // should always return false. - return false; - }, - get wasFetchedViaSpdy() { - // SPDY is deprecated in favor of HTTP/2, but this implementation returns - // true for HTTP/2 or HTTP2+QUIC/39 as well. - const ntEntry = - performance.getEntriesByType('navigation')[0] || ntEntryFallback; - return ['h2', 'hq'].includes(ntEntry.nextHopProtocol); - }, - get wasNpnNegotiated() { - // NPN is deprecated in favor of ALPN, but this implementation returns true - // for HTTP/2 or HTTP2+QUIC/39 requests negotiated via ALPN. - const ntEntry = - performance.getEntriesByType('navigation')[0] || ntEntryFallback; - return ['h2', 'hq'].includes(ntEntry.nextHopProtocol); - }, -}; - -// Truncate number to specific number of decimals, most of the `loadTimes` stuff has 3 -function toFixed(num, fixed) { - var re = new RegExp('^-?\\d+(?:.\\d{0,' + (fixed || -1) + '})?'); - return num.toString().match(re)[0]; -} - -const timingInfo = { - get firstPaintAfterLoadTime() { - // This was never actually implemented and always returns 0. - return 0; - }, - get requestTime() { - return timing.navigationStart / 1000; - }, - get startLoadTime() { - return timing.navigationStart / 1000; - }, - get commitLoadTime() { - return timing.responseStart / 1000; - }, - get finishDocumentLoadTime() { - return timing.domContentLoadedEventEnd / 1000; - }, - get finishLoadTime() { - return timing.loadEventEnd / 1000; - }, - get firstPaintTime() { - const fpEntry = performance.getEntriesByType('paint')[0] || { - startTime: timing.loadEventEnd / 1000, // Fallback if no navigation occured (`about:blank`) - }; - return toFixed( - (fpEntry.startTime + performance.timeOrigin) / 1000, - 3, - ); - }, -}; - -window.chrome.loadTimes = function () { - return { - ...protocolInfo, - ...timingInfo, - }; -}; \ No newline at end of file diff --git a/scrapling/engines/toolbelt/navigation.py b/scrapling/engines/toolbelt/navigation.py index 5c19427..9b233d8 100644 --- a/scrapling/engines/toolbelt/navigation.py +++ b/scrapling/engines/toolbelt/navigation.py @@ -2,8 +2,6 @@ Functions related to files and URLs """ -from pathlib import Path -from functools import lru_cache from urllib.parse import urlparse from playwright.async_api import Route as async_Route @@ -14,8 +12,6 @@ from scrapling.core.utils import log from scrapling.core._types import Dict, Set, Tuple, Optional, Callable from scrapling.engines.constants import EXTRA_RESOURCES -__BYPASSES_DIR__ = Path(__file__).parent / "bypasses" - class ProxyDict(Struct): server: str @@ -111,13 +107,3 @@ def construct_proxy_dict(proxy_string: str | Dict[str, str] | Tuple) -> Dict: raise TypeError(f"Invalid proxy dictionary: {e}") raise TypeError(f"Invalid proxy string: {proxy_string}") - - -@lru_cache(10, typed=True) -def js_bypass_path(filename: str) -> str: - """Takes the base filename of a JS file inside the `bypasses` folder, then return the full path of it - - :param filename: The base filename of the JS file. - :return: The full path of the JS file. - """ - return str(__BYPASSES_DIR__ / filename) diff --git a/tests/fetchers/test_utils.py b/tests/fetchers/test_utils.py index 4637a63..0a4be9f 100644 --- a/tests/fetchers/test_utils.py +++ b/tests/fetchers/test_utils.py @@ -1,12 +1,10 @@ import pytest -from pathlib import Path from scrapling.engines.toolbelt.custom import StatusText, Response from scrapling.engines.toolbelt.navigation import ( construct_proxy_dict, create_intercept_handler, create_async_intercept_handler, - js_bypass_path, ) from scrapling.engines.toolbelt.fingerprints import ( generate_convincing_referer, @@ -203,25 +201,6 @@ class TestConstructProxyDict: construct_proxy_dict({"invalid": "structure"}) -class TestJsBypassPath: - """Test JavaScript bypass path utility""" - - def test_js_bypass_path(self): - """Test getting JavaScript bypass file path""" - result = js_bypass_path("webdriver_fully.js") - - assert isinstance(result, str) - assert result.endswith("webdriver_fully.js") - assert Path(result).exists() - - def test_js_bypass_path_caching(self): - """Test that js_bypass_path is cached""" - result1 = js_bypass_path("webdriver_fully.js") - result2 = js_bypass_path("webdriver_fully.js") - - assert result1 == result2 - - class TestFingerprintFunctions: """Test fingerprint generation functions"""