From aadae4a25b374f2dc642a37ecbf83d2ba55ba999 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Thu, 31 Oct 2024 02:12:37 +0300 Subject: [PATCH] Adding the JavaScript bypasses files for stealth mode --- .../engines/bypasses/navigator_plugins.js | 40 ++++ .../bypasses/notification_permission.js | 5 + scrapling/engines/bypasses/pdf_viewer.js | 5 + .../bypasses/playwright_fingerprint.js | 2 + scrapling/engines/bypasses/screen_props.js | 27 +++ scrapling/engines/bypasses/webdriver_fully.js | 27 +++ scrapling/engines/bypasses/window_chrome.js | 213 ++++++++++++++++++ 7 files changed, 319 insertions(+) create mode 100644 scrapling/engines/bypasses/navigator_plugins.js create mode 100644 scrapling/engines/bypasses/notification_permission.js create mode 100644 scrapling/engines/bypasses/pdf_viewer.js create mode 100644 scrapling/engines/bypasses/playwright_fingerprint.js create mode 100644 scrapling/engines/bypasses/screen_props.js create mode 100644 scrapling/engines/bypasses/webdriver_fully.js create mode 100644 scrapling/engines/bypasses/window_chrome.js diff --git a/scrapling/engines/bypasses/navigator_plugins.js b/scrapling/engines/bypasses/navigator_plugins.js new file mode 100644 index 0000000..653fa5f --- /dev/null +++ b/scrapling/engines/bypasses/navigator_plugins.js @@ -0,0 +1,40 @@ +if(navigator.plugins.length == 0){ + Object.defineProperty(navigator, 'plugins', { + get: () => { + const PDFViewerPlugin = Object.create(Plugin.prototype, { + description: { value: 'Portable Document Format', enumerable: false }, + filename: { value: 'internal-pdf-viewer', enumerable: false }, + name: { value: 'PDF Viewer', enumerable: false }, + }); + const ChromePDFViewer = Object.create(Plugin.prototype, { + description: { value: 'Portable Document Format', enumerable: false }, + filename: { value: 'internal-pdf-viewer', enumerable: false }, + name: { value: 'Chrome PDF Viewer', enumerable: false }, + }); + const ChromiumPDFViewer = Object.create(Plugin.prototype, { + description: { value: 'Portable Document Format', enumerable: false }, + filename: { value: 'internal-pdf-viewer', enumerable: false }, + name: { value: 'Chromium PDF Viewer', enumerable: false }, + }); + const EdgePDFViewer = Object.create(Plugin.prototype, { + description: { value: 'Portable Document Format', enumerable: false }, + filename: { value: 'internal-pdf-viewer', enumerable: false }, + name: { value: 'Microsoft Edge PDF Viewer', enumerable: false }, + }); + const WebKitPDFPlugin = Object.create(Plugin.prototype, { + description: { value: 'Portable Document Format', enumerable: false }, + filename: { value: 'internal-pdf-viewer', enumerable: false }, + name: { value: 'WebKit built-in PDF', enumerable: false }, + }); + + return Object.create(PluginArray.prototype, { + length: { value: 5 }, + 0: { value: PDFViewerPlugin }, + 1: { value: ChromePDFViewer }, + 2: { value: ChromiumPDFViewer }, + 3: { value: EdgePDFViewer }, + 4: { value: WebKitPDFPlugin }, + }); + }, + }); +} \ No newline at end of file diff --git a/scrapling/engines/bypasses/notification_permission.js b/scrapling/engines/bypasses/notification_permission.js new file mode 100644 index 0000000..0c9c676 --- /dev/null +++ b/scrapling/engines/bypasses/notification_permission.js @@ -0,0 +1,5 @@ +// Bypasses `notificationIsDenied` test in creepsjs's 'Like Headless' sections +const isSecure = document.location.protocol.startsWith('https') +if (isSecure){ + Object.defineProperty(Notification, 'permission', {get: () => 'default'}) +} \ No newline at end of file diff --git a/scrapling/engines/bypasses/pdf_viewer.js b/scrapling/engines/bypasses/pdf_viewer.js new file mode 100644 index 0000000..4c88702 --- /dev/null +++ b/scrapling/engines/bypasses/pdf_viewer.js @@ -0,0 +1,5 @@ +// PDF viewer enabled +// Bypasses `pdfIsDisabled` test in creepsjs's 'Like Headless' sections +Object.defineProperty(navigator, 'pdfViewerEnabled', { + get: () => true, +}); \ No newline at end of file diff --git a/scrapling/engines/bypasses/playwright_fingerprint.js b/scrapling/engines/bypasses/playwright_fingerprint.js new file mode 100644 index 0000000..bfba6be --- /dev/null +++ b/scrapling/engines/bypasses/playwright_fingerprint.js @@ -0,0 +1,2 @@ +// Remove playwright fingerprint => https://github.com/microsoft/playwright/commit/c9e673c6dca746384338ab6bb0cf63c7e7caa9b2#diff-087773eea292da9db5a3f27de8f1a2940cdb895383ad750c3cd8e01772a35b40R915 +delete __pwInitScripts; \ No newline at end of file diff --git a/scrapling/engines/bypasses/screen_props.js b/scrapling/engines/bypasses/screen_props.js new file mode 100644 index 0000000..5056b4b --- /dev/null +++ b/scrapling/engines/bypasses/screen_props.js @@ -0,0 +1,27 @@ +const windowScreenProps = { + // Dimensions + innerHeight: 0, + innerWidth: 0, + outerHeight: 754, + outerWidth: 1313, + + // Position + screenX: 19, + pageXOffset: 0, + pageYOffset: 0, + + // Display + devicePixelRatio: 2 +}; + +try { + for (const [prop, value] of Object.entries(windowScreenProps)) { + if (value > 0) { + // The 0 values are introduced by collecting in the hidden iframe. + // They are document sizes anyway so no need to test them or inject them. + window[prop] = value; + } + } +} catch (e) { + console.warn(e); +}; \ No newline at end of file diff --git a/scrapling/engines/bypasses/webdriver_fully.js b/scrapling/engines/bypasses/webdriver_fully.js new file mode 100644 index 0000000..4bda260 --- /dev/null +++ b/scrapling/engines/bypasses/webdriver_fully.js @@ -0,0 +1,27 @@ +// Create a function that looks like a native getter +const nativeGetter = function get webdriver() { + return false; +}; + +// Copy over native function properties +Object.defineProperties(nativeGetter, { + name: { value: 'get webdriver', configurable: true }, + length: { value: 0, configurable: true }, + toString: { + value: function() { + return `function get webdriver() { [native code] }`; + }, + configurable: true + } +}); + +// Make it look native +Object.setPrototypeOf(nativeGetter, Function.prototype); + +// Apply the modified descriptor +Object.defineProperty(Navigator.prototype, 'webdriver', { + get: nativeGetter, + set: undefined, + enumerable: true, + configurable: true +}); \ No newline at end of file diff --git a/scrapling/engines/bypasses/window_chrome.js b/scrapling/engines/bypasses/window_chrome.js new file mode 100644 index 0000000..ba63a9c --- /dev/null +++ b/scrapling/engines/bypasses/window_chrome.js @@ -0,0 +1,213 @@ +// To escape `HEADCHR_CHROME_OBJ` test in headless mode => https://github.com/antoinevastel/fp-collect/blob/master/src/fpCollect.js#L322 +// Faking window.chrome fully + +if (!window.chrome) { + // First, save all existing properties + const originalKeys = Object.getOwnPropertyNames(window); + const tempObj = {}; + + // Recreate all properties in original order + for (const key of originalKeys) { + const descriptor = Object.getOwnPropertyDescriptor(window, key); + const value = window[key]; + // delete window[key]; + Object.defineProperty(tempObj, key, descriptor); + } + + // Use the exact property descriptor found in headful Chrome + // fetch it via `Object.getOwnPropertyDescriptor(window, 'chrome')` + const mockChrome = { + loadTimes: {}, + csi: {}, + app: { + isInstalled: false + }, + // Add other Chrome-specific properties + }; + + Object.defineProperty(tempObj, 'chrome', { + writable: true, + enumerable: true, + configurable: false, + value: mockChrome + }); + for (const key of Object.getOwnPropertyNames(tempObj)) { + try { + Object.defineProperty(window, key, + Object.getOwnPropertyDescriptor(tempObj, key)); + } catch (e) {} + }; + // todo: solve this + // Using line below bypasses the hasHighChromeIndex test in creepjs ==> https://github.com/abrahamjuliot/creepjs/blob/master/src/headless/index.ts#L121 + // Chrome object have to be in the end of the window properties + // Object.assign(window, tempObj); + // But makes window.chrome unreadable on 'https://bot.sannysoft.com/' +} + +// That means we're running headful and don't need to mock anything +if ('app' in window.chrome) { + return; // Nothing to do here +} +const makeError = { + ErrorInInvocation: fn => { + const err = new TypeError(`Error in invocation of app.${fn}()`); + return utils.stripErrorWithAnchor( + err, + `at ${fn} (eval at `, + ); + }, +}; +// check with: `JSON.stringify(window.chrome['app'])` +const STATIC_DATA = JSON.parse( + ` +{ + "isInstalled": false, + "InstallState": { + "DISABLED": "disabled", + "INSTALLED": "installed", + "NOT_INSTALLED": "not_installed" + }, + "RunningState": { + "CANNOT_RUN": "cannot_run", + "READY_TO_RUN": "ready_to_run", + "RUNNING": "running" + } +} + `.trim(), + ); +window.chrome.app = { + ...STATIC_DATA, + + get isInstalled() { + return false; + }, + + getDetails: function getDetails() { + if (arguments.length) { + throw makeError.ErrorInInvocation(`getDetails`); + } + return null; + }, + getIsInstalled: function getDetails() { + if (arguments.length) { + throw makeError.ErrorInInvocation(`getIsInstalled`); + } + return false; + }, + runningState: function getDetails() { + if (arguments.length) { + throw makeError.ErrorInInvocation(`runningState`); + } + return 'cannot_run'; + }, +}; +// Check that the Navigation Timing API v1 is available, we need that +if (!window.performance || !window.performance.timing) { + return; +} +const {timing} = window.performance; +window.chrome.csi = function () { + return { + onloadT: timing.domContentLoadedEventEnd, + startE: timing.navigationStart, + pageT: Date.now() - timing.navigationStart, + tran: 15, // Transition type or something + }; +}; +if (!window.PerformancePaintTiming){ + return; +} +const {performance} = window; +// Some stuff is not available on about:blank as it requires a navigation to occur, +// let's harden the code to not fail then: +const ntEntryFallback = { + nextHopProtocol: 'h2', + type: 'other', +}; + +// The API exposes some funky info regarding the connection +const protocolInfo = { + get connectionInfo() { + const ntEntry = + performance.getEntriesByType('navigation')[0] || ntEntryFallback; + return ntEntry.nextHopProtocol; + }, + get npnNegotiatedProtocol() { + // NPN is deprecated in favor of ALPN, but this implementation returns the + // HTTP/2 or HTTP2+QUIC/39 requests negotiated via ALPN. + const ntEntry = + performance.getEntriesByType('navigation')[0] || ntEntryFallback; + return ['h2', 'hq'].includes(ntEntry.nextHopProtocol) + ? ntEntry.nextHopProtocol + : 'unknown'; + }, + get navigationType() { + const ntEntry = + performance.getEntriesByType('navigation')[0] || ntEntryFallback; + return ntEntry.type; + }, + get wasAlternateProtocolAvailable() { + // The Alternate-Protocol header is deprecated in favor of Alt-Svc + // (https://www.mnot.net/blog/2016/03/09/alt-svc), so technically this + // should always return false. + return false; + }, + get wasFetchedViaSpdy() { + // SPDY is deprecated in favor of HTTP/2, but this implementation returns + // true for HTTP/2 or HTTP2+QUIC/39 as well. + const ntEntry = + performance.getEntriesByType('navigation')[0] || ntEntryFallback; + return ['h2', 'hq'].includes(ntEntry.nextHopProtocol); + }, + get wasNpnNegotiated() { + // NPN is deprecated in favor of ALPN, but this implementation returns true + // for HTTP/2 or HTTP2+QUIC/39 requests negotiated via ALPN. + const ntEntry = + performance.getEntriesByType('navigation')[0] || ntEntryFallback; + return ['h2', 'hq'].includes(ntEntry.nextHopProtocol); + }, +}; + +// Truncate number to specific number of decimals, most of the `loadTimes` stuff has 3 +function toFixed(num, fixed) { + var re = new RegExp('^-?\\d+(?:.\\d{0,' + (fixed || -1) + '})?'); + return num.toString().match(re)[0]; +} + +const timingInfo = { + get firstPaintAfterLoadTime() { + // This was never actually implemented and always returns 0. + return 0; + }, + get requestTime() { + return timing.navigationStart / 1000; + }, + get startLoadTime() { + return timing.navigationStart / 1000; + }, + get commitLoadTime() { + return timing.responseStart / 1000; + }, + get finishDocumentLoadTime() { + return timing.domContentLoadedEventEnd / 1000; + }, + get finishLoadTime() { + return timing.loadEventEnd / 1000; + }, + get firstPaintTime() { + const fpEntry = performance.getEntriesByType('paint')[0] || { + startTime: timing.loadEventEnd / 1000, // Fallback if no navigation occured (`about:blank`) + }; + return toFixed( + (fpEntry.startTime + performance.timeOrigin) / 1000, + 3, + ); + }, +}; + +window.chrome.loadTimes = function () { + return { + ...protocolInfo, + ...timingInfo, + }; +}; \ No newline at end of file