freedak/Scrapling
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Adding the option to use real chrome browser directly

Browse Source
This commit is contained in:
Karim shoair
2024-11-24 15:14:42 +02:00
parent 088117441d
commit e96e7d4440
1 changed files with 9 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scrapling/engines/pw.py
+9 -3
View File
@@ -28,6 +28,7 @@ class PlaywrightEngine:
wait_selector: Optional[str] = None, wait_selector: Optional[str] = None,
wait_selector_state: Optional[str] = 'attached', wait_selector_state: Optional[str] = 'attached',
stealth: Optional[bool] = False, stealth: Optional[bool] = False,
real_chrome: Optional[bool] = False,
hide_canvas: Optional[bool] = False, hide_canvas: Optional[bool] = False,
disable_webgl: Optional[bool] = False, disable_webgl: Optional[bool] = False,
cdp_url: Optional[str] = None, cdp_url: Optional[str] = None,
@@ -51,6 +52,7 @@ class PlaywrightEngine:
:param wait_selector: Wait for a specific css selector to be in a specific state. :param wait_selector: Wait for a specific css selector to be in a specific state.
:param wait_selector_state: The state to wait for the selector given with `wait_selector`. Default state is `attached`. :param wait_selector_state: The state to wait for the selector given with `wait_selector`. Default state is `attached`.
:param stealth: Enables stealth mode, check the documentation to see what stealth mode does currently. :param stealth: Enables stealth mode, check the documentation to see what stealth mode does currently.
:param real_chrome: If you have chrome browser installed on your device, enable this and the Fetcher will launch an instance of your browser and use it.
:param hide_canvas: Add random noise to canvas operations to prevent fingerprinting. :param hide_canvas: Add random noise to canvas operations to prevent fingerprinting.
:param disable_webgl: Disables WebGL and WebGL 2.0 support entirely. :param disable_webgl: Disables WebGL and WebGL 2.0 support entirely.
:param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers/NSTBrowser through CDP. :param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers/NSTBrowser through CDP.
@@ -67,6 +69,7 @@ class PlaywrightEngine:
self.stealth = bool(stealth) self.stealth = bool(stealth)
self.hide_canvas = bool(hide_canvas) self.hide_canvas = bool(hide_canvas)
self.disable_webgl = bool(disable_webgl) self.disable_webgl = bool(disable_webgl)
self.real_chrome = bool(real_chrome)
self.google_search = bool(google_search) self.google_search = bool(google_search)
self.extra_headers = extra_headers or {} self.extra_headers = extra_headers or {}
self.proxy = construct_proxy_dict(proxy) self.proxy = construct_proxy_dict(proxy)
@@ -119,7 +122,8 @@ class PlaywrightEngine:
:param url: Target url. :param url: Target url.
:return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers` :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
""" """
if not self.stealth: if not self.stealth or self.real_chrome:
# Because rebrowser_playwright doesn't play well with real browsers
from playwright.sync_api import sync_playwright from playwright.sync_api import sync_playwright
else: else:
from rebrowser_playwright.sync_api import sync_playwright from rebrowser_playwright.sync_api import sync_playwright
@@ -146,9 +150,11 @@ class PlaywrightEngine:
browser = p.chromium.connect_over_cdp(endpoint_url=cdp_url) browser = p.chromium.connect_over_cdp(endpoint_url=cdp_url)
else: else:
if self.stealth: if self.stealth:
browser = p.chromium.launch(headless=self.headless, args=flags, ignore_default_args=['--enable-automation'], chromium_sandbox=True) browser = p.chromium.launch(
headless=self.headless, args=flags, ignore_default_args=['--enable-automation'], chromium_sandbox=True, channel='chrome' if self.real_chrome else 'chromium'
)
else: else:
browser = p.chromium.launch(headless=self.headless, ignore_default_args=['--enable-automation']) browser = p.chromium.launch(headless=self.headless, ignore_default_args=['--enable-automation'], channel='chrome' if self.real_chrome else 'chromium')
# Creating the context # Creating the context
if self.stealth: if self.stealth: