From dbf7aa7761a43e6d8c2ecd14b44e1eb3e7aa973c Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Fri, 27 Jun 2025 03:02:46 +0300 Subject: [PATCH] feat(DynamicFetcher): Use persistent context by default for better stealth And might solve the sandbox issue with GitHub --- scrapling/engines/_browsers/_config_tools.py | 31 ++++++++++++- scrapling/engines/_browsers/_controllers.py | 47 ++++++++++---------- 2 files changed, 53 insertions(+), 25 deletions(-) diff --git a/scrapling/engines/_browsers/_config_tools.py b/scrapling/engines/_browsers/_config_tools.py index b63b7f6..a2ee057 100644 --- a/scrapling/engines/_browsers/_config_tools.py +++ b/scrapling/engines/_browsers/_config_tools.py @@ -56,16 +56,43 @@ def _set_flags(hide_canvas, disable_webgl): @lru_cache(2, typed=True) -def _launch_kwargs(headless, real_chrome, stealth, hide_canvas, disable_webgl) -> Tuple: +def _launch_kwargs( + headless, + proxy, + locale, + extra_headers, + useragent, + real_chrome, + stealth, + hide_canvas, + disable_webgl, +) -> Tuple: """Creates the arguments we will use while launching playwright's browser""" launch_kwargs = { "headless": headless, "ignore_default_args": HARMFUL_DEFAULT_ARGS, "channel": "chrome" if real_chrome else "chromium", + "proxy": proxy or tuple(), + "locale": locale, + "color_scheme": "dark", # Bypasses the 'prefersLightColor' check in creepjs + "device_scale_factor": 2, + "extra_http_headers": extra_headers or tuple(), + "user_agent": useragent or __default_useragent__, } if stealth: launch_kwargs.update( - {"args": _set_flags(hide_canvas, disable_webgl), "chromium_sandbox": True} + { + "args": _set_flags(hide_canvas, disable_webgl), + "chromium_sandbox": True, + "is_mobile": False, + "has_touch": False, + # I'm thinking about disabling it to rest from all Service Workers' headache, but let's keep it as it is for now + "service_workers": "allow", + "ignore_https_errors": True, + "screen": {"width": 1920, "height": 1080}, + "viewport": {"width": 1920, "height": 1080}, + "permissions": ["geolocation", "notifications"], + } ) return tuple(launch_kwargs.items()) diff --git a/scrapling/engines/_browsers/_controllers.py b/scrapling/engines/_browsers/_controllers.py index 213e1a6..1c5dd89 100644 --- a/scrapling/engines/_browsers/_controllers.py +++ b/scrapling/engines/_browsers/_controllers.py @@ -5,7 +5,6 @@ from playwright.sync_api import ( Response as SyncPlaywrightResponse, sync_playwright, BrowserType, - Browser, BrowserContext, Playwright, Locator, @@ -14,7 +13,6 @@ from playwright.async_api import ( async_playwright, Response as AsyncPlaywrightResponse, BrowserType as AsyncBrowserType, - Browser as AsyncBrowser, BrowserContext as AsyncBrowserContext, Playwright as AsyncPlaywright, Locator as AsyncLocator, @@ -177,7 +175,6 @@ class DynamicSession: self.wait_selector_state = config.wait_selector_state self.playwright: Optional[Playwright] = None - self.browser: Optional[Union[BrowserType, Browser]] = None self.context: Optional[BrowserContext] = None self.page_pool = PagePool(self.max_pages) self._closed = False @@ -191,15 +188,25 @@ class DynamicSession: self.__initiate_browser_options__() def __initiate_browser_options__(self): + # `launch_options` is used with persistent context self.launch_options = dict( _launch_kwargs( self.headless, + self.proxy, + self.locale, + tuple(self.extra_headers.items()) if self.extra_headers else tuple(), + self.useragent, self.real_chrome, self.stealth, self.hide_canvas, self.disable_webgl, ) ) + self.launch_options["extra_http_headers"] = dict( + self.launch_options["extra_http_headers"] + ) + self.launch_options["proxy"] = dict(self.launch_options["proxy"]) or None + # while `context_options` is left to be used when cdp mode is enabled self.context_options = dict( _context_kwargs( self.proxy, @@ -223,15 +230,17 @@ class DynamicSession: self.playwright = sync_context().start() - browser_launcher = getattr( + browser_launcher: BrowserType = getattr( self.playwright, "chrome" if self.real_chrome else "chromium" ) if self.cdp_url: - self.browser = browser_launcher.connect_over_cdp(endpoint_url=self.cdp_url) + browser = browser_launcher.connect_over_cdp(endpoint_url=self.cdp_url) + self.context = browser.new_context(**self.context_options) else: - self.browser = browser_launcher.launch(**self.launch_options) + self.context = browser_launcher.launch_persistent_context( + user_data_dir="", **self.launch_options + ) - self.context = self.browser.new_context(**self.context_options) if self.cookies: self.context.add_cookies(self.cookies) @@ -251,10 +260,6 @@ class DynamicSession: self.context.close() self.context = None - if self.browser: - self.browser.close() - self.browser = None - if self.playwright: self.playwright.stop() self.playwright = None @@ -459,7 +464,6 @@ class AsyncDynamicSession(DynamicSession): ) self.playwright: Optional[AsyncPlaywright] = None - self.browser: Optional[Union[AsyncBrowserType, AsyncBrowser]] = None self.context: Optional[AsyncBrowserContext] = None self._lock = Lock() self.__enter__ = None @@ -478,15 +482,16 @@ class AsyncDynamicSession(DynamicSession): self.playwright, "chrome" if self.real_chrome else "chromium" ) if self.cdp_url: - self.browser = await browser_launcher.connect_over_cdp( - endpoint_url=self.cdp_url + browser = await browser_launcher.connect_over_cdp(endpoint_url=self.cdp_url) + self.context: AsyncBrowserContext = await browser.new_context( + **self.context_options ) else: - self.browser = await browser_launcher.launch(**self.launch_options) - - self.context: AsyncBrowserContext = await self.browser.new_context( - **self.context_options - ) + self.context: AsyncBrowserContext = ( + await browser_launcher.launch_persistent_context( + user_data_dir="", **self.launch_options + ) + ) if self.cookies: await self.context.add_cookies(self.cookies) @@ -507,10 +512,6 @@ class AsyncDynamicSession(DynamicSession): await self.context.close() self.context = None - if self.browser: - await self.browser.close() - self.browser = None - if self.playwright: await self.playwright.stop() self.playwright = None