feat(DynamicFetcher): Use persistent context by default for better stealth

And might solve the sandbox issue with GitHub

scrapling/engines/_browsers/_config_tools.py

@@ -56,16 +56,43 @@ def _set_flags(hide_canvas, disable_webgl):
 
 
 @lru_cache(2, typed=True)
-def _launch_kwargs(headless, real_chrome, stealth, hide_canvas, disable_webgl) -> Tuple:
+def _launch_kwargs(
+    headless,
+    proxy,
+    locale,
+    extra_headers,
+    useragent,
+    real_chrome,
+    stealth,
+    hide_canvas,
+    disable_webgl,
+) -> Tuple:
     """Creates the arguments we will use while launching playwright's browser"""
     launch_kwargs = {
         "headless": headless,
         "ignore_default_args": HARMFUL_DEFAULT_ARGS,
         "channel": "chrome" if real_chrome else "chromium",
+        "proxy": proxy or tuple(),
+        "locale": locale,
+        "color_scheme": "dark",  # Bypasses the 'prefersLightColor' check in creepjs
+        "device_scale_factor": 2,
+        "extra_http_headers": extra_headers or tuple(),
+        "user_agent": useragent or __default_useragent__,
     }
     if stealth:
         launch_kwargs.update(
-            {"args": _set_flags(hide_canvas, disable_webgl), "chromium_sandbox": True}
+            {
+                "args": _set_flags(hide_canvas, disable_webgl),
+                "chromium_sandbox": True,
+                "is_mobile": False,
+                "has_touch": False,
+                # I'm thinking about disabling it to rest from all Service Workers' headache, but let's keep it as it is for now
+                "service_workers": "allow",
+                "ignore_https_errors": True,
+                "screen": {"width": 1920, "height": 1080},
+                "viewport": {"width": 1920, "height": 1080},
+                "permissions": ["geolocation", "notifications"],
+            }
         )
 
     return tuple(launch_kwargs.items())

scrapling/engines/_browsers/_controllers.py

@@ -5,7 +5,6 @@ from playwright.sync_api import (
     Response as SyncPlaywrightResponse,
     sync_playwright,
     BrowserType,
-    Browser,
     BrowserContext,
     Playwright,
     Locator,
@@ -14,7 +13,6 @@ from playwright.async_api import (
     async_playwright,
     Response as AsyncPlaywrightResponse,
     BrowserType as AsyncBrowserType,
-    Browser as AsyncBrowser,
     BrowserContext as AsyncBrowserContext,
     Playwright as AsyncPlaywright,
     Locator as AsyncLocator,
@@ -177,7 +175,6 @@ class DynamicSession:
         self.wait_selector_state = config.wait_selector_state
 
         self.playwright: Optional[Playwright] = None
-        self.browser: Optional[Union[BrowserType, Browser]] = None
         self.context: Optional[BrowserContext] = None
         self.page_pool = PagePool(self.max_pages)
         self._closed = False
@@ -191,15 +188,25 @@ class DynamicSession:
         self.__initiate_browser_options__()
 
     def __initiate_browser_options__(self):
+        # `launch_options` is used with persistent context
         self.launch_options = dict(
             _launch_kwargs(
                 self.headless,
+                self.proxy,
+                self.locale,
+                tuple(self.extra_headers.items()) if self.extra_headers else tuple(),
+                self.useragent,
                 self.real_chrome,
                 self.stealth,
                 self.hide_canvas,
                 self.disable_webgl,
             )
         )
+        self.launch_options["extra_http_headers"] = dict(
+            self.launch_options["extra_http_headers"]
+        )
+        self.launch_options["proxy"] = dict(self.launch_options["proxy"]) or None
+        # while `context_options` is left to be used when cdp mode is enabled
         self.context_options = dict(
             _context_kwargs(
                 self.proxy,
@@ -223,15 +230,17 @@ class DynamicSession:
 
         self.playwright = sync_context().start()
 
-        browser_launcher = getattr(
+        browser_launcher: BrowserType = getattr(
             self.playwright, "chrome" if self.real_chrome else "chromium"
         )
         if self.cdp_url:
-            self.browser = browser_launcher.connect_over_cdp(endpoint_url=self.cdp_url)
+            browser = browser_launcher.connect_over_cdp(endpoint_url=self.cdp_url)
+            self.context = browser.new_context(**self.context_options)
         else:
-            self.browser = browser_launcher.launch(**self.launch_options)
+            self.context = browser_launcher.launch_persistent_context(
+                user_data_dir="", **self.launch_options
+            )
 
-        self.context = self.browser.new_context(**self.context_options)
         if self.cookies:
             self.context.add_cookies(self.cookies)
 
@@ -251,10 +260,6 @@ class DynamicSession:
             self.context.close()
             self.context = None
 
-        if self.browser:
-            self.browser.close()
-            self.browser = None
-
         if self.playwright:
             self.playwright.stop()
             self.playwright = None
@@ -459,7 +464,6 @@ class AsyncDynamicSession(DynamicSession):
         )
 
         self.playwright: Optional[AsyncPlaywright] = None
-        self.browser: Optional[Union[AsyncBrowserType, AsyncBrowser]] = None
         self.context: Optional[AsyncBrowserContext] = None
         self._lock = Lock()
         self.__enter__ = None
@@ -478,15 +482,16 @@ class AsyncDynamicSession(DynamicSession):
             self.playwright, "chrome" if self.real_chrome else "chromium"
         )
         if self.cdp_url:
-            self.browser = await browser_launcher.connect_over_cdp(
-                endpoint_url=self.cdp_url
+            browser = await browser_launcher.connect_over_cdp(endpoint_url=self.cdp_url)
+            self.context: AsyncBrowserContext = await browser.new_context(
+                **self.context_options
             )
         else:
-            self.browser = await browser_launcher.launch(**self.launch_options)
-
-        self.context: AsyncBrowserContext = await self.browser.new_context(
-            **self.context_options
-        )
+            self.context: AsyncBrowserContext = (
+                await browser_launcher.launch_persistent_context(
+                    user_data_dir="", **self.launch_options
+                )
+            )
 
         if self.cookies:
             await self.context.add_cookies(self.cookies)
@@ -507,10 +512,6 @@ class AsyncDynamicSession(DynamicSession):
             await self.context.close()
             self.context = None
 
-        if self.browser:
-            await self.browser.close()
-            self.browser = None
-
         if self.playwright:
             await self.playwright.stop()
             self.playwright = None