From c34e10faed194d4a47135d52d00265b03057968c Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Sun, 12 Oct 2025 04:48:59 +0300 Subject: [PATCH] feat(fetchers): Add argument to control the user data directory --- scrapling/engines/_browsers/_base.py | 9 ++++++--- scrapling/engines/_browsers/_camoufox.py | 20 +++++++++++--------- scrapling/engines/_browsers/_controllers.py | 10 ++++++++-- scrapling/engines/_browsers/_validators.py | 2 ++ 4 files changed, 27 insertions(+), 14 deletions(-) diff --git a/scrapling/engines/_browsers/_base.py b/scrapling/engines/_browsers/_base.py index f16ef14..ce82dbb 100644 --- a/scrapling/engines/_browsers/_base.py +++ b/scrapling/engines/_browsers/_base.py @@ -149,7 +149,8 @@ class DynamicSessionMixin: self.wait_selector_state = config.wait_selector_state self.selector_config = config.selector_config self.page_action = config.page_action - self._headers_keys = set(map(str.lower, self.extra_headers.keys())) if self.extra_headers else set() + self.user_data_dir = config.user_data_dir + self._headers_keys = {header.lower() for header in self.extra_headers.keys()} if self.extra_headers else set() self.__initiate_browser_options__() def __initiate_browser_options__(self): @@ -173,6 +174,7 @@ class DynamicSessionMixin: ) self.launch_options["extra_http_headers"] = dict(self.launch_options["extra_http_headers"]) self.launch_options["proxy"] = dict(self.launch_options["proxy"]) or None + self.launch_options["user_data_dir"] = self.user_data_dir self.context_options = dict() else: # while `context_options` is left to be used when cdp mode is enabled @@ -221,7 +223,8 @@ class StealthySessionMixin: self.selector_config = config.selector_config self.additional_args = config.additional_args self.page_action = config.page_action - self._headers_keys = set(map(str.lower, self.extra_headers.keys())) if self.extra_headers else set() + self.user_data_dir = config.user_data_dir + self._headers_keys = {header.lower() for header in self.extra_headers.keys()} if self.extra_headers else set() self.__initiate_browser_options__() def __initiate_browser_options__(self): @@ -239,7 +242,7 @@ class StealthySessionMixin: "block_webrtc": self.block_webrtc, "block_images": self.block_images, # Careful! it makes some websites don't finish loading at all like stackoverflow even in headful mode. "os": None if self.os_randomize else get_os_name(), - "user_data_dir": "", + "user_data_dir": self.user_data_dir, "ff_version": __ff_version_str__, "firefox_user_prefs": { # This is what enabling `enable_cache` does internally, so we do it from here instead diff --git a/scrapling/engines/_browsers/_camoufox.py b/scrapling/engines/_browsers/_camoufox.py index 6e2b620..7076ff8 100644 --- a/scrapling/engines/_browsers/_camoufox.py +++ b/scrapling/engines/_browsers/_camoufox.py @@ -104,6 +104,7 @@ class StealthySession(StealthySessionMixin, SyncSession): os_randomize: bool = False, disable_ads: bool = False, geoip: bool = False, + user_data_dir: str = "", selector_config: Optional[Dict] = None, additional_args: Optional[Dict] = None, ): @@ -136,6 +137,7 @@ class StealthySession(StealthySessionMixin, SyncSession): :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name. :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only. + :param user_data_dir: Path to a User Data Directory, which stores browser session data like cookies and local storage. The default is to create a temporary directory. :param selector_config: The arguments that will be passed in the end while creating the final Selector's class. :param additional_args: Additional arguments to be passed to Camoufox as additional settings, and it takes higher priority than Scrapling's settings. """ @@ -159,6 +161,7 @@ class StealthySession(StealthySessionMixin, SyncSession): block_images=block_images, block_webrtc=block_webrtc, os_randomize=os_randomize, + user_data_dir=user_data_dir, wait_selector=wait_selector, google_search=google_search, extra_headers=extra_headers, @@ -173,9 +176,7 @@ class StealthySession(StealthySessionMixin, SyncSession): def __create__(self): """Create a browser for this instance and context.""" self.playwright = sync_playwright().start() - self.context = self.playwright.firefox.launch_persistent_context( # pragma: no cover - **self.launch_options - ) + self.context = self.playwright.firefox.launch_persistent_context(**self.launch_options) if self.init_script: # pragma: no cover self.context.add_init_script(path=self.init_script) @@ -226,7 +227,6 @@ class StealthySession(StealthySessionMixin, SyncSession): :param page: The targeted page :return: """ - page.wait_for_load_state("networkidle") challenge_type = self._detect_cloudflare(self._get_page_content(page)) if not challenge_type: log.error("No Cloudflare challenge found.") @@ -441,6 +441,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession): os_randomize: bool = False, disable_ads: bool = False, geoip: bool = False, + user_data_dir: str = "", selector_config: Optional[Dict] = None, additional_args: Optional[Dict] = None, ): @@ -474,6 +475,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession): :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only. :param max_pages: The maximum number of tabs to be opened at the same time. It will be used in rotation through a PagePool. + :param user_data_dir: Path to a User Data Directory, which stores browser session data like cookies and local storage. The default is to create a temporary directory. :param selector_config: The arguments that will be passed in the end while creating the final Selector's class. :param additional_args: Additional arguments to be passed to Camoufox as additional settings, and it takes higher priority than Scrapling's settings. """ @@ -499,6 +501,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession): wait_selector=wait_selector, google_search=google_search, extra_headers=extra_headers, + user_data_dir=user_data_dir, additional_args=additional_args, selector_config=selector_config, solve_cloudflare=solve_cloudflare, @@ -509,8 +512,8 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession): async def __create__(self): """Create a browser for this instance and context.""" - self.playwright: AsyncPlaywright | None = await async_playwright().start() - self.context: AsyncBrowserContext | None = await self.playwright.firefox.launch_persistent_context( + self.playwright: AsyncPlaywright = await async_playwright().start() + self.context: AsyncBrowserContext = await self.playwright.firefox.launch_persistent_context( **self.launch_options ) @@ -534,11 +537,11 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession): if self.context: await self.context.close() - self.context = None + self.context = None # pyright: ignore if self.playwright: await self.playwright.stop() - self.playwright = None + self.playwright = None # pyright: ignore self._closed = True @@ -563,7 +566,6 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession): :param page: The async targeted page :return: """ - await page.wait_for_load_state("networkidle") challenge_type = self._detect_cloudflare(await self._get_page_content(page)) if not challenge_type: log.error("No Cloudflare challenge found.") diff --git a/scrapling/engines/_browsers/_controllers.py b/scrapling/engines/_browsers/_controllers.py index 1fbb9a3..ccfbb11 100644 --- a/scrapling/engines/_browsers/_controllers.py +++ b/scrapling/engines/_browsers/_controllers.py @@ -97,6 +97,7 @@ class DynamicSession(DynamicSessionMixin, SyncSession): network_idle: bool = False, load_dom: bool = True, wait_selector_state: SelectorWaitStates = "attached", + user_data_dir: str = "", selector_config: Optional[Dict] = None, ): """A Browser session manager with page pooling, it's using a persistent browser Context by default with a temporary user profile directory. @@ -124,6 +125,7 @@ class DynamicSession(DynamicSessionMixin, SyncSession): :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name. :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only. + :param user_data_dir: Path to a User Data Directory, which stores browser session data like cookies and local storage. The default is to create a temporary directory. :param selector_config: The arguments that will be passed in the end while creating the final Selector's class. """ self.__validate__( @@ -143,6 +145,7 @@ class DynamicSession(DynamicSessionMixin, SyncSession): hide_canvas=hide_canvas, init_script=init_script, network_idle=network_idle, + user_data_dir=user_data_dir, google_search=google_search, extra_headers=extra_headers, wait_selector=wait_selector, @@ -164,7 +167,7 @@ class DynamicSession(DynamicSessionMixin, SyncSession): **self.context_options ) else: - self.context = self.playwright.chromium.launch_persistent_context(user_data_dir="", **self.launch_options) + self.context = self.playwright.chromium.launch_persistent_context(**self.launch_options) if self.init_script: # pragma: no cover self.context.add_init_script(path=self.init_script) @@ -341,6 +344,7 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession): network_idle: bool = False, load_dom: bool = True, wait_selector_state: SelectorWaitStates = "attached", + user_data_dir: str = "", selector_config: Optional[Dict] = None, ): """A Browser session manager with page pooling @@ -369,6 +373,7 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession): :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only. :param max_pages: The maximum number of tabs to be opened at the same time. It will be used in rotation through a PagePool. + :param user_data_dir: Path to a User Data Directory, which stores browser session data like cookies and local storage. The default is to create a temporary directory. :param selector_config: The arguments that will be passed in the end while creating the final Selector's class. """ @@ -389,6 +394,7 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession): hide_canvas=hide_canvas, init_script=init_script, network_idle=network_idle, + user_data_dir=user_data_dir, google_search=google_search, extra_headers=extra_headers, wait_selector=wait_selector, @@ -410,7 +416,7 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession): self.context: AsyncBrowserContext = await browser.new_context(**self.context_options) else: self.context: AsyncBrowserContext = await self.playwright.chromium.launch_persistent_context( - user_data_dir="", **self.launch_options + **self.launch_options ) if self.init_script: # pragma: no cover diff --git a/scrapling/engines/_browsers/_validators.py b/scrapling/engines/_browsers/_validators.py index 448f0de..af90d9e 100644 --- a/scrapling/engines/_browsers/_validators.py +++ b/scrapling/engines/_browsers/_validators.py @@ -87,6 +87,7 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False): network_idle: bool = False load_dom: bool = True wait_selector_state: SelectorWaitStates = "attached" + user_data_dir: str = "" selector_config: Optional[Dict] = {} def __post_init__(self): @@ -134,6 +135,7 @@ class CamoufoxConfig(Struct, kw_only=True, frozen=False): os_randomize: bool = False disable_ads: bool = False geoip: bool = False + user_data_dir: str = "" selector_config: Optional[Dict] = {} additional_args: Optional[Dict] = {}