freedak/Scrapling
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(fetchers): Add argument to control the user data directory

Browse Source
This commit is contained in:
Karim shoair
2025-10-12 04:48:59 +03:00
parent 8c995c936d
commit c34e10faed
4 changed files with 27 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scrapling/engines/_browsers/_base.py
+6 -3
View File
@@ -149,7 +149,8 @@ class DynamicSessionMixin:
self.wait_selector_state = config.wait_selector_state
self.selector_config = config.selector_config
self.page_action = config.page_action
self._headers_keys = set(map(str.lower, self.extra_headers.keys())) if self.extra_headers else set()
self.user_data_dir = config.user_data_dir
self._headers_keys = {header.lower() for header in self.extra_headers.keys()} if self.extra_headers else set()
self.__initiate_browser_options__()
def __initiate_browser_options__(self):
@@ -173,6 +174,7 @@ class DynamicSessionMixin:
)
self.launch_options["extra_http_headers"] = dict(self.launch_options["extra_http_headers"])
self.launch_options["proxy"] = dict(self.launch_options["proxy"]) or None
self.launch_options["user_data_dir"] = self.user_data_dir
self.context_options = dict()
else:
# while `context_options` is left to be used when cdp mode is enabled
@@ -221,7 +223,8 @@ class StealthySessionMixin:
self.selector_config = config.selector_config
self.additional_args = config.additional_args
self.page_action = config.page_action
self._headers_keys = set(map(str.lower, self.extra_headers.keys())) if self.extra_headers else set()
self.user_data_dir = config.user_data_dir
self._headers_keys = {header.lower() for header in self.extra_headers.keys()} if self.extra_headers else set()
self.__initiate_browser_options__()
def __initiate_browser_options__(self):
@@ -239,7 +242,7 @@ class StealthySessionMixin:
"block_webrtc": self.block_webrtc,
"block_images": self.block_images, # Careful! it makes some websites don't finish loading at all like stackoverflow even in headful mode.
"os": None if self.os_randomize else get_os_name(),
"user_data_dir": "",
"user_data_dir": self.user_data_dir,
"ff_version": __ff_version_str__,
"firefox_user_prefs": {
# This is what enabling `enable_cache` does internally, so we do it from here instead
scrapling/engines/_browsers/_camoufox.py
+11 -9
View File
@@ -104,6 +104,7 @@ class StealthySession(StealthySessionMixin, SyncSession):
os_randomize: bool = False,
disable_ads: bool = False,
geoip: bool = False,
user_data_dir: str = "",
selector_config: Optional[Dict] = None,
additional_args: Optional[Dict] = None,
):
@@ -136,6 +137,7 @@ class StealthySession(StealthySessionMixin, SyncSession):
:param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name.
:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
:param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.
:param user_data_dir: Path to a User Data Directory, which stores browser session data like cookies and local storage. The default is to create a temporary directory.
:param selector_config: The arguments that will be passed in the end while creating the final Selector's class.
:param additional_args: Additional arguments to be passed to Camoufox as additional settings, and it takes higher priority than Scrapling's settings.
"""
@@ -159,6 +161,7 @@ class StealthySession(StealthySessionMixin, SyncSession):
block_images=block_images,
block_webrtc=block_webrtc,
os_randomize=os_randomize,
user_data_dir=user_data_dir,
wait_selector=wait_selector,
google_search=google_search,
extra_headers=extra_headers,
@@ -173,9 +176,7 @@ class StealthySession(StealthySessionMixin, SyncSession):
def __create__(self):
"""Create a browser for this instance and context."""
self.playwright = sync_playwright().start()
self.context = self.playwright.firefox.launch_persistent_context( # pragma: no cover
**self.launch_options
)
self.context = self.playwright.firefox.launch_persistent_context(**self.launch_options)
if self.init_script: # pragma: no cover
self.context.add_init_script(path=self.init_script)
@@ -226,7 +227,6 @@ class StealthySession(StealthySessionMixin, SyncSession):
:param page: The targeted page
:return:
"""
page.wait_for_load_state("networkidle")
challenge_type = self._detect_cloudflare(self._get_page_content(page))
if not challenge_type:
log.error("No Cloudflare challenge found.")
@@ -441,6 +441,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):
os_randomize: bool = False,
disable_ads: bool = False,
geoip: bool = False,
user_data_dir: str = "",
selector_config: Optional[Dict] = None,
additional_args: Optional[Dict] = None,
):
@@ -474,6 +475,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):
:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
:param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.
:param max_pages: The maximum number of tabs to be opened at the same time. It will be used in rotation through a PagePool.
:param user_data_dir: Path to a User Data Directory, which stores browser session data like cookies and local storage. The default is to create a temporary directory.
:param selector_config: The arguments that will be passed in the end while creating the final Selector's class.
:param additional_args: Additional arguments to be passed to Camoufox as additional settings, and it takes higher priority than Scrapling's settings.
"""
@@ -499,6 +501,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):
wait_selector=wait_selector,
google_search=google_search,
extra_headers=extra_headers,
user_data_dir=user_data_dir,
additional_args=additional_args,
selector_config=selector_config,
solve_cloudflare=solve_cloudflare,
@@ -509,8 +512,8 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):
async def __create__(self):
"""Create a browser for this instance and context."""
self.playwright: AsyncPlaywright | None = await async_playwright().start()
self.context: AsyncBrowserContext | None = await self.playwright.firefox.launch_persistent_context(
self.playwright: AsyncPlaywright = await async_playwright().start()
self.context: AsyncBrowserContext = await self.playwright.firefox.launch_persistent_context(
**self.launch_options
)
@@ -534,11 +537,11 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):
if self.context:
await self.context.close()
self.context = None
self.context = None # pyright: ignore
if self.playwright:
await self.playwright.stop()
self.playwright = None
self.playwright = None # pyright: ignore
self._closed = True
@@ -563,7 +566,6 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):
:param page: The async targeted page
:return:
"""
await page.wait_for_load_state("networkidle")
challenge_type = self._detect_cloudflare(await self._get_page_content(page))
if not challenge_type:
log.error("No Cloudflare challenge found.")
scrapling/engines/_browsers/_controllers.py
+8 -2
View File
@@ -97,6 +97,7 @@ class DynamicSession(DynamicSessionMixin, SyncSession):
network_idle: bool = False,
load_dom: bool = True,
wait_selector_state: SelectorWaitStates = "attached",
user_data_dir: str = "",
selector_config: Optional[Dict] = None,
):
"""A Browser session manager with page pooling, it's using a persistent browser Context by default with a temporary user profile directory.
@@ -124,6 +125,7 @@ class DynamicSession(DynamicSessionMixin, SyncSession):
:param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name.
:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
:param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.
:param user_data_dir: Path to a User Data Directory, which stores browser session data like cookies and local storage. The default is to create a temporary directory.
:param selector_config: The arguments that will be passed in the end while creating the final Selector's class.
"""
self.__validate__(
@@ -143,6 +145,7 @@ class DynamicSession(DynamicSessionMixin, SyncSession):
hide_canvas=hide_canvas,
init_script=init_script,
network_idle=network_idle,
user_data_dir=user_data_dir,
google_search=google_search,
extra_headers=extra_headers,
wait_selector=wait_selector,
@@ -164,7 +167,7 @@ class DynamicSession(DynamicSessionMixin, SyncSession):
**self.context_options
)
else:
self.context = self.playwright.chromium.launch_persistent_context(user_data_dir="", **self.launch_options)
self.context = self.playwright.chromium.launch_persistent_context(**self.launch_options)
if self.init_script: # pragma: no cover
self.context.add_init_script(path=self.init_script)
@@ -341,6 +344,7 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession):
network_idle: bool = False,
load_dom: bool = True,
wait_selector_state: SelectorWaitStates = "attached",
user_data_dir: str = "",
selector_config: Optional[Dict] = None,
):
"""A Browser session manager with page pooling
@@ -369,6 +373,7 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession):
:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
:param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.
:param max_pages: The maximum number of tabs to be opened at the same time. It will be used in rotation through a PagePool.
:param user_data_dir: Path to a User Data Directory, which stores browser session data like cookies and local storage. The default is to create a temporary directory.
:param selector_config: The arguments that will be passed in the end while creating the final Selector's class.
"""
@@ -389,6 +394,7 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession):
hide_canvas=hide_canvas,
init_script=init_script,
network_idle=network_idle,
user_data_dir=user_data_dir,
google_search=google_search,
extra_headers=extra_headers,
wait_selector=wait_selector,
@@ -410,7 +416,7 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession):
self.context: AsyncBrowserContext = await browser.new_context(**self.context_options)
else:
self.context: AsyncBrowserContext = await self.playwright.chromium.launch_persistent_context(
user_data_dir="", **self.launch_options
**self.launch_options
)
if self.init_script: # pragma: no cover
scrapling/engines/_browsers/_validators.py
+2
View File
@@ -87,6 +87,7 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False):
network_idle: bool = False
load_dom: bool = True
wait_selector_state: SelectorWaitStates = "attached"
user_data_dir: str = ""
selector_config: Optional[Dict] = {}
def __post_init__(self):
@@ -134,6 +135,7 @@ class CamoufoxConfig(Struct, kw_only=True, frozen=False):
os_randomize: bool = False
disable_ads: bool = False
geoip: bool = False
user_data_dir: str = ""
selector_config: Optional[Dict] = {}
additional_args: Optional[Dict] = {}