From b507e4d4a0c120817a6f5350dd389bbc130c12fc Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Wed, 17 Dec 2025 01:27:40 +0200 Subject: [PATCH] refactor(fetchers): rename internal api To make it easier to use to use sessions outside `with` context --- scrapling/engines/_browsers/_base.py | 8 +-- scrapling/engines/_browsers/_camoufox.py | 38 ++++++------ scrapling/engines/_browsers/_controllers.py | 64 +++++++++++---------- 3 files changed, 61 insertions(+), 49 deletions(-) diff --git a/scrapling/engines/_browsers/_base.py b/scrapling/engines/_browsers/_base.py index 6e5734e..7aa5aeb 100644 --- a/scrapling/engines/_browsers/_base.py +++ b/scrapling/engines/_browsers/_base.py @@ -40,7 +40,7 @@ class SyncSession: self.context: BrowserContext | Any = None self._closed = False - def __create__(self): + def start(self): pass def close(self): # pragma: no cover @@ -59,7 +59,7 @@ class SyncSession: self._closed = True def __enter__(self): - self.__create__() + self.start() return self def __exit__(self, exc_type, exc_val, exc_tb): @@ -145,7 +145,7 @@ class AsyncSession: self._closed = False self._lock = Lock() - async def __create__(self): + async def start(self): pass async def close(self): @@ -164,7 +164,7 @@ class AsyncSession: self._closed = True async def __aenter__(self): - await self.__create__() + await self.start() return self async def __aexit__(self, exc_type, exc_val, exc_tb): diff --git a/scrapling/engines/_browsers/_camoufox.py b/scrapling/engines/_browsers/_camoufox.py index 436a006..e3cbc67 100644 --- a/scrapling/engines/_browsers/_camoufox.py +++ b/scrapling/engines/_browsers/_camoufox.py @@ -102,16 +102,19 @@ class StealthySession(StealthySessionMixin, SyncSession): self.__validate__(**kwargs) super().__init__(max_pages=self._max_pages) - def __create__(self): + def start(self): """Create a browser for this instance and context.""" - self.playwright = sync_playwright().start() - self.context = self.playwright.firefox.launch_persistent_context(**self.launch_options) + if not self.playwright: + self.playwright = sync_playwright().start() + self.context = self.playwright.firefox.launch_persistent_context(**self.launch_options) - if self._init_script: # pragma: no cover - self.context.add_init_script(path=self._init_script) + if self._init_script: # pragma: no cover + self.context.add_init_script(path=self._init_script) - if self._cookies: # pragma: no cover - self.context.add_cookies(self._cookies) + if self._cookies: # pragma: no cover + self.context.add_cookies(self._cookies) + else: + raise RuntimeError("Session has been already started") def _cloudflare_solver(self, page: Page) -> None: # pragma: no cover """Solve the cloudflare challenge displayed on the playwright page passed @@ -299,18 +302,21 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession): self.__validate__(**kwargs) super().__init__(max_pages=self._max_pages) - async def __create__(self): + async def start(self): """Create a browser for this instance and context.""" - self.playwright: AsyncPlaywright = await async_playwright().start() - self.context: AsyncBrowserContext = await self.playwright.firefox.launch_persistent_context( - **self.launch_options - ) + if not self.playwright: + self.playwright: AsyncPlaywright = await async_playwright().start() + self.context: AsyncBrowserContext = await self.playwright.firefox.launch_persistent_context( + **self.launch_options + ) - if self._init_script: # pragma: no cover - await self.context.add_init_script(path=self._init_script) + if self._init_script: # pragma: no cover + await self.context.add_init_script(path=self._init_script) - if self._cookies: - await self.context.add_cookies(self._cookies) # pyright: ignore [reportArgumentType] + if self._cookies: + await self.context.add_cookies(self._cookies) # pyright: ignore [reportArgumentType] + else: + raise RuntimeError("Session has been already started") async def _cloudflare_solver(self, page: async_Page): # pragma: no cover """Solve the cloudflare challenge displayed on the playwright page passed. The async version diff --git a/scrapling/engines/_browsers/_controllers.py b/scrapling/engines/_browsers/_controllers.py index 9ab0e3c..a338c93 100644 --- a/scrapling/engines/_browsers/_controllers.py +++ b/scrapling/engines/_browsers/_controllers.py @@ -95,24 +95,27 @@ class DynamicSession(DynamicSessionMixin, SyncSession): self.__validate__(**kwargs) super().__init__(max_pages=self._max_pages) - def __create__(self): + def start(self): """Create a browser for this instance and context.""" - sync_context = sync_patchright if self._stealth else sync_playwright + if not self.playwright: + sync_context = sync_patchright if self._stealth else sync_playwright - self.playwright: Playwright = sync_context().start() # pyright: ignore [reportAttributeAccessIssue] + self.playwright: Playwright = sync_context().start() # pyright: ignore [reportAttributeAccessIssue] - if self._cdp_url: # pragma: no cover - self.context = self.playwright.chromium.connect_over_cdp(endpoint_url=self._cdp_url).new_context( - **self.context_options - ) + if self._cdp_url: # pragma: no cover + self.context = self.playwright.chromium.connect_over_cdp(endpoint_url=self._cdp_url).new_context( + **self.context_options + ) + else: + self.context = self.playwright.chromium.launch_persistent_context(**self.launch_options) + + if self._init_script: # pragma: no cover + self.context.add_init_script(path=self._init_script) + + if self._cookies: # pragma: no cover + self.context.add_cookies(self._cookies) else: - self.context = self.playwright.chromium.launch_persistent_context(**self.launch_options) - - if self._init_script: # pragma: no cover - self.context.add_init_script(path=self._init_script) - - if self._cookies: # pragma: no cover - self.context.add_cookies(self._cookies) + raise RuntimeError("Session has been already started") def fetch(self, url: str, **kwargs: Unpack[PlaywrightFetchParams]) -> Response: """Opens up the browser and do your request based on your chosen options. @@ -227,25 +230,28 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession): self.__validate__(**kwargs) super().__init__(max_pages=self._max_pages) - async def __create__(self): + async def start(self): """Create a browser for this instance and context.""" - async_context = async_patchright if self._stealth else async_playwright + if not self.playwright: + async_context = async_patchright if self._stealth else async_playwright - self.playwright: AsyncPlaywright = await async_context().start() # pyright: ignore [reportAttributeAccessIssue] + self.playwright: AsyncPlaywright = await async_context().start() # pyright: ignore [reportAttributeAccessIssue] - if self._cdp_url: - browser = await self.playwright.chromium.connect_over_cdp(endpoint_url=self._cdp_url) - self.context: AsyncBrowserContext = await browser.new_context(**self.context_options) + if self._cdp_url: + browser = await self.playwright.chromium.connect_over_cdp(endpoint_url=self._cdp_url) + self.context: AsyncBrowserContext = await browser.new_context(**self.context_options) + else: + self.context: AsyncBrowserContext = await self.playwright.chromium.launch_persistent_context( + **self.launch_options + ) + + if self._init_script: # pragma: no cover + await self.context.add_init_script(path=self._init_script) + + if self._cookies: + await self.context.add_cookies(self._cookies) # pyright: ignore else: - self.context: AsyncBrowserContext = await self.playwright.chromium.launch_persistent_context( - **self.launch_options - ) - - if self._init_script: # pragma: no cover - await self.context.add_init_script(path=self._init_script) - - if self._cookies: - await self.context.add_cookies(self._cookies) # pyright: ignore + raise RuntimeError("Session has been already started") async def fetch(self, url: str, **kwargs: Unpack[PlaywrightFetchParams]) -> Response: """Opens up the browser and do your request based on your chosen options.