freedak/Scrapling
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor(fetchers): rename internal api

Browse Source 
To make it easier to use to use sessions outside `with` context
This commit is contained in:
Karim shoair
2025-12-17 01:27:40 +02:00
parent ae719a9d54
commit b507e4d4a0
3 changed files with 61 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scrapling/engines/_browsers/_base.py
+4 -4
View File
@@ -40,7 +40,7 @@ class SyncSession:
self.context: BrowserContext | Any = None self.context: BrowserContext | Any = None
self._closed = False self._closed = False
def __create__(self): def start(self):
pass pass
def close(self): # pragma: no cover def close(self): # pragma: no cover
@@ -59,7 +59,7 @@ class SyncSession:
self._closed = True self._closed = True
def __enter__(self): def __enter__(self):
self.__create__() self.start()
return self return self
def __exit__(self, exc_type, exc_val, exc_tb): def __exit__(self, exc_type, exc_val, exc_tb):
@@ -145,7 +145,7 @@ class AsyncSession:
self._closed = False self._closed = False
self._lock = Lock() self._lock = Lock()
async def __create__(self): async def start(self):
pass pass
async def close(self): async def close(self):
@@ -164,7 +164,7 @@ class AsyncSession:
self._closed = True self._closed = True
async def __aenter__(self): async def __aenter__(self):
await self.__create__() await self.start()
return self return self
async def __aexit__(self, exc_type, exc_val, exc_tb): async def __aexit__(self, exc_type, exc_val, exc_tb):
scrapling/engines/_browsers/_camoufox.py
+22 -16
View File
@@ -102,16 +102,19 @@ class StealthySession(StealthySessionMixin, SyncSession):
self.__validate__(**kwargs) self.__validate__(**kwargs)
super().__init__(max_pages=self._max_pages) super().__init__(max_pages=self._max_pages)
def __create__(self): def start(self):
"""Create a browser for this instance and context.""" """Create a browser for this instance and context."""
self.playwright = sync_playwright().start() if not self.playwright:
self.context = self.playwright.firefox.launch_persistent_context(**self.launch_options) self.playwright = sync_playwright().start()
self.context = self.playwright.firefox.launch_persistent_context(**self.launch_options)
if self._init_script: # pragma: no cover if self._init_script: # pragma: no cover
self.context.add_init_script(path=self._init_script) self.context.add_init_script(path=self._init_script)
if self._cookies: # pragma: no cover if self._cookies: # pragma: no cover
self.context.add_cookies(self._cookies) self.context.add_cookies(self._cookies)
else:
raise RuntimeError("Session has been already started")
def _cloudflare_solver(self, page: Page) -> None: # pragma: no cover def _cloudflare_solver(self, page: Page) -> None: # pragma: no cover
"""Solve the cloudflare challenge displayed on the playwright page passed """Solve the cloudflare challenge displayed on the playwright page passed
@@ -299,18 +302,21 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):
self.__validate__(**kwargs) self.__validate__(**kwargs)
super().__init__(max_pages=self._max_pages) super().__init__(max_pages=self._max_pages)
async def __create__(self): async def start(self):
"""Create a browser for this instance and context.""" """Create a browser for this instance and context."""
self.playwright: AsyncPlaywright = await async_playwright().start() if not self.playwright:
self.context: AsyncBrowserContext = await self.playwright.firefox.launch_persistent_context( self.playwright: AsyncPlaywright = await async_playwright().start()
**self.launch_options self.context: AsyncBrowserContext = await self.playwright.firefox.launch_persistent_context(
) **self.launch_options
)
if self._init_script: # pragma: no cover if self._init_script: # pragma: no cover
await self.context.add_init_script(path=self._init_script) await self.context.add_init_script(path=self._init_script)
if self._cookies: if self._cookies:
await self.context.add_cookies(self._cookies) # pyright: ignore [reportArgumentType] await self.context.add_cookies(self._cookies) # pyright: ignore [reportArgumentType]
else:
raise RuntimeError("Session has been already started")
async def _cloudflare_solver(self, page: async_Page): # pragma: no cover async def _cloudflare_solver(self, page: async_Page): # pragma: no cover
"""Solve the cloudflare challenge displayed on the playwright page passed. The async version """Solve the cloudflare challenge displayed on the playwright page passed. The async version
scrapling/engines/_browsers/_controllers.py
+35 -29
View File
@@ -95,24 +95,27 @@ class DynamicSession(DynamicSessionMixin, SyncSession):
self.__validate__(**kwargs) self.__validate__(**kwargs)
super().__init__(max_pages=self._max_pages) super().__init__(max_pages=self._max_pages)
def __create__(self): def start(self):
"""Create a browser for this instance and context.""" """Create a browser for this instance and context."""
sync_context = sync_patchright if self._stealth else sync_playwright if not self.playwright:
sync_context = sync_patchright if self._stealth else sync_playwright
self.playwright: Playwright = sync_context().start() # pyright: ignore [reportAttributeAccessIssue] self.playwright: Playwright = sync_context().start() # pyright: ignore [reportAttributeAccessIssue]
if self._cdp_url: # pragma: no cover if self._cdp_url: # pragma: no cover
self.context = self.playwright.chromium.connect_over_cdp(endpoint_url=self._cdp_url).new_context( self.context = self.playwright.chromium.connect_over_cdp(endpoint_url=self._cdp_url).new_context(
**self.context_options **self.context_options
) )
else:
self.context = self.playwright.chromium.launch_persistent_context(**self.launch_options)
if self._init_script: # pragma: no cover
self.context.add_init_script(path=self._init_script)
if self._cookies: # pragma: no cover
self.context.add_cookies(self._cookies)
else: else:
self.context = self.playwright.chromium.launch_persistent_context(**self.launch_options) raise RuntimeError("Session has been already started")
if self._init_script: # pragma: no cover
self.context.add_init_script(path=self._init_script)
if self._cookies: # pragma: no cover
self.context.add_cookies(self._cookies)
def fetch(self, url: str, **kwargs: Unpack[PlaywrightFetchParams]) -> Response: def fetch(self, url: str, **kwargs: Unpack[PlaywrightFetchParams]) -> Response:
"""Opens up the browser and do your request based on your chosen options. """Opens up the browser and do your request based on your chosen options.
@@ -227,25 +230,28 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession):
self.__validate__(**kwargs) self.__validate__(**kwargs)
super().__init__(max_pages=self._max_pages) super().__init__(max_pages=self._max_pages)
async def __create__(self): async def start(self):
"""Create a browser for this instance and context.""" """Create a browser for this instance and context."""
async_context = async_patchright if self._stealth else async_playwright if not self.playwright:
async_context = async_patchright if self._stealth else async_playwright
self.playwright: AsyncPlaywright = await async_context().start() # pyright: ignore [reportAttributeAccessIssue] self.playwright: AsyncPlaywright = await async_context().start() # pyright: ignore [reportAttributeAccessIssue]
if self._cdp_url: if self._cdp_url:
browser = await self.playwright.chromium.connect_over_cdp(endpoint_url=self._cdp_url) browser = await self.playwright.chromium.connect_over_cdp(endpoint_url=self._cdp_url)
self.context: AsyncBrowserContext = await browser.new_context(**self.context_options) self.context: AsyncBrowserContext = await browser.new_context(**self.context_options)
else:
self.context: AsyncBrowserContext = await self.playwright.chromium.launch_persistent_context(
**self.launch_options
)
if self._init_script: # pragma: no cover
await self.context.add_init_script(path=self._init_script)
if self._cookies:
await self.context.add_cookies(self._cookies) # pyright: ignore
else: else:
self.context: AsyncBrowserContext = await self.playwright.chromium.launch_persistent_context( raise RuntimeError("Session has been already started")
**self.launch_options
)
if self._init_script: # pragma: no cover
await self.context.add_init_script(path=self._init_script)
if self._cookies:
await self.context.add_cookies(self._cookies) # pyright: ignore
async def fetch(self, url: str, **kwargs: Unpack[PlaywrightFetchParams]) -> Response: async def fetch(self, url: str, **kwargs: Unpack[PlaywrightFetchParams]) -> Response:
"""Opens up the browser and do your request based on your chosen options. """Opens up the browser and do your request based on your chosen options.