freedak/Scrapling
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(browsers): solving an issues with leaving playwright loop open when cdp connection fails

Browse Source 
This has been causing issues with tests for a long time, and now finally found the reason.
This commit is contained in:
Karim shoair
2026-01-18 16:46:48 +02:00
parent 02c9dffc77
commit bf5aa021d4
2 changed files with 76 additions and 58 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scrapling/engines/_browsers/_controllers.py
+36 -27
View File
@@ -3,13 +3,11 @@ from asyncio import sleep as asyncio_sleep
from playwright.sync_api import ( from playwright.sync_api import (
Locator, Locator,
Playwright,
sync_playwright, sync_playwright,
) )
from playwright.async_api import ( from playwright.async_api import (
async_playwright, async_playwright,
Locator as AsyncLocator, Locator as AsyncLocator,
Playwright as AsyncPlaywright,
BrowserContext as AsyncBrowserContext, BrowserContext as AsyncBrowserContext,
) )
@@ -71,21 +69,27 @@ class DynamicSession(SyncSession, DynamicSessionMixin):
def start(self): def start(self):
"""Create a browser for this instance and context.""" """Create a browser for this instance and context."""
if not self.playwright: if not self.playwright:
self.playwright: Playwright = sync_playwright().start() # pyright: ignore [reportAttributeAccessIssue] self.playwright = sync_playwright().start()
if self._config.cdp_url: # pragma: no cover try:
browser = self.playwright.chromium.connect_over_cdp(endpoint_url=self._config.cdp_url) if self._config.cdp_url: # pragma: no cover
self.context = browser.new_context(**self._context_options) browser = self.playwright.chromium.connect_over_cdp(endpoint_url=self._config.cdp_url)
else: self.context = browser.new_context(**self._context_options)
self.context = self.playwright.chromium.launch_persistent_context(**self._launch_options) else:
self.context = self.playwright.chromium.launch_persistent_context(**self._launch_options)
if self._config.init_script: # pragma: no cover if self._config.init_script: # pragma: no cover
self.context.add_init_script(path=self._config.init_script) self.context.add_init_script(path=self._config.init_script)
if self._config.cookies: # pragma: no cover if self._config.cookies: # pragma: no cover
self.context.add_cookies(self._config.cookies) self.context.add_cookies(self._config.cookies)
self._is_alive = True self._is_alive = True
except Exception:
# Clean up playwright if browser setup fails
self.playwright.stop()
self.playwright = None
raise
else: else:
raise RuntimeError("Session has been already started") raise RuntimeError("Session has been already started")
@@ -209,23 +213,28 @@ class AsyncDynamicSession(AsyncSession, DynamicSessionMixin):
async def start(self): async def start(self):
"""Create a browser for this instance and context.""" """Create a browser for this instance and context."""
if not self.playwright: if not self.playwright:
self.playwright: AsyncPlaywright = await async_playwright().start() # pyright: ignore [reportAttributeAccessIssue] self.playwright = await async_playwright().start()
try:
if self._config.cdp_url:
browser = await self.playwright.chromium.connect_over_cdp(endpoint_url=self._config.cdp_url)
self.context: AsyncBrowserContext = await browser.new_context(**self._context_options)
else:
self.context: AsyncBrowserContext = await self.playwright.chromium.launch_persistent_context(
**self._launch_options
)
if self._config.cdp_url: if self._config.init_script: # pragma: no cover
browser = await self.playwright.chromium.connect_over_cdp(endpoint_url=self._config.cdp_url) await self.context.add_init_script(path=self._config.init_script)
self.context: AsyncBrowserContext = await browser.new_context(**self._context_options)
else:
self.context: AsyncBrowserContext = await self.playwright.chromium.launch_persistent_context(
**self._launch_options
)
if self._config.init_script: # pragma: no cover if self._config.cookies:
await self.context.add_init_script(path=self._config.init_script) await self.context.add_cookies(self._config.cookies) # pyright: ignore
if self._config.cookies: self._is_alive = True
await self.context.add_cookies(self._config.cookies) # pyright: ignore except Exception:
# Clean up playwright if browser setup fails
self._is_alive = True await self.playwright.stop()
self.playwright = None
raise
else: else:
raise RuntimeError("Session has been already started") raise RuntimeError("Session has been already started")
scrapling/engines/_browsers/_stealth.py
+40 -31
View File
@@ -6,12 +6,10 @@ from asyncio import sleep as asyncio_sleep
from playwright.sync_api import ( from playwright.sync_api import (
Locator, Locator,
Page, Page,
Playwright,
) )
from playwright.async_api import ( from playwright.async_api import (
Page as async_Page, Page as async_Page,
Locator as AsyncLocator, Locator as AsyncLocator,
Playwright as AsyncPlaywright,
BrowserContext as AsyncBrowserContext, BrowserContext as AsyncBrowserContext,
) )
from patchright.sync_api import sync_playwright from patchright.sync_api import sync_playwright
@@ -82,24 +80,30 @@ class StealthySession(SyncSession, StealthySessionMixin):
def start(self): def start(self):
"""Create a browser for this instance and context.""" """Create a browser for this instance and context."""
if not self.playwright: if not self.playwright:
self.playwright: Playwright = sync_playwright().start() # pyright: ignore [reportAttributeAccessIssue] self.playwright = sync_playwright().start()
if self._config.cdp_url: # pragma: no cover try:
browser = self.playwright.chromium.connect_over_cdp(endpoint_url=self._config.cdp_url) if self._config.cdp_url: # pragma: no cover
self.context = browser.new_context(**self._context_options) browser = self.playwright.chromium.connect_over_cdp(endpoint_url=self._config.cdp_url)
else: self.context = browser.new_context(**self._context_options)
self.context = self.playwright.chromium.launch_persistent_context(**self._launch_options) else:
self.context = self.playwright.chromium.launch_persistent_context(**self._launch_options)
for script in _compiled_stealth_scripts(): for script in _compiled_stealth_scripts():
self.context.add_init_script(script=script) self.context.add_init_script(script=script)
if self._config.init_script: # pragma: no cover if self._config.init_script: # pragma: no cover
self.context.add_init_script(path=self._config.init_script) self.context.add_init_script(path=self._config.init_script)
if self._config.cookies: # pragma: no cover if self._config.cookies: # pragma: no cover
self.context.add_cookies(self._config.cookies) self.context.add_cookies(self._config.cookies)
self._is_alive = True self._is_alive = True
except Exception:
# Clean up playwright if browser setup fails
self.playwright.stop()
self.playwright = None
raise
else: else:
raise RuntimeError("Session has been already started") raise RuntimeError("Session has been already started")
@@ -308,26 +312,31 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin):
async def start(self): async def start(self):
"""Create a browser for this instance and context.""" """Create a browser for this instance and context."""
if not self.playwright: if not self.playwright:
self.playwright: AsyncPlaywright = await async_playwright().start() # pyright: ignore [reportAttributeAccessIssue] self.playwright = await async_playwright().start()
try:
if self._config.cdp_url:
browser = await self.playwright.chromium.connect_over_cdp(endpoint_url=self._config.cdp_url)
self.context: AsyncBrowserContext = await browser.new_context(**self._context_options)
else:
self.context: AsyncBrowserContext = await self.playwright.chromium.launch_persistent_context(
**self._launch_options
)
if self._config.cdp_url: for script in _compiled_stealth_scripts():
browser = await self.playwright.chromium.connect_over_cdp(endpoint_url=self._config.cdp_url) await self.context.add_init_script(script=script)
self.context: AsyncBrowserContext = await browser.new_context(**self._context_options)
else:
self.context: AsyncBrowserContext = await self.playwright.chromium.launch_persistent_context(
**self._launch_options
)
for script in _compiled_stealth_scripts(): if self._config.init_script: # pragma: no cover
await self.context.add_init_script(script=script) await self.context.add_init_script(path=self._config.init_script)
if self._config.init_script: # pragma: no cover if self._config.cookies:
await self.context.add_init_script(path=self._config.init_script) await self.context.add_cookies(self._config.cookies) # pyright: ignore
if self._config.cookies: self._is_alive = True
await self.context.add_cookies(self._config.cookies) # pyright: ignore except Exception:
# Clean up playwright if browser setup fails
self._is_alive = True await self.playwright.stop()
self.playwright = None
raise
else: else:
raise RuntimeError("Session has been already started") raise RuntimeError("Session has been already started")