diff --git a/scrapling/engines/_browsers/_base.py b/scrapling/engines/_browsers/_base.py index eefb9f3..2d42b08 100644 --- a/scrapling/engines/_browsers/_base.py +++ b/scrapling/engines/_browsers/_base.py @@ -196,11 +196,14 @@ class SyncSession: context_options = self._build_context_with_proxy(proxy) context: BrowserContext = self.browser.new_context(**context_options) + page_info = None try: context = self._initialize_context(self._config, context) page_info = self._get_page(timeout, extra_headers, disable_resources, blocked_domains, context=context) yield page_info finally: + if page_info is not None and page_info in self.page_pool.pages: + self.page_pool.pages.remove(page_info) context.close() else: # Standard mode: use PagePool with persistent context @@ -380,6 +383,7 @@ class AsyncSession: context_options = self._build_context_with_proxy(proxy) context: AsyncBrowserContext = await self.browser.new_context(**context_options) + page_info = None try: context = await self._initialize_context(self._config, context) page_info = await self._get_page( @@ -387,6 +391,8 @@ class AsyncSession: ) yield page_info finally: + if page_info is not None and page_info in self.page_pool.pages: + self.page_pool.pages.remove(page_info) await context.close() else: # Standard mode: use PagePool with persistent context diff --git a/tests/fetchers/test_pages.py b/tests/fetchers/test_pages.py index 69dc4ba..b726c74 100644 --- a/tests/fetchers/test_pages.py +++ b/tests/fetchers/test_pages.py @@ -84,6 +84,15 @@ class TestPagePool: with pytest.raises(RuntimeError): pool.add_page(Mock()) + def test_proxy_rotation_pool_leak(self): + pool = PagePool(max_pages=1) + page_info = pool.add_page(Mock()) + assert pool.pages_count == 1 + pool.pages.remove(page_info) + assert pool.pages_count == 0 + pool.add_page(Mock()) + assert pool.pages_count == 1 + def test_cleanup_error_pages(self):