From 7ba333c3c9f6678a0a15787fb0cbc2136904e842 Mon Sep 17 00:00:00 2001 From: Abdullah <52079299+AbdullahY36@users.noreply.github.com> Date: Wed, 17 Sep 2025 18:49:45 +0300 Subject: [PATCH] perf(session): improve page management and speed up execution Close pages immediately after tasks finish to reduce resource usage and improve overall performance. Update corresponding test files to reflect the new page lifecycle management. --- scrapling/engines/_browsers/_base.py | 10 ------ scrapling/engines/_browsers/_camoufox.py | 10 +++--- scrapling/engines/_browsers/_controllers.py | 11 +++--- scrapling/engines/_browsers/_page.py | 35 ------------------- tests/fetchers/async/test_camoufox_session.py | 12 ++++--- tests/fetchers/async/test_dynamic_session.py | 14 ++++---- tests/fetchers/test_pages.py | 18 ---------- 7 files changed, 27 insertions(+), 83 deletions(-) diff --git a/scrapling/engines/_browsers/_base.py b/scrapling/engines/_browsers/_base.py index 8e94559..6445e00 100644 --- a/scrapling/engines/_browsers/_base.py +++ b/scrapling/engines/_browsers/_base.py @@ -44,16 +44,11 @@ class SyncSession: ) -> PageInfo: # pragma: no cover """Get a new page to use""" - # Close all finished pages to ensure clean state - self.page_pool.close_all_finished_pages() - # If we're at max capacity after cleanup, wait for busy pages to finish if self.page_pool.pages_count >= self.max_pages: start_time = time() while time() - start_time < self._max_wait_for_page: - # Wait for any pages to finish, then clean them up sleep(0.05) - self.page_pool.close_all_finished_pages() if self.page_pool.pages_count < self.max_pages: break else: @@ -105,16 +100,11 @@ class AsyncSession(SyncSession): ) -> PageInfo: # pragma: no cover """Get a new page to use""" async with self._lock: - # Close all finished pages to ensure clean state - await self.page_pool.aclose_all_finished_pages() - # If we're at max capacity after cleanup, wait for busy pages to finish if self.page_pool.pages_count >= self.max_pages: start_time = time() while time() - start_time < self._max_wait_for_page: - # Wait for any pages to finish, then clean them up await asyncio_sleep(0.05) - await self.page_pool.aclose_all_finished_pages() if self.page_pool.pages_count < self.max_pages: break else: diff --git a/scrapling/engines/_browsers/_camoufox.py b/scrapling/engines/_browsers/_camoufox.py index a4ca6f0..18ec633 100644 --- a/scrapling/engines/_browsers/_camoufox.py +++ b/scrapling/engines/_browsers/_camoufox.py @@ -381,8 +381,9 @@ class StealthySession(StealthySessionMixin, SyncSession): page_info.page, first_response, final_response, params.selector_config ) - # Mark the page as finished for next use - page_info.mark_finished() + # Close the page, to free up resources + page_info.page.close() + self.page_pool.pages.remove(page_info) return response @@ -701,8 +702,9 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession): page_info.page, first_response, final_response, params.selector_config ) - # Mark the page as finished for next use - page_info.mark_finished() + # Close the page, to free up resources + await page_info.page.close() + self.page_pool.pages.remove(page_info) return response diff --git a/scrapling/engines/_browsers/_controllers.py b/scrapling/engines/_browsers/_controllers.py index 70b3771..722d7c2 100644 --- a/scrapling/engines/_browsers/_controllers.py +++ b/scrapling/engines/_browsers/_controllers.py @@ -305,8 +305,9 @@ class DynamicSession(DynamicSessionMixin, SyncSession): page_info.page, first_response, final_response, params.selector_config ) - # Mark the page as finished for next use - page_info.mark_finished() + # Close the page, to free up resources + page_info.page.close() + self.page_pool.pages.remove(page_info) return response @@ -554,9 +555,9 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession): page_info.page, first_response, final_response, params.selector_config ) - # Mark the page as finished for next use - page_info.mark_finished() - + # Close the page, to free up resources + await page_info.page.close() + self.page_pool.pages.remove(page_info) return response except Exception as e: # pragma: no cover diff --git a/scrapling/engines/_browsers/_page.py b/scrapling/engines/_browsers/_page.py index ffa4b22..8c80944 100644 --- a/scrapling/engines/_browsers/_page.py +++ b/scrapling/engines/_browsers/_page.py @@ -23,11 +23,6 @@ class PageInfo: self.state = "busy" self.url = url - def mark_finished(self): - """Mark the page as finished for new requests""" - self.state = "finished" - self.url = "" - def mark_error(self): """Mark the page as having an error""" self.state = "error" @@ -83,33 +78,3 @@ class PagePool: """Remove pages in error state""" with self._lock: self.pages = [p for p in self.pages if p.state != "error"] - - def close_all_finished_pages(self): - """Close all pages in finished state and remove them from the pool""" - with self._lock: - pages_to_remove = [] - for page_info in self.pages: - if page_info.state == "finished": - try: - page_info.page.close() - except Exception: - pass - pages_to_remove.append(page_info) - - for page_info in pages_to_remove: - self.pages.remove(page_info) - - async def aclose_all_finished_pages(self): - """Async version: Close all pages in finished state and remove them from the pool""" - with self._lock: - pages_to_remove = [] - for page_info in self.pages: - if page_info.state == "finished": - try: - await page_info.page.close() - except Exception: - pass - pages_to_remove.append(page_info) - - for page_info in pages_to_remove: - self.pages.remove(page_info) diff --git a/tests/fetchers/async/test_camoufox_session.py b/tests/fetchers/async/test_camoufox_session.py index 2971488..af9b32b 100644 --- a/tests/fetchers/async/test_camoufox_session.py +++ b/tests/fetchers/async/test_camoufox_session.py @@ -54,16 +54,18 @@ class TestAsyncStealthySession: """Test page pool creation and reuse""" async with AsyncStealthySession() as session: # The first request creates a page - _ = await session.fetch(urls["basic"]) - assert session.page_pool.pages_count == 1 + response = await session.fetch(urls["basic"]) + assert response.status == 200 + assert session.page_pool.pages_count == 0 # The second request should reuse the page - _ = await session.fetch(urls["html"]) - assert session.page_pool.pages_count == 1 + response = await session.fetch(urls["html"]) + assert response.status == 200 + assert session.page_pool.pages_count == 0 # Check pool stats stats = session.get_pool_stats() - assert stats["total_pages"] == 1 + assert stats["total_pages"] == 0 assert stats["max_pages"] == 1 async def test_stealthy_session_with_options(self, urls): diff --git a/tests/fetchers/async/test_dynamic_session.py b/tests/fetchers/async/test_dynamic_session.py index d7a4ea9..e6f0b5a 100644 --- a/tests/fetchers/async/test_dynamic_session.py +++ b/tests/fetchers/async/test_dynamic_session.py @@ -53,16 +53,18 @@ class TestAsyncDynamicSession: """Test page pool creation and reuse""" async with AsyncDynamicSession() as session: # The first request creates a page - _ = await session.fetch(urls["basic"]) - assert session.page_pool.pages_count == 1 - + response = await session.fetch(urls["basic"]) + assert response.status == 200 + assert session.page_pool.pages_count == 0 + # The second request should reuse the page - _ = await session.fetch(urls["html"]) - assert session.page_pool.pages_count == 1 + response = await session.fetch(urls["html"]) + assert response.status == 200 + assert session.page_pool.pages_count == 0 # Check pool stats stats = session.get_pool_stats() - assert stats["total_pages"] == 1 + assert stats["total_pages"] == 0 assert stats["max_pages"] == 1 async def test_dynamic_session_with_options(self, urls): diff --git a/tests/fetchers/test_pages.py b/tests/fetchers/test_pages.py index e3ba3bc..69dc4ba 100644 --- a/tests/fetchers/test_pages.py +++ b/tests/fetchers/test_pages.py @@ -24,10 +24,6 @@ class TestPageInfo: assert page_info.state == "busy" assert page_info.url == "https://example.com" - page_info.mark_finished() - assert page_info.state == "finished" - assert page_info.url == "" - page_info.mark_error() assert page_info.state == "error" @@ -88,21 +84,7 @@ class TestPagePool: with pytest.raises(RuntimeError): pool.add_page(Mock()) - def test_get_ready_page(self): - """Test getting ready page""" - pool = PagePool(max_pages=3) - # Add pages - page1 = pool.add_page(Mock()) - page2 = pool.add_page(Mock()) - - # Mark them as finished - page1.mark_finished() - page2.mark_finished() - - # test - pool.close_all_finished_pages() - assert pool.pages_count == 0 def test_cleanup_error_pages(self): """Test cleaning up error pages"""