refactor/fix(browser fetchers): Solving a logic bug in page rotation

- Pages that get reused in rotation are possibly contaminated from previous settings used on them, and some are stubborn to remove, so the new approach replaces finished pages with new ones.
- Removed the `max_pages` from sync `StealthySession` to match `DynamicSession` (Doesn't mean anything in sync code)
This commit is contained in:
Karim shoair
2025-09-08 16:08:57 +03:00
parent f298b76b8e
commit ecaec65049
5 changed files with 521 additions and 455 deletions
+39 -15
View File
@@ -6,7 +6,9 @@ from playwright.async_api import Page as AsyncPage
from scrapling.core._types import Optional, List, Literal
PageState = Literal["ready", "busy", "error"] # States that a page can be in
PageState = Literal[
"finished", "ready", "busy", "error"
] # States that a page can be in
@dataclass
@@ -23,9 +25,9 @@ class PageInfo:
self.state = "busy"
self.url = url
def mark_ready(self):
"""Mark the page as ready for new requests"""
self.state = "ready"
def mark_finished(self):
"""Mark the page as finished for new requests"""
self.state = "finished"
self.url = ""
def mark_error(self):
@@ -62,24 +64,16 @@ class PagePool:
self.pages.append(page_info)
return page_info
def get_ready_page(self) -> Optional[PageInfo]:
"""Get a page that's ready for use"""
with self._lock:
for page_info in self.pages:
if page_info.state == "ready":
return page_info
return None
@property
def pages_count(self) -> int:
"""Get the total number of pages"""
return len(self.pages)
@property
def ready_count(self) -> int:
"""Get the number of ready pages"""
def finished_count(self) -> int:
"""Get the number of finished pages"""
with self._lock:
return sum(1 for p in self.pages if p.state == "ready")
return sum(1 for p in self.pages if p.state == "finished")
@property
def busy_count(self) -> int:
@@ -91,3 +85,33 @@ class PagePool:
"""Remove pages in error state"""
with self._lock:
self.pages = [p for p in self.pages if p.state != "error"]
def close_all_finished_pages(self):
"""Close all pages in finished state and remove them from the pool"""
with self._lock:
pages_to_remove = []
for page_info in self.pages:
if page_info.state == "finished":
try:
page_info.page.close()
except Exception:
pass
pages_to_remove.append(page_info)
for page_info in pages_to_remove:
self.pages.remove(page_info)
async def aclose_all_finished_pages(self):
"""Async version: Close all pages in finished state and remove them from the pool"""
with self._lock:
pages_to_remove = []
for page_info in self.pages:
if page_info.state == "finished":
try:
await page_info.page.close()
except Exception:
pass
pages_to_remove.append(page_info)
for page_info in pages_to_remove:
self.pages.remove(page_info)