From ecaec65049598476ffbe15c05790eca58a04b3ae Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Mon, 8 Sep 2025 16:08:57 +0300 Subject: [PATCH] refactor/fix(browser fetchers): Solving a logic bug in page rotation - Pages that get reused in rotation are possibly contaminated from previous settings used on them, and some are stubborn to remove, so the new approach replaces finished pages with new ones. - Removed the `max_pages` from sync `StealthySession` to match `DynamicSession` (Doesn't mean anything in sync code) --- scrapling/engines/_browsers/_base.py | 403 ++++++++++++++++++++ scrapling/engines/_browsers/_camoufox.py | 269 ++----------- scrapling/engines/_browsers/_controllers.py | 249 ++---------- scrapling/engines/_browsers/_page.py | 54 ++- scrapling/fetchers.py | 1 - 5 files changed, 521 insertions(+), 455 deletions(-) create mode 100644 scrapling/engines/_browsers/_base.py diff --git a/scrapling/engines/_browsers/_base.py b/scrapling/engines/_browsers/_base.py new file mode 100644 index 0000000..55b3310 --- /dev/null +++ b/scrapling/engines/_browsers/_base.py @@ -0,0 +1,403 @@ +from time import time, sleep +from asyncio import sleep as asyncio_sleep, Lock + +from camoufox import DefaultAddons +from playwright.sync_api import BrowserContext, Playwright +from playwright.async_api import ( + BrowserContext as AsyncBrowserContext, + Playwright as AsyncPlaywright, +) +from camoufox.utils import ( + launch_options as generate_launch_options, + installed_verstr as camoufox_version, +) + +from scrapling.engines.toolbelt import ( + intercept_route, + async_intercept_route, + get_os_name, +) +from ._page import PageInfo, PagePool +from ._config_tools import _compiled_stealth_scripts +from ._validators import validate, PlaywrightConfig, CamoufoxConfig +from ._config_tools import _launch_kwargs, _context_kwargs +from scrapling.core._types import ( + Dict, + Optional, +) + +__ff_version_str__ = camoufox_version().split(".", 1)[0] + + +class SyncSession: + def __init__(self, max_pages: int = 1): + self.max_pages = max_pages + self.page_pool = PagePool(max_pages) + self.__max_wait_for_page = 60 + self.playwright: Optional[Playwright] = None + self.context: Optional[BrowserContext] = None + self._closed = False + + def _get_page(self) -> PageInfo: # pragma: no cover + """Get a new page to use""" + + # Close all finished pages to ensure clean state + self.page_pool.close_all_finished_pages() + + # If we're at max capacity after cleanup, wait for busy pages to finish + if self.page_pool.pages_count >= self.max_pages: + start_time = time() + while time() - start_time < self.__max_wait_for_page: + # Wait for any pages to finish, then clean them up + sleep(0.05) + self.page_pool.close_all_finished_pages() + if self.page_pool.pages_count < self.max_pages: + break + else: + raise TimeoutError( + f"No pages finished to clear place in the pool within the {self.__max_wait_for_page}s timeout period" + ) + + page = self.context.new_page() + timeout = getattr(self, "timeout", 30000) + page.set_default_navigation_timeout(timeout) + page.set_default_timeout(timeout) + if getattr(self, "extra_headers", False): + page.set_extra_http_headers(getattr(self, "extra_headers")) + + if getattr(self, "disable_resources", False): + page.route("**/*", intercept_route) + + if getattr(self, "stealth", False): + for script in _compiled_stealth_scripts(): + page.add_init_script(script=script) + + return self.page_pool.add_page(page) + + def get_pool_stats(self) -> Dict[str, int]: + """Get statistics about the current page pool""" + return { + "total_pages": self.page_pool.pages_count, + "busy_pages": self.page_pool.busy_count, + "max_pages": self.max_pages, + } + + +class AsyncSession(SyncSession): + def __init__(self, max_pages: int = 1): + super().__init__(max_pages) + self.playwright: Optional[AsyncPlaywright] = None + self.context: Optional[AsyncBrowserContext] = None + self._lock = Lock() + + async def _get_page(self) -> PageInfo: # pragma: no cover + """Get a new page to use""" + async with self._lock: + # Close all finished pages to ensure clean state + await self.page_pool.aclose_all_finished_pages() + + # If we're at max capacity after cleanup, wait for busy pages to finish + if self.page_pool.pages_count >= self.max_pages: + start_time = time() + while time() - start_time < self.__max_wait_for_page: + # Wait for any pages to finish, then clean them up + await asyncio_sleep(0.05) + await self.page_pool.aclose_all_finished_pages() + if self.page_pool.pages_count < self.max_pages: + break + else: + raise TimeoutError( + f"No pages finished to clear place in the pool within the {self.__max_wait_for_page}s timeout period" + ) + + page = await self.context.new_page() + timeout = getattr(self, "timeout", 30000) + page.set_default_navigation_timeout(timeout) + page.set_default_timeout(timeout) + if getattr(self, "extra_headers", False): + await page.set_extra_http_headers(getattr(self, "extra_headers")) + + if getattr(self, "disable_resources", False): + await page.route("**/*", async_intercept_route) + + if getattr(self, "stealth", False): + for script in _compiled_stealth_scripts(): + await page.add_init_script(script=script) + + return self.page_pool.add_page(page) + + +class DynamicSessionMixin: + def __validate__( + self, + __max_pages, + headless, + google_search, + hide_canvas, + disable_webgl, + real_chrome, + stealth, + wait, + page_action, + proxy, + locale, + extra_headers, + useragent, + cdp_url, + timeout, + disable_resources, + wait_selector, + init_script, + cookies, + network_idle, + wait_selector_state, + selector_config, + ): + params = { + "max_pages": __max_pages, + "headless": headless, + "google_search": google_search, + "hide_canvas": hide_canvas, + "disable_webgl": disable_webgl, + "real_chrome": real_chrome, + "stealth": stealth, + "wait": wait, + "page_action": page_action, + "proxy": proxy, + "locale": locale, + "extra_headers": extra_headers, + "useragent": useragent, + "timeout": timeout, + "selector_config": selector_config, + "disable_resources": disable_resources, + "wait_selector": wait_selector, + "init_script": init_script, + "cookies": cookies, + "network_idle": network_idle, + "wait_selector_state": wait_selector_state, + "cdp_url": cdp_url, + } + config = validate(params, PlaywrightConfig) + + self.max_pages = config.max_pages + self.headless = config.headless + self.hide_canvas = config.hide_canvas + self.disable_webgl = config.disable_webgl + self.real_chrome = config.real_chrome + self.stealth = config.stealth + self.google_search = config.google_search + self.wait = config.wait + self.proxy = config.proxy + self.locale = config.locale + self.extra_headers = config.extra_headers + self.useragent = config.useragent + self.timeout = config.timeout + self.cookies = config.cookies + self.disable_resources = config.disable_resources + self.cdp_url = config.cdp_url + self.network_idle = config.network_idle + self.wait_selector = config.wait_selector + self.init_script = config.init_script + self.wait_selector_state = config.wait_selector_state + self.selector_config = config.selector_config + self.page_action = config.page_action + self._headers_keys = ( + set(map(str.lower, self.extra_headers.keys())) + if self.extra_headers + else set() + ) + self.__initiate_browser_options__() + + def __initiate_browser_options__(self): + if not self.cdp_url: + # `launch_options` is used with persistent context + self.launch_options = dict( + _launch_kwargs( + self.headless, + self.proxy, + self.locale, + tuple(self.extra_headers.items()) + if self.extra_headers + else tuple(), + self.useragent, + self.real_chrome, + self.stealth, + self.hide_canvas, + self.disable_webgl, + ) + ) + self.launch_options["extra_http_headers"] = dict( + self.launch_options["extra_http_headers"] + ) + self.launch_options["proxy"] = dict(self.launch_options["proxy"]) or None + self.context_options = dict() + else: + # while `context_options` is left to be used when cdp mode is enabled + self.launch_options = dict() + self.context_options = dict( + _context_kwargs( + self.proxy, + self.locale, + tuple(self.extra_headers.items()) + if self.extra_headers + else tuple(), + self.useragent, + self.stealth, + ) + ) + self.context_options["extra_http_headers"] = dict( + self.context_options["extra_http_headers"] + ) + self.context_options["proxy"] = dict(self.context_options["proxy"]) or None + + +class StealthySessionMixin: + def __validate__( + self, + max_pages, + headless, + block_images, + disable_resources, + block_webrtc, + allow_webgl, + network_idle, + humanize, + solve_cloudflare, + wait, + timeout, + page_action, + wait_selector, + init_script, + addons, + wait_selector_state, + cookies, + google_search, + extra_headers, + proxy, + os_randomize, + disable_ads, + geoip, + selector_config, + additional_args, + ): + params = { + "max_pages": max_pages, + "headless": headless, + "block_images": block_images, + "disable_resources": disable_resources, + "block_webrtc": block_webrtc, + "allow_webgl": allow_webgl, + "network_idle": network_idle, + "humanize": humanize, + "solve_cloudflare": solve_cloudflare, + "wait": wait, + "timeout": timeout, + "page_action": page_action, + "wait_selector": wait_selector, + "init_script": init_script, + "addons": addons, + "wait_selector_state": wait_selector_state, + "cookies": cookies, + "google_search": google_search, + "extra_headers": extra_headers, + "proxy": proxy, + "os_randomize": os_randomize, + "disable_ads": disable_ads, + "geoip": geoip, + "selector_config": selector_config, + "additional_args": additional_args, + } + config = validate(params, CamoufoxConfig) + + self.max_pages = config.max_pages + self.headless = config.headless + self.block_images = config.block_images + self.disable_resources = config.disable_resources + self.block_webrtc = config.block_webrtc + self.allow_webgl = config.allow_webgl + self.network_idle = config.network_idle + self.humanize = config.humanize + self.solve_cloudflare = config.solve_cloudflare + self.wait = config.wait + self.timeout = config.timeout + self.page_action = config.page_action + self.wait_selector = config.wait_selector + self.init_script = config.init_script + self.addons = config.addons + self.wait_selector_state = config.wait_selector_state + self.cookies = config.cookies + self.google_search = config.google_search + self.extra_headers = config.extra_headers + self.proxy = config.proxy + self.os_randomize = config.os_randomize + self.disable_ads = config.disable_ads + self.geoip = config.geoip + self.selector_config = config.selector_config + self.additional_args = config.additional_args + self.selector_config = config.selector_config + self.page_action = config.page_action + self._headers_keys = ( + set(map(str.lower, self.extra_headers.keys())) + if self.extra_headers + else set() + ) + self.__initiate_browser_options__() + + def __initiate_browser_options__(self): + """Initiate browser options.""" + self.launch_options = generate_launch_options( + **{ + "geoip": self.geoip, + "proxy": dict(self.proxy) if self.proxy else self.proxy, + "addons": self.addons, + "exclude_addons": [] if self.disable_ads else [DefaultAddons.UBO], + "headless": self.headless, + "humanize": True if self.solve_cloudflare else self.humanize, + "i_know_what_im_doing": True, # To turn warnings off with the user configurations + "allow_webgl": self.allow_webgl, + "block_webrtc": self.block_webrtc, + "block_images": self.block_images, # Careful! it makes some websites don't finish loading at all like stackoverflow even in headful mode. + "os": None if self.os_randomize else get_os_name(), + "user_data_dir": "", + "ff_version": __ff_version_str__, + "firefox_user_prefs": { + # This is what enabling `enable_cache` does internally, so we do it from here instead + "browser.sessionhistory.max_entries": 10, + "browser.sessionhistory.max_total_viewers": -1, + "browser.cache.memory.enable": True, + "browser.cache.disk_cache_ssl": True, + "browser.cache.disk.smart_size.enabled": True, + }, + **self.additional_args, + } + ) + + @staticmethod + def _detect_cloudflare(page_content: str) -> str | None: + """ + Detect the type of Cloudflare challenge present in the provided page content. + + This function analyzes the given page content to identify whether a specific + type of Cloudflare challenge is present. It checks for three predefined + challenge types: non-interactive, managed, and interactive. If a challenge + type is detected, it returns the corresponding type as a string. If no + challenge type is detected, it returns None. + + Args: + page_content (str): The content of the page to analyze for Cloudflare + challenge types. + + Returns: + str: A string representing the detected Cloudflare challenge type, if + found. Returns None if no challenge matches. + """ + challenge_types = ( + "non-interactive", + "managed", + "interactive", + ) + for ctype in challenge_types: + if f"cType: '{ctype}'" in page_content: + return ctype + + return None diff --git a/scrapling/engines/_browsers/_camoufox.py b/scrapling/engines/_browsers/_camoufox.py index 5afccb1..cd814c1 100644 --- a/scrapling/engines/_browsers/_camoufox.py +++ b/scrapling/engines/_browsers/_camoufox.py @@ -1,17 +1,8 @@ -from time import time, sleep from re import compile as re_compile -from asyncio import sleep as asyncio_sleep, Lock -from camoufox import DefaultAddons -from camoufox.utils import ( - launch_options as generate_launch_options, - installed_verstr as camoufox_version, -) from playwright.sync_api import ( Response as SyncPlaywrightResponse, sync_playwright, - BrowserContext, - Playwright, Locator, Page, ) @@ -24,9 +15,8 @@ from playwright.async_api import ( Page as async_Page, ) +from ._base import SyncSession, AsyncSession, StealthySessionMixin from scrapling.core.utils import log -from ._page import PageInfo, PagePool -from ._validators import validate, CamoufoxConfig from scrapling.core._types import ( Dict, List, @@ -37,17 +27,13 @@ from scrapling.core._types import ( from scrapling.engines.toolbelt import ( Response, ResponseFactory, - async_intercept_route, generate_convincing_referer, - get_os_name, - intercept_route, ) __CF_PATTERN__ = re_compile("challenges.cloudflare.com/cdn-cgi/challenge-platform/.*") -__ff_version_str__ = camoufox_version().split(".", 1)[0] -class StealthySession: +class StealthySession(StealthySessionMixin, SyncSession): """A Stealthy session manager with page pooling.""" __slots__ = ( @@ -87,7 +73,7 @@ class StealthySession: def __init__( self, - max_pages: int = 1, + __max_pages: int = 1, headless: bool = True, # noqa: F821 block_images: bool = False, disable_resources: bool = False, @@ -141,107 +127,38 @@ class StealthySession: :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name. :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only. - :param max_pages: The maximum number of tabs to be opened at the same time. It will be used in rotation through a PagePool. :param selector_config: The arguments that will be passed in the end while creating the final Selector's class. :param additional_args: Additional arguments to be passed to Camoufox as additional settings, and it takes higher priority than Scrapling's settings. """ - params = { - "max_pages": max_pages, - "headless": headless, - "block_images": block_images, - "disable_resources": disable_resources, - "block_webrtc": block_webrtc, - "allow_webgl": allow_webgl, - "network_idle": network_idle, - "humanize": humanize, - "solve_cloudflare": solve_cloudflare, - "wait": wait, - "timeout": timeout, - "page_action": page_action, - "wait_selector": wait_selector, - "init_script": init_script, - "addons": addons, - "wait_selector_state": wait_selector_state, - "cookies": cookies, - "google_search": google_search, - "extra_headers": extra_headers, - "proxy": proxy, - "os_randomize": os_randomize, - "disable_ads": disable_ads, - "geoip": geoip, - "selector_config": selector_config, - "additional_args": additional_args, - } - config = validate(params, CamoufoxConfig) - - self.max_pages = config.max_pages - self.headless = config.headless - self.block_images = config.block_images - self.disable_resources = config.disable_resources - self.block_webrtc = config.block_webrtc - self.allow_webgl = config.allow_webgl - self.network_idle = config.network_idle - self.humanize = config.humanize - self.solve_cloudflare = config.solve_cloudflare - self.wait = config.wait - self.timeout = config.timeout - self.page_action = config.page_action - self.wait_selector = config.wait_selector - self.init_script = config.init_script - self.addons = config.addons - self.wait_selector_state = config.wait_selector_state - self.cookies = config.cookies - self.google_search = config.google_search - self.extra_headers = config.extra_headers - self.proxy = config.proxy - self.os_randomize = config.os_randomize - self.disable_ads = config.disable_ads - self.geoip = config.geoip - self.selector_config = config.selector_config - self.additional_args = config.additional_args - - self.playwright: Optional[Playwright] = None - self.context: Optional[BrowserContext] = None - self.page_pool = PagePool(self.max_pages) - self._closed = False - self.selector_config = config.selector_config - self.page_action = config.page_action - self._headers_keys = ( - set(map(str.lower, self.extra_headers.keys())) - if self.extra_headers - else set() - ) - self.__initiate_browser_options__() - - def __initiate_browser_options__(self): - """Initiate browser options.""" - self.launch_options = generate_launch_options( - **{ - "geoip": self.geoip, - "proxy": dict(self.proxy) if self.proxy else self.proxy, - "addons": self.addons, - "exclude_addons": [] if self.disable_ads else [DefaultAddons.UBO], - "headless": self.headless, - "humanize": True if self.solve_cloudflare else self.humanize, - "i_know_what_im_doing": True, # To turn warnings off with the user configurations - "allow_webgl": self.allow_webgl, - "block_webrtc": self.block_webrtc, - "block_images": self.block_images, # Careful! it makes some websites don't finish loading at all like stackoverflow even in headful mode. - "os": None if self.os_randomize else get_os_name(), - "user_data_dir": "", - "ff_version": __ff_version_str__, - "firefox_user_prefs": { - # This is what enabling `enable_cache` does internally, so we do it from here instead - "browser.sessionhistory.max_entries": 10, - "browser.sessionhistory.max_total_viewers": -1, - "browser.cache.memory.enable": True, - "browser.cache.disk_cache_ssl": True, - "browser.cache.disk.smart_size.enabled": True, - }, - **self.additional_args, - } + self.__validate__( + __max_pages, + headless, + block_images, + disable_resources, + block_webrtc, + allow_webgl, + network_idle, + humanize, + solve_cloudflare, + wait, + timeout, + page_action, + wait_selector, + init_script, + addons, + wait_selector_state, + cookies, + google_search, + extra_headers, + proxy, + os_randomize, + disable_ads, + geoip, + selector_config, + additional_args, ) + super().__init__(max_pages=self.max_pages) def __create__(self): """Create a browser for this instance and context.""" @@ -284,68 +201,6 @@ class StealthySession: self._closed = True - def _get_or_create_page(self) -> PageInfo: # pragma: no cover - """Get an available page or create a new one""" - # Try to get a ready page first - page_info = self.page_pool.get_ready_page() - if page_info: - return page_info - - # Create a new page if under limit - if self.page_pool.pages_count < self.max_pages: - page = self.context.new_page() - page.set_default_navigation_timeout(self.timeout) - page.set_default_timeout(self.timeout) - if self.extra_headers: - page.set_extra_http_headers(self.extra_headers) - - if self.disable_resources: - page.route("**/*", intercept_route) - - return self.page_pool.add_page(page) - - # Wait for a page to become available - max_wait = 30 - start_time = time() - - while time() - start_time < max_wait: - page_info = self.page_pool.get_ready_page() - if page_info: - return page_info - sleep(0.05) - - raise TimeoutError("No pages available within timeout period") - - @staticmethod - def _detect_cloudflare(page_content): - """ - Detect the type of Cloudflare challenge present in the provided page content. - - This function analyzes the given page content to identify whether a specific - type of Cloudflare challenge is present. It checks for three predefined - challenge types: non-interactive, managed, and interactive. If a challenge - type is detected, it returns the corresponding type as a string. If no - challenge type is detected, it returns None. - - Args: - page_content (str): The content of the page to analyze for Cloudflare - challenge types. - - Returns: - str: A string representing the detected Cloudflare challenge type, if - found. Returns None if no challenge matches. - """ - challenge_types = ( - "non-interactive", - "managed", - "interactive", - ) - for ctype in challenge_types: - if f"cType: '{ctype}'" in page_content: - return ctype - - return None - def _solve_cloudflare(self, page: Page) -> None: # pragma: no cover """Solve the cloudflare challenge displayed on the playwright page passed @@ -416,7 +271,7 @@ class StealthySession: ): final_response = finished_response - page_info = self._get_or_create_page() + page_info = self._get_page() page_info.mark_busy(url=url) try: # pragma: no cover @@ -462,8 +317,8 @@ class StealthySession: page_info.page, first_response, final_response, self.selector_config ) - # Mark the page as ready for next use - page_info.mark_ready() + # Mark the page as finished for next use + page_info.mark_finished() return response @@ -471,17 +326,8 @@ class StealthySession: page_info.mark_error() raise e - def get_pool_stats(self) -> Dict[str, int]: - """Get statistics about the current page pool""" - return { - "total_pages": self.page_pool.pages_count, - "ready_pages": self.page_pool.ready_count, - "busy_pages": self.page_pool.busy_count, - "max_pages": self.max_pages, - } - -class AsyncStealthySession(StealthySession): +class AsyncStealthySession(StealthySessionMixin, AsyncSession): """A Stealthy session manager with page pooling.""" def __init__( @@ -544,7 +390,7 @@ class AsyncStealthySession(StealthySession): :param selector_config: The arguments that will be passed in the end while creating the final Selector's class. :param additional_args: Additional arguments to be passed to Camoufox as additional settings, and it takes higher priority than Scrapling's settings. """ - super().__init__( + self.__validate__( max_pages, headless, block_images, @@ -571,11 +417,7 @@ class AsyncStealthySession(StealthySession): selector_config, additional_args, ) - self.playwright: Optional[AsyncPlaywright] = None - self.context: Optional[AsyncBrowserContext] = None - self._lock = Lock() - self.__enter__ = None - self.__exit__ = None + super().__init__(max_pages=self.max_pages) async def __create__(self): """Create a browser for this instance and context.""" @@ -618,39 +460,6 @@ class AsyncStealthySession(StealthySession): self._closed = True - async def _get_or_create_page(self) -> PageInfo: - """Get an available page or create a new one""" - async with self._lock: - # Try to get a ready page first - page_info = self.page_pool.get_ready_page() - if page_info: - return page_info - - # Create a new page if under limit - if self.page_pool.pages_count < self.max_pages: - page = await self.context.new_page() - page.set_default_navigation_timeout(self.timeout) - page.set_default_timeout(self.timeout) - if self.extra_headers: - await page.set_extra_http_headers(self.extra_headers) - - if self.disable_resources: - await page.route("**/*", async_intercept_route) - - return self.page_pool.add_page(page) - - # Wait for a page to become available - max_wait = 30 - start_time = time() - - while time() - start_time < max_wait: # pragma: no cover - page_info = self.page_pool.get_ready_page() - if page_info: - return page_info - await asyncio_sleep(0.05) - - raise TimeoutError("No pages available within timeout period") - async def _solve_cloudflare(self, page: async_Page): """Solve the cloudflare challenge displayed on the playwright page passed. The async version @@ -723,7 +532,7 @@ class AsyncStealthySession(StealthySession): ): final_response = finished_response - page_info = await self._get_or_create_page() + page_info = await self._get_page() page_info.mark_busy(url=url) try: @@ -771,8 +580,8 @@ class AsyncStealthySession(StealthySession): page_info.page, first_response, final_response, self.selector_config ) - # Mark the page as ready for next use - page_info.mark_ready() + # Mark the page as finished for next use + page_info.mark_finished() return response diff --git a/scrapling/engines/_browsers/_controllers.py b/scrapling/engines/_browsers/_controllers.py index 85db198..ef3b906 100644 --- a/scrapling/engines/_browsers/_controllers.py +++ b/scrapling/engines/_browsers/_controllers.py @@ -1,10 +1,6 @@ -from time import time, sleep -from asyncio import sleep as asyncio_sleep, Lock - from playwright.sync_api import ( Response as SyncPlaywrightResponse, sync_playwright, - BrowserContext, Playwright, Locator, ) @@ -21,9 +17,7 @@ from rebrowser_playwright.async_api import ( ) from scrapling.core.utils import log -from ._page import PageInfo, PagePool -from ._validators import validate, PlaywrightConfig -from ._config_tools import _compiled_stealth_scripts, _launch_kwargs, _context_kwargs +from ._base import SyncSession, AsyncSession, DynamicSessionMixin from scrapling.core._types import ( Dict, List, @@ -35,12 +29,10 @@ from scrapling.engines.toolbelt import ( Response, ResponseFactory, generate_convincing_referer, - intercept_route, - async_intercept_route, ) -class DynamicSession: +class DynamicSession(DynamicSessionMixin, SyncSession): """A Browser session manager with page pooling.""" __slots__ = ( @@ -127,108 +119,31 @@ class DynamicSession: :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only. :param selector_config: The arguments that will be passed in the end while creating the final Selector's class. """ - - params = { - "max_pages": __max_pages, - "headless": headless, - "google_search": google_search, - "hide_canvas": hide_canvas, - "disable_webgl": disable_webgl, - "real_chrome": real_chrome, - "stealth": stealth, - "wait": wait, - "page_action": page_action, - "proxy": proxy, - "locale": locale, - "extra_headers": extra_headers, - "useragent": useragent, - "timeout": timeout, - "selector_config": selector_config, - "disable_resources": disable_resources, - "wait_selector": wait_selector, - "init_script": init_script, - "cookies": cookies, - "network_idle": network_idle, - "wait_selector_state": wait_selector_state, - "cdp_url": cdp_url, - } - config = validate(params, PlaywrightConfig) - - self.max_pages = config.max_pages - self.headless = config.headless - self.hide_canvas = config.hide_canvas - self.disable_webgl = config.disable_webgl - self.real_chrome = config.real_chrome - self.stealth = config.stealth - self.google_search = config.google_search - self.wait = config.wait - self.proxy = config.proxy - self.locale = config.locale - self.extra_headers = config.extra_headers - self.useragent = config.useragent - self.timeout = config.timeout - self.cookies = config.cookies - self.disable_resources = config.disable_resources - self.cdp_url = config.cdp_url - self.network_idle = config.network_idle - self.wait_selector = config.wait_selector - self.init_script = config.init_script - self.wait_selector_state = config.wait_selector_state - - self.playwright: Optional[Playwright] = None - self.context: Optional[BrowserContext] = None - self.page_pool = PagePool(self.max_pages) - self._closed = False - self.selector_config = config.selector_config - self.page_action = config.page_action - self._headers_keys = ( - set(map(str.lower, self.extra_headers.keys())) - if self.extra_headers - else set() + self.__validate__( + __max_pages, + headless, + google_search, + hide_canvas, + disable_webgl, + real_chrome, + stealth, + wait, + page_action, + proxy, + locale, + extra_headers, + useragent, + cdp_url, + timeout, + disable_resources, + wait_selector, + init_script, + cookies, + network_idle, + wait_selector_state, + selector_config, ) - self.__initiate_browser_options__() - - def __initiate_browser_options__(self): - if not self.cdp_url: - # `launch_options` is used with persistent context - self.launch_options = dict( - _launch_kwargs( - self.headless, - self.proxy, - self.locale, - tuple(self.extra_headers.items()) - if self.extra_headers - else tuple(), - self.useragent, - self.real_chrome, - self.stealth, - self.hide_canvas, - self.disable_webgl, - ) - ) - self.launch_options["extra_http_headers"] = dict( - self.launch_options["extra_http_headers"] - ) - self.launch_options["proxy"] = dict(self.launch_options["proxy"]) or None - self.context_options = dict() - else: - # while `context_options` is left to be used when cdp mode is enabled - self.launch_options = dict() - self.context_options = dict( - _context_kwargs( - self.proxy, - self.locale, - tuple(self.extra_headers.items()) - if self.extra_headers - else tuple(), - self.useragent, - self.stealth, - ) - ) - self.context_options["extra_http_headers"] = dict( - self.context_options["extra_http_headers"] - ) - self.context_options["proxy"] = dict(self.context_options["proxy"]) or None + super().__init__(max_pages=self.max_pages) def __create__(self): """Create a browser for this instance and context.""" @@ -237,7 +152,7 @@ class DynamicSession: # Because rebrowser_playwright doesn't play well with real browsers sync_context = sync_playwright - self.playwright = sync_context().start() + self.playwright: Playwright = sync_context().start() if self.cdp_url: # pragma: no cover self.context = self.playwright.chromium.connect_over_cdp( @@ -280,43 +195,10 @@ class DynamicSession: self._closed = True - def _get_or_create_page(self) -> PageInfo: # pragma: no cover - """Get an available page or create a new one""" - # Try to get a ready page first - page_info = self.page_pool.get_ready_page() - if page_info: - return page_info - - # Create a new page if under limit - if self.page_pool.pages_count < self.max_pages: - page = self.context.new_page() - page.set_default_navigation_timeout(self.timeout) - page.set_default_timeout(self.timeout) - if self.extra_headers: - page.set_extra_http_headers(self.extra_headers) - - if self.disable_resources: - page.route("**/*", intercept_route) - - if self.stealth: - for script in _compiled_stealth_scripts(): - page.add_init_script(script=script) - - return self.page_pool.add_page(page) - - # Wait for a page to become available - max_wait = 30 - start_time = time() - - while time() - start_time < max_wait: - page_info = self.page_pool.get_ready_page() - if page_info: - return page_info - sleep(0.05) - - raise TimeoutError("No pages available within timeout period") - - def fetch(self, url: str) -> Response: + def fetch( + self, + url: str, + ) -> Response: """Opens up the browser and do your request based on your chosen options. :param url: The Target url. @@ -340,7 +222,7 @@ class DynamicSession: ): final_response = finished_response - page_info = self._get_or_create_page() + page_info = self._get_page() page_info.mark_busy(url=url) try: # pragma: no cover @@ -380,8 +262,8 @@ class DynamicSession: page_info.page, first_response, final_response, self.selector_config ) - # Mark the page as ready for next use - page_info.mark_ready() + # Mark the page as finished for next use + page_info.mark_finished() return response @@ -389,17 +271,8 @@ class DynamicSession: page_info.mark_error() raise e - def get_pool_stats(self) -> Dict[str, int]: - """Get statistics about the current page pool""" - return { - "total_pages": self.page_pool.pages_count, - "ready_pages": self.page_pool.ready_count, - "busy_pages": self.page_pool.busy_count, - "max_pages": self.max_pages, - } - -class AsyncDynamicSession(DynamicSession): +class AsyncDynamicSession(DynamicSessionMixin, AsyncSession): """An async Browser session manager with page pooling, it's using a persistent browser Context by default with a temporary user profile directory.""" def __init__( @@ -455,7 +328,7 @@ class AsyncDynamicSession(DynamicSession): :param selector_config: The arguments that will be passed in the end while creating the final Selector's class. """ - super().__init__( + self.__validate__( max_pages, headless, google_search, @@ -479,12 +352,7 @@ class AsyncDynamicSession(DynamicSession): wait_selector_state, selector_config, ) - - self.playwright: Optional[AsyncPlaywright] = None - self.context: Optional[AsyncBrowserContext] = None - self._lock = Lock() - self.__enter__ = None - self.__exit__ = None + super().__init__(max_pages=self.max_pages) async def __create__(self): """Create a browser for this instance and context.""" @@ -541,43 +409,6 @@ class AsyncDynamicSession(DynamicSession): self._closed = True - async def _get_or_create_page(self) -> PageInfo: - """Get an available page or create a new one""" - async with self._lock: - # Try to get a ready page first - page_info = self.page_pool.get_ready_page() - if page_info: - return page_info - - # Create a new page if under limit - if self.page_pool.pages_count < self.max_pages: - page = await self.context.new_page() - page.set_default_navigation_timeout(self.timeout) - page.set_default_timeout(self.timeout) - if self.extra_headers: - await page.set_extra_http_headers(self.extra_headers) - - if self.disable_resources: - await page.route("**/*", async_intercept_route) - - if self.stealth: - for script in _compiled_stealth_scripts(): - await page.add_init_script(script=script) - - return self.page_pool.add_page(page) - - # Wait for a page to become available - max_wait = 30 # seconds - start_time = time() - - while time() - start_time < max_wait: # pragma: no cover - page_info = self.page_pool.get_ready_page() - if page_info: - return page_info - await asyncio_sleep(0.05) - - raise TimeoutError("No pages available within timeout period") - async def fetch(self, url: str) -> Response: """Opens up the browser and do your request based on your chosen options. @@ -602,7 +433,7 @@ class AsyncDynamicSession(DynamicSession): ): final_response = finished_response - page_info = await self._get_or_create_page() + page_info = await self._get_page() page_info.mark_busy(url=url) try: @@ -642,8 +473,8 @@ class AsyncDynamicSession(DynamicSession): page_info.page, first_response, final_response, self.selector_config ) - # Mark the page as ready for next use - page_info.mark_ready() + # Mark the page as finished for next use + page_info.mark_finished() return response diff --git a/scrapling/engines/_browsers/_page.py b/scrapling/engines/_browsers/_page.py index ec418d0..61dd51b 100644 --- a/scrapling/engines/_browsers/_page.py +++ b/scrapling/engines/_browsers/_page.py @@ -6,7 +6,9 @@ from playwright.async_api import Page as AsyncPage from scrapling.core._types import Optional, List, Literal -PageState = Literal["ready", "busy", "error"] # States that a page can be in +PageState = Literal[ + "finished", "ready", "busy", "error" +] # States that a page can be in @dataclass @@ -23,9 +25,9 @@ class PageInfo: self.state = "busy" self.url = url - def mark_ready(self): - """Mark the page as ready for new requests""" - self.state = "ready" + def mark_finished(self): + """Mark the page as finished for new requests""" + self.state = "finished" self.url = "" def mark_error(self): @@ -62,24 +64,16 @@ class PagePool: self.pages.append(page_info) return page_info - def get_ready_page(self) -> Optional[PageInfo]: - """Get a page that's ready for use""" - with self._lock: - for page_info in self.pages: - if page_info.state == "ready": - return page_info - return None - @property def pages_count(self) -> int: """Get the total number of pages""" return len(self.pages) @property - def ready_count(self) -> int: - """Get the number of ready pages""" + def finished_count(self) -> int: + """Get the number of finished pages""" with self._lock: - return sum(1 for p in self.pages if p.state == "ready") + return sum(1 for p in self.pages if p.state == "finished") @property def busy_count(self) -> int: @@ -91,3 +85,33 @@ class PagePool: """Remove pages in error state""" with self._lock: self.pages = [p for p in self.pages if p.state != "error"] + + def close_all_finished_pages(self): + """Close all pages in finished state and remove them from the pool""" + with self._lock: + pages_to_remove = [] + for page_info in self.pages: + if page_info.state == "finished": + try: + page_info.page.close() + except Exception: + pass + pages_to_remove.append(page_info) + + for page_info in pages_to_remove: + self.pages.remove(page_info) + + async def aclose_all_finished_pages(self): + """Async version: Close all pages in finished state and remove them from the pool""" + with self._lock: + pages_to_remove = [] + for page_info in self.pages: + if page_info.state == "finished": + try: + await page_info.page.close() + except Exception: + pass + pages_to_remove.append(page_info) + + for page_info in pages_to_remove: + self.pages.remove(page_info) diff --git a/scrapling/fetchers.py b/scrapling/fetchers.py index c1ce8f7..3e39ba2 100644 --- a/scrapling/fetchers.py +++ b/scrapling/fetchers.py @@ -118,7 +118,6 @@ class StealthyFetcher(BaseFetcher): with StealthySession( wait=wait, - max_pages=1, proxy=proxy, geoip=geoip, addons=addons,