from time import time from asyncio import sleep as asyncio_sleep, Lock from camoufox import DefaultAddons from playwright.sync_api._generated import Page from playwright.sync_api import ( Frame, BrowserContext, Playwright, Response as SyncPlaywrightResponse, ) from playwright.async_api._generated import Page as AsyncPage from playwright.async_api import ( Frame as AsyncFrame, Playwright as AsyncPlaywright, Response as AsyncPlaywrightResponse, BrowserContext as AsyncBrowserContext, ) from playwright._impl._errors import Error as PlaywrightError from camoufox.pkgman import installed_verstr as camoufox_version from camoufox.utils import launch_options as generate_launch_options from ._page import PageInfo, PagePool from scrapling.parser import Selector from scrapling.core._types import Any, cast, Dict, List, Optional, Callable, TYPE_CHECKING from scrapling.engines.toolbelt.fingerprints import get_os_name from ._validators import validate, PlaywrightConfig, CamoufoxConfig from ._config_tools import _compiled_stealth_scripts, _launch_kwargs, _context_kwargs from scrapling.engines.toolbelt.navigation import intercept_route, async_intercept_route __ff_version_str__ = camoufox_version().split(".", 1)[0] class SyncSession: def __init__(self, max_pages: int = 1): self.max_pages = max_pages self.page_pool = PagePool(max_pages) self._max_wait_for_page = 60 self.playwright: Playwright | Any = None self.context: BrowserContext | Any = None self._closed = False def start(self): pass def close(self): # pragma: no cover """Close all resources""" if self._closed: return if self.context: self.context.close() self.context = None if self.playwright: self.playwright.stop() self.playwright = None # pyright: ignore self._closed = True def __enter__(self): self.start() return self def __exit__(self, exc_type, exc_val, exc_tb): self.close() def _get_page( self, timeout: int | float, extra_headers: Optional[Dict[str, str]], disable_resources: bool, ) -> PageInfo[Page]: # pragma: no cover """Get a new page to use""" # No need to check if a page is available or not in sync code because the code blocked before reaching here till the page closed, ofc. assert self.context is not None, "Browser context not initialized" page = self.context.new_page() page.set_default_navigation_timeout(timeout) page.set_default_timeout(timeout) if extra_headers: page.set_extra_http_headers(extra_headers) if disable_resources: page.route("**/*", intercept_route) if getattr(self, "stealth", False): for script in _compiled_stealth_scripts(): page.add_init_script(script=script) page_info = self.page_pool.add_page(page) page_info.mark_busy() return page_info def get_pool_stats(self) -> Dict[str, int]: """Get statistics about the current page pool""" return { "total_pages": self.page_pool.pages_count, "busy_pages": self.page_pool.busy_count, "max_pages": self.max_pages, } @staticmethod def _wait_for_networkidle(page: Page | Frame, timeout: Optional[int] = None): """Wait for the page to become idle (no network activity) even if there are never-ending requests.""" try: page.wait_for_load_state("networkidle", timeout=timeout) except (PlaywrightError, Exception): pass def _wait_for_page_stability(self, page: Page | Frame, load_dom: bool, network_idle: bool): page.wait_for_load_state(state="load") if load_dom: page.wait_for_load_state(state="domcontentloaded") if network_idle: self._wait_for_networkidle(page) @staticmethod def _create_response_handler(page_info: PageInfo[Page], response_container: List) -> Callable: """Create a response handler that captures the final navigation response. :param page_info: The PageInfo object containing the page :param response_container: A list to store the final response (mutable container) :return: A callback function for page.on("response", ...) """ def handle_response(finished_response: SyncPlaywrightResponse): if ( finished_response.request.resource_type == "document" and finished_response.request.is_navigation_request() and finished_response.request.frame == page_info.page.main_frame ): response_container[0] = finished_response return handle_response class AsyncSession: def __init__(self, max_pages: int = 1): self.max_pages = max_pages self.page_pool = PagePool(max_pages) self._max_wait_for_page = 60 self.playwright: AsyncPlaywright | Any = None self.context: AsyncBrowserContext | Any = None self._closed = False self._lock = Lock() async def start(self): pass async def close(self): """Close all resources""" if self._closed: # pragma: no cover return if self.context: await self.context.close() self.context = None # pyright: ignore if self.playwright: await self.playwright.stop() self.playwright = None # pyright: ignore self._closed = True async def __aenter__(self): await self.start() return self async def __aexit__(self, exc_type, exc_val, exc_tb): await self.close() async def _get_page( self, timeout: int | float, extra_headers: Optional[Dict[str, str]], disable_resources: bool, ) -> PageInfo[AsyncPage]: # pragma: no cover """Get a new page to use""" if TYPE_CHECKING: assert self.context is not None, "Browser context not initialized" async with self._lock: # If we're at max capacity after cleanup, wait for busy pages to finish if self.page_pool.pages_count >= self.max_pages: start_time = time() while time() - start_time < self._max_wait_for_page: await asyncio_sleep(0.05) if self.page_pool.pages_count < self.max_pages: break else: raise TimeoutError( f"No pages finished to clear place in the pool within the {self._max_wait_for_page}s timeout period" ) page = await self.context.new_page() page.set_default_navigation_timeout(timeout) page.set_default_timeout(timeout) if extra_headers: await page.set_extra_http_headers(extra_headers) if disable_resources: await page.route("**/*", async_intercept_route) if getattr(self, "stealth", False): for script in _compiled_stealth_scripts(): await page.add_init_script(script=script) return self.page_pool.add_page(page) def get_pool_stats(self) -> Dict[str, int]: """Get statistics about the current page pool""" return { "total_pages": self.page_pool.pages_count, "busy_pages": self.page_pool.busy_count, "max_pages": self.max_pages, } @staticmethod async def _wait_for_networkidle(page: AsyncPage | AsyncFrame, timeout: Optional[int] = None): """Wait for the page to become idle (no network activity) even if there are never-ending requests.""" try: await page.wait_for_load_state("networkidle", timeout=timeout) except (PlaywrightError, Exception): pass async def _wait_for_page_stability(self, page: AsyncPage | AsyncFrame, load_dom: bool, network_idle: bool): await page.wait_for_load_state(state="load") if load_dom: await page.wait_for_load_state(state="domcontentloaded") if network_idle: await self._wait_for_networkidle(page) @staticmethod def _create_response_handler(page_info: PageInfo[AsyncPage], response_container: List) -> Callable: """Create an async response handler that captures the final navigation response. :param page_info: The PageInfo object containing the page :param response_container: A list to store the final response (mutable container) :return: A callback function for page.on("response", ...) """ async def handle_response(finished_response: AsyncPlaywrightResponse): if ( finished_response.request.resource_type == "document" and finished_response.request.is_navigation_request() and finished_response.request.frame == page_info.page.main_frame ): response_container[0] = finished_response return handle_response class DynamicSessionMixin: def __validate__(self, **params): if "__max_pages" in params: params["max_pages"] = params.pop("__max_pages") config = validate(params, model=PlaywrightConfig) self._max_pages = config.max_pages self._headless = config.headless self._hide_canvas = config.hide_canvas self._disable_webgl = config.disable_webgl self._real_chrome = config.real_chrome self._stealth = config.stealth self._google_search = config.google_search self._wait = config.wait self._proxy = config.proxy self._locale = config.locale self._extra_headers = config.extra_headers self._useragent = config.useragent self._timeout = config.timeout self._cookies = config.cookies self._disable_resources = config.disable_resources self._cdp_url = config.cdp_url self._network_idle = config.network_idle self._load_dom = config.load_dom self._wait_selector = config.wait_selector self._init_script = config.init_script self._wait_selector_state = config.wait_selector_state self._extra_flags = config.extra_flags self._selector_config = config.selector_config self._timezone_id = config.timezone_id self._additional_args = config.additional_args self._page_action = config.page_action self._user_data_dir = config.user_data_dir self._headers_keys = {header.lower() for header in self._extra_headers.keys()} if self._extra_headers else set() self.__initiate_browser_options__() def __initiate_browser_options__(self): if TYPE_CHECKING: assert isinstance(self._proxy, tuple) if not self._cdp_url: # `launch_options` is used with persistent context self.launch_options = dict( _launch_kwargs( self._headless, self._proxy, self._locale, tuple(self._extra_headers.items()) if self._extra_headers else tuple(), self._useragent, self._real_chrome, self._stealth, self._hide_canvas, self._disable_webgl, self._timezone_id, tuple(self._extra_flags) if self._extra_flags else tuple(), ) ) self.launch_options["extra_http_headers"] = dict(self.launch_options["extra_http_headers"]) self.launch_options["proxy"] = dict(self.launch_options["proxy"]) or None self.launch_options["user_data_dir"] = self._user_data_dir self.launch_options.update(cast(Dict, self._additional_args)) self.context_options = dict() else: # while `context_options` is left to be used when cdp mode is enabled self.launch_options = dict() self.context_options = dict( _context_kwargs( self._proxy, self._locale, tuple(self._extra_headers.items()) if self._extra_headers else tuple(), self._useragent, self._stealth, ) ) self.context_options["extra_http_headers"] = dict(self.context_options["extra_http_headers"]) self.context_options["proxy"] = dict(self.context_options["proxy"]) or None self.context_options.update(cast(Dict, self._additional_args)) class StealthySessionMixin: def __validate__(self, **params): if "__max_pages" in params: params["max_pages"] = params.pop("__max_pages") config: CamoufoxConfig = validate(params, model=CamoufoxConfig) self._max_pages = config.max_pages self._headless = config.headless self._block_images = config.block_images self._disable_resources = config.disable_resources self._block_webrtc = config.block_webrtc self._allow_webgl = config.allow_webgl self._network_idle = config.network_idle self._load_dom = config.load_dom self._humanize = config.humanize self._solve_cloudflare = config.solve_cloudflare self._wait = config.wait self._timeout = config.timeout self._page_action = config.page_action self._wait_selector = config.wait_selector self._init_script = config.init_script self._addons = config.addons self._wait_selector_state = config.wait_selector_state self._cookies = config.cookies self._google_search = config.google_search self._extra_headers = config.extra_headers self._proxy = config.proxy self._os_randomize = config.os_randomize self._disable_ads = config.disable_ads self._geoip = config.geoip self._selector_config = config.selector_config self._additional_args = config.additional_args self._user_data_dir = config.user_data_dir self._headers_keys = {header.lower() for header in self._extra_headers.keys()} if self._extra_headers else set() self.__initiate_browser_options__() def __initiate_browser_options__(self): """Initiate browser options.""" self.launch_options: Dict[str, Any] = generate_launch_options( **{ "geoip": self._geoip, "proxy": dict(self._proxy) if self._proxy and isinstance(self._proxy, tuple) else self._proxy, "addons": self._addons, "exclude_addons": [] if self._disable_ads else [DefaultAddons.UBO], "headless": self._headless, "humanize": True if self._solve_cloudflare else self._humanize, "i_know_what_im_doing": True, # To turn warnings off with the user configurations "allow_webgl": self._allow_webgl, "block_webrtc": self._block_webrtc, "block_images": self._block_images, # Careful! it makes some websites don't finish loading at all like stackoverflow even in headful mode. "os": None if self._os_randomize else get_os_name(), "user_data_dir": self._user_data_dir, "ff_version": __ff_version_str__, "firefox_user_prefs": { # This is what enabling `enable_cache` does internally, so we do it from here instead "browser.sessionhistory.max_entries": 10, "browser.sessionhistory.max_total_viewers": -1, "browser.cache.memory.enable": True, "browser.cache.disk_cache_ssl": True, "browser.cache.disk.smart_size.enabled": True, }, **cast(Dict, self._additional_args), } ) @staticmethod def _detect_cloudflare(page_content: str) -> str | None: """ Detect the type of Cloudflare challenge present in the provided page content. This function analyzes the given page content to identify whether a specific type of Cloudflare challenge is present. It checks for three predefined challenge types: non-interactive, managed, and interactive. If a challenge type is detected, it returns the corresponding type as a string. If no challenge type is detected, it returns None. Args: page_content (str): The content of the page to analyze for Cloudflare challenge types. Returns: str: A string representing the detected Cloudflare challenge type, if found. Returns None if no challenge matches. """ challenge_types = ( "non-interactive", "managed", "interactive", ) for ctype in challenge_types: if f"cType: '{ctype}'" in page_content: return ctype # Check if turnstile captcha is embedded inside the page (Usually inside a closed Shadow iframe) selector = Selector(content=page_content) if selector.css('script[src*="challenges.cloudflare.com/turnstile/v"]'): return "embedded" return None