v0.3.8

.github/workflows/docker-build.yml

@@ -13,8 +13,8 @@ on:
         default: 'latest'
 
 env:
-  REGISTRY: docker.io
+  DOCKERHUB_IMAGE: pyd4vinci/scrapling
-  IMAGE_NAME: pyd4vinci/scrapling
+  GHCR_IMAGE: ghcr.io/${{ github.repository_owner }}/scrapling
 
 jobs:
   build-and-push:
@@ -35,15 +35,24 @@ jobs:
     - name: Log in to Docker Hub
       uses: docker/login-action@v3
       with:
-        registry: ${{ env.REGISTRY }}
+        registry: docker.io
         username: ${{ secrets.DOCKER_USERNAME }}
         password: ${{ secrets.DOCKER_PASSWORD }}
 
+    - name: Log in to GitHub Container Registry
+      uses: docker/login-action@v3
+      with:
+        registry: ghcr.io
+        username: ${{ github.actor }}
+        password: ${{ secrets.CONTAINER_TOKEN }}
+
     - name: Extract metadata
       id: meta
       uses: docker/metadata-action@v5
       with:
-        images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+        images: |
+          ${{ env.DOCKERHUB_IMAGE }}
+          ${{ env.GHCR_IMAGE }}
         tags: |
           type=ref,event=branch
           type=ref,event=pr
@@ -51,6 +60,14 @@ jobs:
           type=semver,pattern={{major}}.{{minor}}
           type=semver,pattern={{major}}
           type=raw,value=latest,enable={{is_default_branch}}
+        labels: |
+          org.opencontainers.image.title=Scrapling
+          org.opencontainers.image.description=An undetectable, powerful, flexible, high-performance Python library that makes Web Scraping easy and effortless as it should be!
+          org.opencontainers.image.vendor=D4Vinci
+          org.opencontainers.image.licenses=BSD
+          org.opencontainers.image.url=https://scrapling.readthedocs.io/en/latest/
+          org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }}
+          org.opencontainers.image.documentation=https://scrapling.readthedocs.io/en/latest/
 
     - name: Build and push Docker image
       uses: docker/build-push-action@v5

docs/fetching/dynamic.md

@@ -89,6 +89,7 @@ Scrapling provides many options with this fetcher and its session classes. To ma
 |       locale        | Set the locale for the browser if wanted. The default value is `en-US`.                                                                                                                                                                                                                                                                                                                                                     |    ✔️    |
 |       cdp_url       | Instead of launching a new browser instance, connect to this CDP URL to control real browsers through CDP.                                                                                                                                                                                                                                                                                                                  |    ✔️    |
 |    user_data_dir    | Path to a User Data Directory, which stores browser session data like cookies and local storage. The default is to create a temporary directory. **Only Works with sessions**                                                                                                                                                                                                                                               |    ✔️    |
+|     extra_flags     | A list of additional browser flags to pass to the browser on launch.                                                                                                                                                                                                                                                                                                                                                        |    ✔️    |
 |   additional_args   | Additional arguments to be passed to Playwright's context as additional settings, and they take higher priority than Scrapling's settings.                                                                                                                                                                                                                                                                                  |    ✔️    |
 |   selector_config   | A dictionary of custom parsing arguments to be used when creating the final `Selector`/`Response` class.                                                                                                                                                                                                                                                                                                                    |    ✔️    |
 
@@ -300,4 +301,4 @@ Use DynamicFetcher when:
 - Need custom browser config
 - Want flexible stealth options 
 
-If you want more stealth and control without much config, check out the [StealthyFetcher](stealthy.md).
+If you want more stealth and control without much config, check out the [StealthyFetcher](stealthy.md).

docs/tutorials/external.md

@@ -0,0 +1,34 @@
+
+If you have issues with the browser installation, such as resource management, we recommend you try the Cloud Browser from [Scrapeless](https://www.scrapeless.com/en/product/scraping-browser?utm_source=official&utm_term=scrapling) for free!
+
+The usage is straightforward: create an account and [get your API key](https://docs.scrapeless.com/en/scraping-browser/quickstart/getting-started/?utm_source=official&utm_term=scrapling), then pass it to the `DynamicSession` like this:
+
+```python
+from urllib.parse import urlencode
+
+from scrapling.fetchers import DynamicSession
+
+# Configure your browser session
+config = {
+    "token": "YOUR_API_KEY",
+    "sessionName": "scrapling-session",
+    "sessionTTL": "300",  # 5 minutes
+    "proxyCountry": "ANY",
+    "sessionRecording": "false",
+}
+
+# Build WebSocket URL
+ws_endpoint = f"wss://browser.scrapeless.com/api/v2/browser?{urlencode(config)}"
+print('Connecting to Scrapeless...')
+
+with DynamicSession(cdp_url=ws_endpoint, disable_resources=True) as s:
+    print("Connected!")
+    page = s.fetch("https://httpbin.org/headers", network_idle=True)
+    print(f"Page loaded, content length: {len(page.body)}")
+    print(page.json())
+```
+The `DynamicSession` class instance will work as usual, so no further explanation is needed.
+
+However, the Scrapeless Cloud Browser can be configured with proxy options, like the proxy country in the config above, [custom fingerprint](https://docs.scrapeless.com/en/scraping-browser/features/advanced-privacy-anti-detection/custom-fingerprint/?utm_source=official&utm_term=scrapling) configuration, [captcha solving](https://docs.scrapeless.com/en/scraping-browser/features/advanced-privacy-anti-detection/supported-captchas/?utm_source=official&utm_term=scrapling), and more.
+
+Check out the [Scrapeless's browser documentation](https://docs.scrapeless.com/en/scraping-browser/quickstart/introduction/?utm_source=official&utm_term=scrapling) for more details.

mkdocs.yml

@@ -83,6 +83,7 @@ nav:
   - Tutorials:
     - A Free Alternative to AI for Robust Web Scraping: tutorials/replacing_ai.md
     - Migrating from BeautifulSoup: tutorials/migrating_from_beautifulsoup.md
+    - Using Scrapeless browser: tutorials/external.md
 #    - Migrating from AutoScraper: tutorials/migrating_from_autoscraper.md
   - Development:
     - API Reference:

pyproject.toml

@@ -4,8 +4,8 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "scrapling"
-# Static version instead of dynamic version so we can get better layer caching while building docker, check the docker file to understand
+# Static version instead of a dynamic version so we can get better layer caching while building docker, check the docker file to understand
-version = "0.3.7"
+version = "0.3.8"
 description = "Scrapling is an undetectable, powerful, flexible, high-performance Python library that makes Web Scraping easy and effortless as it should be!"
 readme = {file = "README.md", content-type = "text/markdown"}
 license = {file = "LICENSE"}
@@ -59,7 +59,7 @@ classifiers = [
 dependencies = [
     "lxml>=6.0.2",
     "cssselect>=1.3.0",
-    "orjson>=3.11.3",
+    "orjson>=3.11.4",
     "tldextract>=5.3.0",
 ]
 
@@ -74,7 +74,7 @@ fetchers = [
     "msgspec>=0.19.0",
 ]
 ai = [
-    "mcp>=1.16.0",
+    "mcp>=1.19.0",
     "markdownify>=1.2.0",
     "scrapling[fetchers]",
 ]
@@ -89,6 +89,7 @@ all = [
 
 [project.urls]
 Homepage = "https://github.com/D4Vinci/Scrapling"
+Changelog = "https://github.com/D4Vinci/Scrapling/releases"
 Documentation = "https://scrapling.readthedocs.io/en/latest/"
 Repository = "https://github.com/D4Vinci/Scrapling"
 "Bug Tracker" = "https://github.com/D4Vinci/Scrapling/issues"

scrapling/__init__.py

@@ -1,5 +1,5 @@
 __author__ = "Karim Shoair (karim.shoair@pm.me)"
-__version__ = "0.3.7"
+__version__ = "0.3.8"
 __copyright__ = "Copyright (c) 2024 Karim Shoair"
 
 from typing import Any, TYPE_CHECKING

scrapling/engines/_browsers/_base.py

@@ -2,17 +2,27 @@ from time import time
 from asyncio import sleep as asyncio_sleep, Lock
 
 from camoufox import DefaultAddons
-from playwright.sync_api import BrowserContext, Playwright
+from playwright.sync_api import (
-from playwright.async_api import (
+    Page,
-    BrowserContext as AsyncBrowserContext,
+    Frame,
-    Playwright as AsyncPlaywright,
+    BrowserContext,
+    Playwright,
+    Response as SyncPlaywrightResponse,
 )
+from playwright.async_api import (
+    Page as AsyncPage,
+    Frame as AsyncFrame,
+    Playwright as AsyncPlaywright,
+    Response as AsyncPlaywrightResponse,
+    BrowserContext as AsyncBrowserContext,
+)
+from playwright._impl._errors import Error as PlaywrightError
 from camoufox.pkgman import installed_verstr as camoufox_version
 from camoufox.utils import launch_options as generate_launch_options
 
 from ._page import PageInfo, PagePool
 from scrapling.parser import Selector
-from scrapling.core._types import Any, cast, Dict, Optional, TYPE_CHECKING
+from scrapling.core._types import Any, cast, Dict, List, Optional, Callable, TYPE_CHECKING
 from scrapling.engines.toolbelt.fingerprints import get_os_name
 from ._validators import validate, PlaywrightConfig, CamoufoxConfig
 from ._config_tools import _compiled_stealth_scripts, _launch_kwargs, _context_kwargs
@@ -26,10 +36,35 @@ class SyncSession:
         self.max_pages = max_pages
         self.page_pool = PagePool(max_pages)
         self._max_wait_for_page = 60
-        self.playwright: Optional[Playwright] = None
+        self.playwright: Playwright | Any = None
-        self.context: Optional[BrowserContext] = None
+        self.context: BrowserContext | Any = None
         self._closed = False
 
+    def __create__(self):
+        pass
+
+    def close(self):  # pragma: no cover
+        """Close all resources"""
+        if self._closed:
+            return
+
+        if self.context:
+            self.context.close()
+            self.context = None
+
+        if self.playwright:
+            self.playwright.stop()
+            self.playwright = None  # pyright: ignore
+
+        self._closed = True
+
+    def __enter__(self):
+        self.__create__()
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.close()
+
     def _get_page(
         self,
         timeout: int | float,
@@ -53,7 +88,9 @@ class SyncSession:
             for script in _compiled_stealth_scripts():
                 page.add_init_script(script=script)
 
-        return self.page_pool.add_page(page)
+        page_info = self.page_pool.add_page(page)
+        page_info.mark_busy()
+        return page_info
 
     def get_pool_stats(self) -> Dict[str, int]:
         """Get statistics about the current page pool"""
@@ -63,17 +100,76 @@ class SyncSession:
             "max_pages": self.max_pages,
         }
 
+    @staticmethod
+    def _wait_for_networkidle(page: Page | Frame, timeout: Optional[int] = None):
+        """Wait for the page to become idle (no network activity) even if there are never-ending requests."""
+        try:
+            page.wait_for_load_state("networkidle", timeout=timeout)
+        except PlaywrightError:
+            pass
+
+    def _wait_for_page_stability(self, page: Page | Frame, load_dom: bool, network_idle: bool):
+        page.wait_for_load_state(state="load")
+        if load_dom:
+            page.wait_for_load_state(state="domcontentloaded")
+        if network_idle:
+            self._wait_for_networkidle(page)
+
+    @staticmethod
+    def _create_response_handler(page_info: PageInfo, response_container: List) -> Callable:
+        """Create a response handler that captures the final navigation response.
+
+        :param page_info: The PageInfo object containing the page
+        :param response_container: A list to store the final response (mutable container)
+        :return: A callback function for page.on("response", ...)
+        """
+
+        def handle_response(finished_response: SyncPlaywrightResponse):
+            if (
+                finished_response.request.resource_type == "document"
+                and finished_response.request.is_navigation_request()
+                and finished_response.request.frame == page_info.page.main_frame
+            ):
+                response_container[0] = finished_response
+
+        return handle_response
+
 
 class AsyncSession:
     def __init__(self, max_pages: int = 1):
         self.max_pages = max_pages
         self.page_pool = PagePool(max_pages)
         self._max_wait_for_page = 60
-        self.playwright: Optional[AsyncPlaywright] = None
+        self.playwright: AsyncPlaywright | Any = None
-        self.context: Optional[AsyncBrowserContext] = None
+        self.context: AsyncBrowserContext | Any = None
         self._closed = False
         self._lock = Lock()
 
+    async def __create__(self):
+        pass
+
+    async def close(self):
+        """Close all resources"""
+        if self._closed:  # pragma: no cover
+            return
+
+        if self.context:
+            await self.context.close()
+            self.context = None  # pyright: ignore
+
+        if self.playwright:
+            await self.playwright.stop()
+            self.playwright = None  # pyright: ignore
+
+        self._closed = True
+
+    async def __aenter__(self):
+        await self.__create__()
+        return self
+
+    async def __aexit__(self, exc_type, exc_val, exc_tb):
+        await self.close()
+
     async def _get_page(
         self,
         timeout: int | float,
@@ -97,7 +193,6 @@ class AsyncSession:
                         f"No pages finished to clear place in the pool within the {self._max_wait_for_page}s timeout period"
                     )
 
-            assert self.context is not None, "Browser context not initialized"
             page = await self.context.new_page()
             page.set_default_navigation_timeout(timeout)
             page.set_default_timeout(timeout)
@@ -121,6 +216,40 @@ class AsyncSession:
             "max_pages": self.max_pages,
         }
 
+    @staticmethod
+    async def _wait_for_networkidle(page: AsyncPage | AsyncFrame, timeout: Optional[int] = None):
+        """Wait for the page to become idle (no network activity) even if there are never-ending requests."""
+        try:
+            await page.wait_for_load_state("networkidle", timeout=timeout)
+        except PlaywrightError:
+            pass
+
+    async def _wait_for_page_stability(self, page: AsyncPage | AsyncFrame, load_dom: bool, network_idle: bool):
+        await page.wait_for_load_state(state="load")
+        if load_dom:
+            await page.wait_for_load_state(state="domcontentloaded")
+        if network_idle:
+            await self._wait_for_networkidle(page)
+
+    @staticmethod
+    def _create_response_handler(page_info: PageInfo, response_container: List) -> Callable:
+        """Create an async response handler that captures the final navigation response.
+
+        :param page_info: The PageInfo object containing the page
+        :param response_container: A list to store the final response (mutable container)
+        :return: A callback function for page.on("response", ...)
+        """
+
+        async def handle_response(finished_response: AsyncPlaywrightResponse):
+            if (
+                finished_response.request.resource_type == "document"
+                and finished_response.request.is_navigation_request()
+                and finished_response.request.frame == page_info.page.main_frame
+            ):
+                response_container[0] = finished_response
+
+        return handle_response
+
 
 class DynamicSessionMixin:
     def __validate__(self, **params):
@@ -147,6 +276,7 @@ class DynamicSessionMixin:
         self.wait_selector = config.wait_selector
         self.init_script = config.init_script
         self.wait_selector_state = config.wait_selector_state
+        self.extra_flags = config.extra_flags
         self.selector_config = config.selector_config
         self.additional_args = config.additional_args
         self.page_action = config.page_action
@@ -171,6 +301,7 @@ class DynamicSessionMixin:
                     self.stealth,
                     self.hide_canvas,
                     self.disable_webgl,
+                    tuple(self.extra_flags) if self.extra_flags else tuple(),
                 )
             )
             self.launch_options["extra_http_headers"] = dict(self.launch_options["extra_http_headers"])

scrapling/engines/_browsers/_camoufox.py

@@ -2,22 +2,19 @@ from random import randint
 from re import compile as re_compile
 
 from playwright.sync_api import (
-    Response as SyncPlaywrightResponse,
-    sync_playwright,
-    Locator,
     Page,
+    Locator,
+    sync_playwright,
 )
 from playwright.async_api import (
     async_playwright,
-    Response as AsyncPlaywrightResponse,
-    BrowserContext as AsyncBrowserContext,
-    Playwright as AsyncPlaywright,
-    Locator as AsyncLocator,
     Page as async_Page,
+    Locator as AsyncLocator,
+    Playwright as AsyncPlaywright,
+    BrowserContext as AsyncBrowserContext,
 )
-from playwright._impl._errors import Error as PlaywrightError
 
-from ._validators import validate_fetch as _validate
+from ._validators import validate_fetch as _validate, CamoufoxConfig
 from ._base import SyncSession, AsyncSession, StealthySessionMixin
 from scrapling.core.utils import log
 from scrapling.core._types import (
@@ -184,61 +181,21 @@ class StealthySession(StealthySessionMixin, SyncSession):
         if self.cookies:  # pragma: no cover
             self.context.add_cookies(self.cookies)
 
-    def __enter__(self):  # pragma: no cover
-        self.__create__()
-        return self
-
-    def __exit__(self, exc_type, exc_val, exc_tb):
-        self.close()
-
-    def close(self):  # pragma: no cover
-        """Close all resources"""
-        if self._closed:  # pragma: no cover
-            return
-
-        if self.context:
-            self.context.close()
-            self.context = None
-
-        if self.playwright:
-            self.playwright.stop()
-            self.playwright = None
-
-        self._closed = True
-
-    @staticmethod
-    def _get_page_content(page: Page) -> str:
-        """
-        A workaround for the Playwright issue with `page.content()` on Windows. Ref.: https://github.com/microsoft/playwright/issues/16108
-        :param page: The page to extract content from.
-        :return:
-        """
-        while True:
-            try:
-                return page.content() or ""
-            except PlaywrightError:
-                page.wait_for_timeout(1000)
-                continue
-        return ""  # pyright: ignore
-
     def _solve_cloudflare(self, page: Page) -> None:  # pragma: no cover
         """Solve the cloudflare challenge displayed on the playwright page passed
 
         :param page: The targeted page
         :return:
         """
-        try:
+        self._wait_for_networkidle(page, timeout=5000)
-            page.wait_for_load_state("networkidle", timeout=5000)
+        challenge_type = self._detect_cloudflare(ResponseFactory._get_page_content(page))
-        except PlaywrightError:
-            pass
-        challenge_type = self._detect_cloudflare(self._get_page_content(page))
         if not challenge_type:
             log.error("No Cloudflare challenge found.")
             return
         else:
             log.info(f'The turnstile version discovered is "{challenge_type}"')
             if challenge_type == "non-interactive":
-                while "<title>Just a moment...</title>" in (self._get_page_content(page)):
+                while "<title>Just a moment...</title>" in (ResponseFactory._get_page_content(page)):
                     log.info("Waiting for Cloudflare wait page to disappear.")
                     page.wait_for_timeout(1000)
                     page.wait_for_load_state()
@@ -249,15 +206,14 @@ class StealthySession(StealthySessionMixin, SyncSession):
                 box_selector = "#cf_turnstile div, #cf-turnstile div, .turnstile>div>div"
                 if challenge_type != "embedded":
                     box_selector = ".main-content p+div>div>div"
-                    while "Verifying you are human." in self._get_page_content(page):
+                    while "Verifying you are human." in ResponseFactory._get_page_content(page):
                         # Waiting for the verify spinner to disappear, checking every 1s if it disappeared
                         page.wait_for_timeout(500)
 
                 outer_box = {}
                 iframe = page.frame(url=__CF_PATTERN__)
                 if iframe is not None:
-                    iframe.wait_for_load_state(state="domcontentloaded")
+                    self._wait_for_page_stability(iframe, True, True)
-                    iframe.wait_for_load_state("networkidle")
 
                     if challenge_type != "embedded":
                         while not iframe.frame_element().is_visible():
@@ -273,16 +229,20 @@ class StealthySession(StealthySessionMixin, SyncSession):
 
                 # Move the mouse to the center of the window, then press and hold the left mouse button
                 page.mouse.click(captcha_x, captcha_y, delay=60, button="left")
-                page.wait_for_load_state("networkidle")
+                self._wait_for_networkidle(page)
                 if iframe is not None:
-                    # Wait for the frame to be removed from the page
+                    # Wait for the frame to be removed from the page (with 30s timeout = 300 iterations * 100 ms)
+                    attempts = 0
                     while iframe in page.frames:
+                        if attempts >= 300:
+                            log.info("Cloudflare iframe didn't disappear after 30s, continuing...")
+                            break
                         page.wait_for_timeout(100)
+                        attempts += 1
                 if challenge_type != "embedded":
                     page.locator(box_selector).last.wait_for(state="detached")
                     page.locator(".zone-name-title").wait_for(state="hidden")
-                page.wait_for_load_state(state="load")
+                self._wait_for_page_stability(page, True, False)
-                page.wait_for_load_state(state="domcontentloaded")
 
                 log.info("Cloudflare captcha is solved")
                 return
@@ -337,38 +297,26 @@ class StealthySession(StealthySessionMixin, SyncSession):
                 ("solve_cloudflare", solve_cloudflare, self.solve_cloudflare),
                 ("selector_config", selector_config, self.selector_config),
             ],
+            CamoufoxConfig,
             _UNSET,
         )
 
         if self._closed:  # pragma: no cover
             raise RuntimeError("Context manager has been closed")
 
-        final_response = None
         referer = (
             generate_convincing_referer(url) if (params.google_search and "referer" not in self._headers_keys) else None
         )
 
-        def handle_response(finished_response: SyncPlaywrightResponse):
-            nonlocal final_response
-            if (
-                finished_response.request.resource_type == "document"
-                and finished_response.request.is_navigation_request()
-                and finished_response.request.frame == page_info.page.main_frame
-            ):
-                final_response = finished_response
-
         page_info = self._get_page(params.timeout, params.extra_headers, params.disable_resources)
-        page_info.mark_busy(url=url)
+        final_response = [None]
+        handle_response = self._create_response_handler(page_info, final_response)
 
         try:  # pragma: no cover
             # Navigate to URL and wait for a specified state
             page_info.page.on("response", handle_response)
             first_response = page_info.page.goto(url, referer=referer)
-            if params.load_dom:
+            self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle)
-                page_info.page.wait_for_load_state(state="domcontentloaded")
-
-            if params.network_idle:
-                page_info.page.wait_for_load_state("networkidle")
 
             if not first_response:
                 raise RuntimeError(f"Failed to get response for {url}")
@@ -376,11 +324,7 @@ class StealthySession(StealthySessionMixin, SyncSession):
             if params.solve_cloudflare:
                 self._solve_cloudflare(page_info.page)
                 # Make sure the page is fully loaded after the captcha
-                page_info.page.wait_for_load_state(state="load")
+                self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle)
-                if params.load_dom:
-                    page_info.page.wait_for_load_state(state="domcontentloaded")
-                if params.network_idle:
-                    page_info.page.wait_for_load_state("networkidle")
 
             if params.page_action:
                 try:
@@ -393,17 +337,13 @@ class StealthySession(StealthySessionMixin, SyncSession):
                     waiter: Locator = page_info.page.locator(params.wait_selector)
                     waiter.first.wait_for(state=params.wait_selector_state)
                     # Wait again after waiting for the selector, helpful with protections like Cloudflare
-                    page_info.page.wait_for_load_state(state="load")
+                    self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle)
-                    if params.load_dom:
-                        page_info.page.wait_for_load_state(state="domcontentloaded")
-                    if params.network_idle:
-                        page_info.page.wait_for_load_state("networkidle")
                 except Exception as e:
                     log.error(f"Error waiting for selector {params.wait_selector}: {e}")
 
             page_info.page.wait_for_timeout(params.wait)
             response = ResponseFactory.from_playwright_response(
-                page_info.page, first_response, final_response, params.selector_config
+                page_info.page, first_response, final_response[0], params.selector_config, bool(params.page_action)
             )
 
             # Close the page to free up resources
@@ -528,61 +468,21 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):
         if self.cookies:
             await self.context.add_cookies(self.cookies)  # pyright: ignore [reportArgumentType]
 
-    async def __aenter__(self):
+    async def _solve_cloudflare(self, page: async_Page):  # pragma: no cover
-        await self.__create__()
-        return self
-
-    async def __aexit__(self, exc_type, exc_val, exc_tb):
-        await self.close()
-
-    async def close(self):
-        """Close all resources"""
-        if self._closed:  # pragma: no cover
-            return
-
-        if self.context:
-            await self.context.close()
-            self.context = None  # pyright: ignore
-
-        if self.playwright:
-            await self.playwright.stop()
-            self.playwright = None  # pyright: ignore
-
-        self._closed = True
-
-    @staticmethod
-    async def _get_page_content(page: async_Page) -> str:
-        """
-        A workaround for the Playwright issue with `page.content()` on Windows. Ref.: https://github.com/microsoft/playwright/issues/16108
-        :param page: The page to extract content from.
-        :return:
-        """
-        while True:
-            try:
-                return (await page.content()) or ""
-            except PlaywrightError:
-                await page.wait_for_timeout(1000)
-                continue
-        return ""  # pyright: ignore
-
-    async def _solve_cloudflare(self, page: async_Page):
         """Solve the cloudflare challenge displayed on the playwright page passed. The async version
 
         :param page: The async targeted page
         :return:
         """
-        try:
+        await self._wait_for_networkidle(page, timeout=5000)
-            await page.wait_for_load_state("networkidle", timeout=5000)
+        challenge_type = self._detect_cloudflare(await ResponseFactory._get_async_page_content(page))
-        except PlaywrightError:
-            pass
-        challenge_type = self._detect_cloudflare(await self._get_page_content(page))
         if not challenge_type:
             log.error("No Cloudflare challenge found.")
             return
         else:
             log.info(f'The turnstile version discovered is "{challenge_type}"')
             if challenge_type == "non-interactive":  # pragma: no cover
-                while "<title>Just a moment...</title>" in (await self._get_page_content(page)):
+                while "<title>Just a moment...</title>" in (await ResponseFactory._get_async_page_content(page)):
                     log.info("Waiting for Cloudflare wait page to disappear.")
                     await page.wait_for_timeout(1000)
                     await page.wait_for_load_state()
@@ -593,15 +493,14 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):
                 box_selector = "#cf_turnstile div, #cf-turnstile div, .turnstile>div>div"
                 if challenge_type != "embedded":
                     box_selector = ".main-content p+div>div>div"
-                    while "Verifying you are human." in (await self._get_page_content(page)):
+                    while "Verifying you are human." in (await ResponseFactory._get_async_page_content(page)):
                         # Waiting for the verify spinner to disappear, checking every 1s if it disappeared
                         await page.wait_for_timeout(500)
 
                 outer_box = {}
                 iframe = page.frame(url=__CF_PATTERN__)
                 if iframe is not None:
-                    await iframe.wait_for_load_state(state="domcontentloaded")
+                    await self._wait_for_page_stability(iframe, True, True)
-                    await iframe.wait_for_load_state("networkidle")
 
                     if challenge_type != "embedded":
                         while not await (await iframe.frame_element()).is_visible():
@@ -617,16 +516,20 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):
 
                 # Move the mouse to the center of the window, then press and hold the left mouse button
                 await page.mouse.click(captcha_x, captcha_y, delay=60, button="left")
-                await page.wait_for_load_state("networkidle")
+                await self._wait_for_networkidle(page)
                 if iframe is not None:
-                    # Wait for the frame to be removed from the page
+                    # Wait for the frame to be removed from the page (with 30s timeout = 300 iterations * 100 ms)
+                    attempts = 0
                     while iframe in page.frames:
+                        if attempts >= 300:
+                            log.info("Cloudflare iframe didn't disappear after 30s, continuing...")
+                            break
                         await page.wait_for_timeout(100)
+                        attempts += 1
                 if challenge_type != "embedded":
                     await page.locator(box_selector).wait_for(state="detached")
                     await page.locator(".zone-name-title").wait_for(state="hidden")
-                await page.wait_for_load_state(state="load")
+                await self._wait_for_page_stability(page, True, False)
-                await page.wait_for_load_state(state="domcontentloaded")
 
                 log.info("Cloudflare captcha is solved")
                 return
@@ -681,28 +584,20 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):
                 ("solve_cloudflare", solve_cloudflare, self.solve_cloudflare),
                 ("selector_config", selector_config, self.selector_config),
             ],
+            CamoufoxConfig,
             _UNSET,
         )
 
         if self._closed:  # pragma: no cover
             raise RuntimeError("Context manager has been closed")
 
-        final_response = None
         referer = (
             generate_convincing_referer(url) if (params.google_search and "referer" not in self._headers_keys) else None
         )
 
-        async def handle_response(finished_response: AsyncPlaywrightResponse):
-            nonlocal final_response
-            if (
-                finished_response.request.resource_type == "document"
-                and finished_response.request.is_navigation_request()
-                and finished_response.request.frame == page_info.page.main_frame
-            ):
-                final_response = finished_response
-
         page_info = await self._get_page(params.timeout, params.extra_headers, params.disable_resources)
-        page_info.mark_busy(url=url)
+        final_response = [None]
+        handle_response = self._create_response_handler(page_info, final_response)
 
         if TYPE_CHECKING:
             if not isinstance(page_info.page, async_Page):
@@ -712,11 +607,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):
             # Navigate to URL and wait for a specified state
             page_info.page.on("response", handle_response)
             first_response = await page_info.page.goto(url, referer=referer)
-            if params.load_dom:
+            await self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle)
-                await page_info.page.wait_for_load_state(state="domcontentloaded")
-
-            if params.network_idle:
-                await page_info.page.wait_for_load_state("networkidle")
 
             if not first_response:
                 raise RuntimeError(f"Failed to get response for {url}")
@@ -724,11 +615,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):
             if params.solve_cloudflare:
                 await self._solve_cloudflare(page_info.page)
                 # Make sure the page is fully loaded after the captcha
-                await page_info.page.wait_for_load_state(state="load")
+                await self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle)
-                if params.load_dom:
-                    await page_info.page.wait_for_load_state(state="domcontentloaded")
-                if params.network_idle:
-                    await page_info.page.wait_for_load_state("networkidle")
 
             if params.page_action:
                 try:
@@ -741,11 +628,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):
                     waiter: AsyncLocator = page_info.page.locator(params.wait_selector)
                     await waiter.first.wait_for(state=params.wait_selector_state)
                     # Wait again after waiting for the selector, helpful with protections like Cloudflare
-                    await page_info.page.wait_for_load_state(state="load")
+                    await self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle)
-                    if params.load_dom:
-                        await page_info.page.wait_for_load_state(state="domcontentloaded")
-                    if params.network_idle:
-                        await page_info.page.wait_for_load_state("networkidle")
                 except Exception as e:
                     log.error(f"Error waiting for selector {params.wait_selector}: {e}")
 
@@ -753,7 +636,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):
 
             # Create response object
             response = await ResponseFactory.from_async_playwright_response(
-                page_info.page, first_response, final_response, params.selector_config
+                page_info.page, first_response, final_response[0], params.selector_config, bool(params.page_action)
             )
 
             # Close the page to free up resources

scrapling/engines/_browsers/_config_tools.py

@@ -70,12 +70,17 @@ def _launch_kwargs(
     stealth,
     hide_canvas,
     disable_webgl,
+    extra_flags: Tuple,
 ) -> Tuple:
     """Creates the arguments we will use while launching playwright's browser"""
+    base_args = DEFAULT_FLAGS
+    if extra_flags:
+        base_args = base_args + extra_flags
+
     launch_kwargs = {
         "locale": locale,
         "headless": headless,
-        "args": DEFAULT_FLAGS,
+        "args": base_args,
         "color_scheme": "dark",  # Bypasses the 'prefersLightColor' check in creepjs
         "proxy": proxy or tuple(),
         "device_scale_factor": 2,
@@ -85,9 +90,10 @@ def _launch_kwargs(
         "user_agent": useragent or __default_useragent__,
     }
     if stealth:
+        stealth_args = base_args + _set_flags(hide_canvas, disable_webgl)
         launch_kwargs.update(
             {
-                "args": DEFAULT_FLAGS + _set_flags(hide_canvas, disable_webgl),
+                "args": stealth_args,
                 "chromium_sandbox": True,
                 "is_mobile": False,
                 "has_touch": False,

scrapling/engines/_browsers/_controllers.py

@@ -1,23 +1,20 @@
 from playwright.sync_api import (
-    Response as SyncPlaywrightResponse,
-    sync_playwright,
-    Playwright,
     Locator,
+    Playwright,
+    sync_playwright,
 )
 from playwright.async_api import (
     async_playwright,
-    Response as AsyncPlaywrightResponse,
-    BrowserContext as AsyncBrowserContext,
-    Playwright as AsyncPlaywright,
     Locator as AsyncLocator,
-    Page as async_Page,
+    Playwright as AsyncPlaywright,
+    BrowserContext as AsyncBrowserContext,
 )
 from patchright.sync_api import sync_playwright as sync_patchright
 from patchright.async_api import async_playwright as async_patchright
 
 from scrapling.core.utils import log
 from ._base import SyncSession, AsyncSession, DynamicSessionMixin
-from ._validators import validate_fetch as _validate
+from ._validators import validate_fetch as _validate, PlaywrightConfig
 from scrapling.core._types import (
     Any,
     Dict,
@@ -98,6 +95,7 @@ class DynamicSession(DynamicSessionMixin, SyncSession):
         load_dom: bool = True,
         wait_selector_state: SelectorWaitStates = "attached",
         user_data_dir: str = "",
+        extra_flags: Optional[List[str]] = None,
         selector_config: Optional[Dict] = None,
         additional_args: Optional[Dict] = None,
     ):
@@ -127,6 +125,7 @@ class DynamicSession(DynamicSessionMixin, SyncSession):
         :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
         :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.
         :param user_data_dir: Path to a User Data Directory, which stores browser session data like cookies and local storage. The default is to create a temporary directory.
+        :param extra_flags: A list of additional browser flags to pass to the browser on launch.
         :param selector_config: The arguments that will be passed in the end while creating the final Selector's class.
         :param additional_args: Additional arguments to be passed to Playwright's context as additional settings, and it takes higher priority than Scrapling's settings.
         """
@@ -152,6 +151,7 @@ class DynamicSession(DynamicSessionMixin, SyncSession):
             extra_headers=extra_headers,
             wait_selector=wait_selector,
             disable_webgl=disable_webgl,
+            extra_flags=extra_flags,
             selector_config=selector_config,
             additional_args=additional_args,
             disable_resources=disable_resources,
@@ -178,28 +178,6 @@ class DynamicSession(DynamicSessionMixin, SyncSession):
         if self.cookies:  # pragma: no cover
             self.context.add_cookies(self.cookies)
 
-    def __enter__(self):
-        self.__create__()
-        return self
-
-    def __exit__(self, exc_type, exc_val, exc_tb):
-        self.close()
-
-    def close(self):  # pragma: no cover
-        """Close all resources"""
-        if self._closed:
-            return
-
-        if self.context:
-            self.context.close()
-            self.context = None
-
-        if self.playwright:
-            self.playwright.stop()
-            self.playwright = None  # pyright: ignore
-
-        self._closed = True
-
     def fetch(
         self,
         url: str,
@@ -247,38 +225,26 @@ class DynamicSession(DynamicSessionMixin, SyncSession):
                 ("load_dom", load_dom, self.load_dom),
                 ("selector_config", selector_config, self.selector_config),
             ],
+            PlaywrightConfig,
             _UNSET,
         )
 
         if self._closed:  # pragma: no cover
             raise RuntimeError("Context manager has been closed")
 
-        final_response = None
         referer = (
             generate_convincing_referer(url) if (params.google_search and "referer" not in self._headers_keys) else None
         )
 
-        def handle_response(finished_response: SyncPlaywrightResponse):
-            nonlocal final_response
-            if (
-                finished_response.request.resource_type == "document"
-                and finished_response.request.is_navigation_request()
-                and finished_response.request.frame == page_info.page.main_frame
-            ):
-                final_response = finished_response
-
         page_info = self._get_page(params.timeout, params.extra_headers, params.disable_resources)
-        page_info.mark_busy(url=url)
+        final_response = [None]
+        handle_response = self._create_response_handler(page_info, final_response)
 
         try:  # pragma: no cover
             # Navigate to URL and wait for a specified state
             page_info.page.on("response", handle_response)
             first_response = page_info.page.goto(url, referer=referer)
-            if params.load_dom:
+            self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle)
-                page_info.page.wait_for_load_state(state="domcontentloaded")
-
-            if params.network_idle:
-                page_info.page.wait_for_load_state("networkidle")
 
             if not first_response:
                 raise RuntimeError(f"Failed to get response for {url}")
@@ -294,11 +260,7 @@ class DynamicSession(DynamicSessionMixin, SyncSession):
                     waiter: Locator = page_info.page.locator(params.wait_selector)
                     waiter.first.wait_for(state=params.wait_selector_state)
                     # Wait again after waiting for the selector, helpful with protections like Cloudflare
-                    page_info.page.wait_for_load_state(state="load")
+                    self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle)
-                    if params.load_dom:
-                        page_info.page.wait_for_load_state(state="domcontentloaded")
-                    if params.network_idle:
-                        page_info.page.wait_for_load_state("networkidle")
                 except Exception as e:  # pragma: no cover
                     log.error(f"Error waiting for selector {params.wait_selector}: {e}")
 
@@ -306,7 +268,7 @@ class DynamicSession(DynamicSessionMixin, SyncSession):
 
             # Create response object
             response = ResponseFactory.from_playwright_response(
-                page_info.page, first_response, final_response, params.selector_config
+                page_info.page, first_response, final_response[0], params.selector_config, bool(params.page_action)
             )
 
             # Close the page to free up resources
@@ -348,6 +310,7 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession):
         load_dom: bool = True,
         wait_selector_state: SelectorWaitStates = "attached",
         user_data_dir: str = "",
+        extra_flags: Optional[List[str]] = None,
         selector_config: Optional[Dict] = None,
         additional_args: Optional[Dict] = None,
     ):
@@ -378,6 +341,7 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession):
         :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.
         :param max_pages: The maximum number of tabs to be opened at the same time. It will be used in rotation through a PagePool.
         :param user_data_dir: Path to a User Data Directory, which stores browser session data like cookies and local storage. The default is to create a temporary directory.
+        :param extra_flags: A list of additional browser flags to pass to the browser on launch.
         :param selector_config: The arguments that will be passed in the end while creating the final Selector's class.
         :param additional_args: Additional arguments to be passed to Playwright's context as additional settings, and it takes higher priority than Scrapling's settings.
         """
@@ -404,6 +368,7 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession):
             extra_headers=extra_headers,
             wait_selector=wait_selector,
             disable_webgl=disable_webgl,
+            extra_flags=extra_flags,
             selector_config=selector_config,
             additional_args=additional_args,
             disable_resources=disable_resources,
@@ -431,28 +396,6 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession):
         if self.cookies:
             await self.context.add_cookies(self.cookies)  # pyright: ignore
 
-    async def __aenter__(self):
-        await self.__create__()
-        return self
-
-    async def __aexit__(self, exc_type, exc_val, exc_tb):
-        await self.close()
-
-    async def close(self):
-        """Close all resources"""
-        if self._closed:  # pragma: no cover
-            return
-
-        if self.context:
-            await self.context.close()
-            self.context = None  # pyright: ignore
-
-        if self.playwright:
-            await self.playwright.stop()
-            self.playwright = None  # pyright: ignore
-
-        self._closed = True
-
     async def fetch(
         self,
         url: str,
@@ -500,30 +443,24 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession):
                 ("load_dom", load_dom, self.load_dom),
                 ("selector_config", selector_config, self.selector_config),
             ],
+            PlaywrightConfig,
             _UNSET,
         )
 
         if self._closed:  # pragma: no cover
             raise RuntimeError("Context manager has been closed")
 
-        final_response = None
         referer = (
             generate_convincing_referer(url) if (params.google_search and "referer" not in self._headers_keys) else None
         )
 
-        async def handle_response(finished_response: AsyncPlaywrightResponse):
-            nonlocal final_response
-            if (
-                finished_response.request.resource_type == "document"
-                and finished_response.request.is_navigation_request()
-                and finished_response.request.frame == page_info.page.main_frame
-            ):
-                final_response = finished_response
-
         page_info = await self._get_page(params.timeout, params.extra_headers, params.disable_resources)
-        page_info.mark_busy(url=url)
+        final_response = [None]
+        handle_response = self._create_response_handler(page_info, final_response)
 
         if TYPE_CHECKING:
+            from playwright.async_api import Page as async_Page
+
             if not isinstance(page_info.page, async_Page):
                 raise TypeError
 
@@ -531,11 +468,7 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession):
             # Navigate to URL and wait for a specified state
             page_info.page.on("response", handle_response)
             first_response = await page_info.page.goto(url, referer=referer)
-            if self.load_dom:
+            await self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle)
-                await page_info.page.wait_for_load_state(state="domcontentloaded")
-
-            if params.network_idle:
-                await page_info.page.wait_for_load_state("networkidle")
 
             if not first_response:
                 raise RuntimeError(f"Failed to get response for {url}")
@@ -551,11 +484,7 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession):
                     waiter: AsyncLocator = page_info.page.locator(params.wait_selector)
                     await waiter.first.wait_for(state=params.wait_selector_state)
                     # Wait again after waiting for the selector, helpful with protections like Cloudflare
-                    await page_info.page.wait_for_load_state(state="load")
+                    await self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle)
-                    if self.load_dom:
-                        await page_info.page.wait_for_load_state(state="domcontentloaded")
-                    if params.network_idle:
-                        await page_info.page.wait_for_load_state("networkidle")
                 except Exception as e:
                     log.error(f"Error waiting for selector {params.wait_selector}: {e}")
 
@@ -563,7 +492,7 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession):
 
             # Create response object
             response = await ResponseFactory.from_async_playwright_response(
-                page_info.page, first_response, final_response, params.selector_config
+                page_info.page, first_response, final_response[0], params.selector_config, bool(params.page_action)
             )
 
             # Close the page to free up resources

scrapling/engines/_browsers/_validators.py

@@ -1,7 +1,8 @@
 from pathlib import Path
 from typing import Annotated
-from dataclasses import dataclass
+from functools import lru_cache
 from urllib.parse import urlparse
+from dataclasses import dataclass, fields
 
 from msgspec import Struct, Meta, convert, ValidationError
 
@@ -19,18 +20,20 @@ from scrapling.engines.toolbelt.navigation import construct_proxy_dict
 
 
 # Custom validators for msgspec
-def _validate_file_path(value: str):
+@lru_cache(8)
+def _is_invalid_file_path(value: str) -> bool | str:
     """Fast file path validation"""
     path = Path(value)
     if not path.exists():
-        raise ValueError(f"Init script path not found: {value}")
+        return f"Init script path not found: {value}"
     if not path.is_file():
-        raise ValueError(f"Init script is not a file: {value}")
+        return f"Init script is not a file: {value}"
     if not path.is_absolute():
-        raise ValueError(f"Init script is not a absolute path: {value}")
+        return f"Init script is not a absolute path: {value}"
+    return False
 
 
-def _validate_addon_path(value: str):
+def _validate_addon_path(value: str) -> None:
     """Fast addon path validation"""
     path = Path(value)
     if not path.exists():
@@ -39,22 +42,16 @@ def _validate_addon_path(value: str):
         raise ValueError(f"Addon path must be a directory of the extracted addon: {value}")
 
 
-def _validate_cdp_url(cdp_url: str):
+@lru_cache(2)
+def _is_invalid_cdp_url(cdp_url: str) -> bool | str:
     """Fast CDP URL validation"""
-    try:
+    if not cdp_url.startswith(("ws://", "wss://")):
-        # Check the scheme
+        return "CDP URL must use 'ws://' or 'wss://' scheme"
-        if not cdp_url.startswith(("ws://", "wss://")):
-            raise ValueError("CDP URL must use 'ws://' or 'wss://' scheme")
 
-        # Validate hostname and port
+    netloc = urlparse(cdp_url).netloc
-        if not urlparse(cdp_url).netloc:
+    if not netloc:
-            raise ValueError("Invalid hostname for the CDP URL")
+        return "Invalid hostname for the CDP URL"
-
+    return False
-    except AttributeError as e:
-        raise ValueError(f"Malformed CDP URL: {cdp_url}: {str(e)}")
-
-    except Exception as e:
-        raise ValueError(f"Invalid CDP URL '{cdp_url}': {str(e)}")
 
 
 # Type aliases for cleaner annotations
@@ -62,7 +59,7 @@ PagesCount = Annotated[int, Meta(ge=1, le=50)]
 Seconds = Annotated[int, float, Meta(ge=0)]
 
 
-class PlaywrightConfig(Struct, kw_only=True, frozen=False):
+class PlaywrightConfig(Struct, kw_only=True, frozen=False, weakref=True):
     """Configuration struct for validation"""
 
     max_pages: PagesCount = 1
@@ -88,6 +85,7 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False):
     load_dom: bool = True
     wait_selector_state: SelectorWaitStates = "attached"
     user_data_dir: str = ""
+    extra_flags: Optional[List[str]] = None
     selector_config: Optional[Dict] = {}
     additional_args: Optional[Dict] = {}
 
@@ -98,20 +96,26 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False):
         if self.proxy:
             self.proxy = construct_proxy_dict(self.proxy, as_tuple=True)
         if self.cdp_url:
-            _validate_cdp_url(self.cdp_url)
+            cdp_msg = _is_invalid_cdp_url(self.cdp_url)
+            if cdp_msg:
+                raise ValueError(cdp_msg)
 
         if not self.cookies:
             self.cookies = []
+        if not self.extra_flags:
+            self.extra_flags = []
         if not self.selector_config:
             self.selector_config = {}
         if not self.additional_args:
             self.additional_args = {}
 
         if self.init_script is not None:
-            _validate_file_path(self.init_script)
+            validation_msg = _is_invalid_file_path(self.init_script)
+            if validation_msg:
+                raise ValueError(validation_msg)
 
 
-class CamoufoxConfig(Struct, kw_only=True, frozen=False):
+class CamoufoxConfig(Struct, kw_only=True, frozen=False, weakref=True):
     """Configuration struct for validation"""
 
     max_pages: PagesCount = 1
@@ -149,14 +153,16 @@ class CamoufoxConfig(Struct, kw_only=True, frozen=False):
         if self.proxy:
             self.proxy = construct_proxy_dict(self.proxy, as_tuple=True)
 
-        if self.addons and isinstance(self.addons, list):
+        if self.addons:
             for addon in self.addons:
                 _validate_addon_path(addon)
         else:
             self.addons = []
 
         if self.init_script is not None:
-            _validate_file_path(self.init_script)
+            validation_msg = _is_invalid_file_path(self.init_script)
+            if validation_msg:
+                raise ValueError(validation_msg)
 
         if not self.cookies:
             self.cookies = []
@@ -169,27 +175,6 @@ class CamoufoxConfig(Struct, kw_only=True, frozen=False):
             self.additional_args = {}
 
 
-# Code parts to validate `fetch` in the least possible numbers of lines overall
-class FetchConfig(Struct, kw_only=True):
-    """Configuration struct for `fetch` calls validation"""
-
-    google_search: bool = True
-    timeout: Seconds = 30000
-    wait: Seconds = 0
-    page_action: Optional[Callable] = None
-    extra_headers: Optional[Dict[str, str]] = None
-    disable_resources: bool = False
-    wait_selector: Optional[str] = None
-    wait_selector_state: SelectorWaitStates = "attached"
-    network_idle: bool = False
-    load_dom: bool = True
-    solve_cloudflare: bool = False
-    selector_config: Dict = {}
-
-    def to_dict(self):
-        return {f: getattr(self, f) for f in self.__struct_fields__}
-
-
 @dataclass
 class _fetch_params:
     """A dataclass of all parameters used by `fetch` calls"""
@@ -208,7 +193,9 @@ class _fetch_params:
     selector_config: Dict
 
 
-def validate_fetch(params: List[Tuple], sentinel=None) -> _fetch_params:
+def validate_fetch(
+    params: List[Tuple], model: type[PlaywrightConfig] | type[CamoufoxConfig], sentinel=None
+) -> _fetch_params:
     result = {}
     overrides = {}
 
@@ -219,16 +206,44 @@ def validate_fetch(params: List[Tuple], sentinel=None) -> _fetch_params:
             result[arg] = session_value
 
     if overrides:
-        overrides = validate(overrides, FetchConfig).to_dict()
+        validated_config = validate(overrides, model)
-        overrides.update(result)
+        # Extract only the fields that _fetch_params needs from validated_config
-        return _fetch_params(**overrides)
+        validated_dict = {
+            f.name: getattr(validated_config, f.name)
+            for f in fields(_fetch_params)
+            if hasattr(validated_config, f.name)
+        }
+        # solve_cloudflare defaults to False for models that don't have it (PlaywrightConfig)
+        validated_dict.setdefault("solve_cloudflare", False)
 
-    if not result.get("solve_cloudflare"):
+        validated_dict.update(result)
-        result["solve_cloudflare"] = False
+        return _fetch_params(**validated_dict)
+
+    result.setdefault("solve_cloudflare", False)
 
     return _fetch_params(**result)
 
 
+# Cache default values for each model to reduce validation overhead
+models_default_values = {}
+
+for _model in (CamoufoxConfig, PlaywrightConfig):
+    _defaults = {}
+    if hasattr(_model, "__struct_defaults__") and hasattr(_model, "__struct_fields__"):
+        for field_name, default_value in zip(_model.__struct_fields__, _model.__struct_defaults__):  # type: ignore
+            # Skip factory defaults - these are msgspec._core.Factory instances
+            if type(default_value).__name__ != "Factory":
+                _defaults[field_name] = default_value
+
+    models_default_values[_model.__name__] = _defaults.copy()
+
+
+def _filter_defaults(params: Dict, model: str) -> Dict:
+    """Filter out parameters that match their default values to reduce validation overhead."""
+    defaults = models_default_values[model]
+    return {k: v for k, v in params.items() if k not in defaults or v != defaults[k]}
+
+
 @overload
 def validate(params: Dict, model: type[PlaywrightConfig]) -> PlaywrightConfig: ...
 
@@ -237,14 +252,10 @@ def validate(params: Dict, model: type[PlaywrightConfig]) -> PlaywrightConfig: .
 def validate(params: Dict, model: type[CamoufoxConfig]) -> CamoufoxConfig: ...
 
 
-@overload
+def validate(params: Dict, model: type[PlaywrightConfig] | type[CamoufoxConfig]) -> PlaywrightConfig | CamoufoxConfig:
-def validate(params: Dict, model: type[FetchConfig]) -> FetchConfig: ...
-
-
-def validate(
-    params: Dict, model: type[PlaywrightConfig] | type[CamoufoxConfig] | type[FetchConfig]
-) -> PlaywrightConfig | CamoufoxConfig | FetchConfig:
     try:
-        return convert(params, model)
+        # Filter out params with the default values (no need to validate them) to speed up validation
+        filtered = _filter_defaults(params, model.__name__)
+        return convert(filtered, model)
     except ValidationError as e:
         raise TypeError(f"Invalid argument type: {e}") from e

scrapling/engines/toolbelt/convertor.py

@@ -2,6 +2,7 @@ from functools import lru_cache
 from re import compile as re_compile
 
 from curl_cffi.requests import Response as CurlResponse
+from playwright._impl._errors import Error as PlaywrightError
 from playwright.sync_api import Page as SyncPage, Response as SyncResponse
 from playwright.async_api import Page as AsyncPage, Response as AsyncResponse
 
@@ -84,6 +85,7 @@ class ResponseFactory:
         first_response: SyncResponse,
         final_response: Optional[SyncResponse],
         parser_arguments: Dict,
+        automated_page: bool = False,
     ) -> Response:
         """
         Transforms a Playwright response into an internal `Response` object, encapsulating
@@ -99,6 +101,7 @@ class ResponseFactory:
         :param first_response: An earlier or initial Playwright `Response` object that may serve as a fallback response in the absence of the final one.
         :param parser_arguments: A dictionary containing additional arguments needed for parsing or further customization of the returned `Response`. These arguments are dynamically unpacked into
             the `Response` object.
+        :param automated_page: If True, it means the `page_action` argument was being used, so the response retrieving method changes to use Playwright's page instead of the final response.
 
         :return: A fully populated `Response` object containing the page's URL, content, status, headers, cookies, and other derived metadata.
         :rtype: Response
@@ -114,7 +117,7 @@ class ResponseFactory:
 
         history = cls._process_response_history(first_response, parser_arguments)
         try:
-            page_content = final_response.text()
+            page_content = final_response.text() if not automated_page else cls._get_page_content(page)
         except Exception as e:  # pragma: no cover
             log.error(f"Error getting page content: {e}")
             page_content = ""
@@ -179,6 +182,36 @@ class ResponseFactory:
 
         return history
 
+    @classmethod
+    def _get_page_content(cls, page: SyncPage) -> str:
+        """
+        A workaround for the Playwright issue with `page.content()` on Windows. Ref.: https://github.com/microsoft/playwright/issues/16108
+        :param page: The page to extract content from.
+        :return:
+        """
+        while True:
+            try:
+                return page.content() or ""
+            except PlaywrightError:
+                page.wait_for_timeout(500)
+                continue
+        return ""  # pyright: ignore
+
+    @classmethod
+    async def _get_async_page_content(cls, page: AsyncPage) -> str:
+        """
+        A workaround for the Playwright issue with `page.content()` on Windows. Ref.: https://github.com/microsoft/playwright/issues/16108
+        :param page: The page to extract content from.
+        :return:
+        """
+        while True:
+            try:
+                return (await page.content()) or ""
+            except PlaywrightError:
+                await page.wait_for_timeout(500)
+                continue
+        return ""  # pyright: ignore
+
     @classmethod
     async def from_async_playwright_response(
         cls,
@@ -186,6 +219,7 @@ class ResponseFactory:
         first_response: AsyncResponse,
         final_response: Optional[AsyncResponse],
         parser_arguments: Dict,
+        automated_page: bool = False,
     ) -> Response:
         """
         Transforms a Playwright response into an internal `Response` object, encapsulating
@@ -201,6 +235,7 @@ class ResponseFactory:
         :param first_response: An earlier or initial Playwright `Response` object that may serve as a fallback response in the absence of the final one.
         :param parser_arguments: A dictionary containing additional arguments needed for parsing or further customization of the returned `Response`. These arguments are dynamically unpacked into
             the `Response` object.
+        :param automated_page: If True, it means the `page_action` argument was being used, so the response retrieving method changes to use Playwright's page instead of the final response.
 
         :return: A fully populated `Response` object containing the page's URL, content, status, headers, cookies, and other derived metadata.
         :rtype: Response
@@ -216,7 +251,7 @@ class ResponseFactory:
 
         history = await cls._async_process_response_history(first_response, parser_arguments)
         try:
-            page_content = await final_response.text()
+            page_content = await (final_response.text() if not automated_page else cls._get_async_page_content(page))
         except Exception as e:  # pragma: no cover
             log.error(f"Error getting page content in async: {e}")
             page_content = ""

scrapling/engines/toolbelt/custom.py

@@ -209,15 +209,3 @@ class StatusText:
     def get(cls, status_code: int) -> str:
         """Get the phrase for a given HTTP status code."""
         return cls._phrases.get(status_code, "Unknown Status Code")
-
-
-def get_variable_name(var: Any) -> Optional[str]:
-    """Get the name of a variable using global and local scopes.
-    :param var: The variable to find the name for
-    :return: The name of the variable if found, None otherwise
-    """
-    for scope in [globals(), locals()]:
-        for name, value in scope.items():
-            if value is var:
-                return name
-    return None

scrapling/engines/toolbelt/fingerprints.py

@@ -7,8 +7,9 @@ from platform import system as platform_system
 
 from tldextract import extract
 from browserforge.headers import Browser, HeaderGenerator
+from browserforge.headers.generator import SUPPORTED_OPERATING_SYSTEMS
 
-from scrapling.core._types import Dict, Literal
+from scrapling.core._types import Dict, Literal, Tuple
 
 __OS_NAME__ = platform_system()
 OSName = Literal["linux", "macos", "windows"]
@@ -29,12 +30,12 @@ def generate_convincing_referer(url: str) -> str:
 
 
 @lru_cache(1, typed=True)
-def get_os_name() -> OSName | None:
+def get_os_name() -> OSName | Tuple:
     """Get the current OS name in the same format needed for browserforge, if the OS is Unknown, return None so browserforge uses all.
 
     :return: Current OS name or `None` otherwise
     """
-    match __OS_NAME__:
+    match __OS_NAME__:  # pragma: no cover
         case "Linux":
             return "linux"
         case "Darwin":
@@ -42,7 +43,7 @@ def get_os_name() -> OSName | None:
         case "Windows":
             return "windows"
         case _:
-            return None
+            return SUPPORTED_OPERATING_SYSTEMS
 
 
 def generate_headers(browser_mode: bool = False) -> Dict:
@@ -63,10 +64,7 @@ def generate_headers(browser_mode: bool = False) -> Dict:
                 Browser(name="edge", min_version=130),
             ]
         )
-    if os_name:
+    return HeaderGenerator(browser=browsers, os=os_name, device="desktop").generate()
-        return HeaderGenerator(browser=browsers, os=os_name, device="desktop").generate()
-    else:
-        return HeaderGenerator(browser=browsers, device="desktop").generate()
 
 
 __default_useragent__ = generate_headers(browser_mode=False).get("User-Agent")

scrapling/fetchers/chrome.py

@@ -50,6 +50,7 @@ class DynamicFetcher(BaseFetcher):
         network_idle: bool = False,
         load_dom: bool = True,
         wait_selector_state: SelectorWaitStates = "attached",
+        extra_flags: Optional[List[str]] = None,
         additional_args: Optional[Dict] = None,
         custom_config: Optional[Dict] = None,
     ) -> Response:
@@ -79,6 +80,7 @@ class DynamicFetcher(BaseFetcher):
         :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name.
         :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
         :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.
+        :param extra_flags: A list of additional browser flags to pass to the browser on launch.
         :param custom_config: A dictionary of custom parser arguments to use with this request. Any argument passed will override any class parameters values.
         :param additional_args: Additional arguments to be passed to Playwright's context as additional settings, and it takes higher priority than Scrapling's settings.
         :return: A `Response` object.
@@ -108,6 +110,7 @@ class DynamicFetcher(BaseFetcher):
             extra_headers=extra_headers,
             wait_selector=wait_selector,
             disable_webgl=disable_webgl,
+            extra_flags=extra_flags,
             additional_args=additional_args,
             disable_resources=disable_resources,
             wait_selector_state=wait_selector_state,
@@ -140,6 +143,7 @@ class DynamicFetcher(BaseFetcher):
         network_idle: bool = False,
         load_dom: bool = True,
         wait_selector_state: SelectorWaitStates = "attached",
+        extra_flags: Optional[List[str]] = None,
         additional_args: Optional[Dict] = None,
         custom_config: Optional[Dict] = None,
     ) -> Response:
@@ -169,6 +173,7 @@ class DynamicFetcher(BaseFetcher):
         :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name.
         :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
         :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.
+        :param extra_flags: A list of additional browser flags to pass to the browser on launch.
         :param custom_config: A dictionary of custom parser arguments to use with this request. Any argument passed will override any class parameters values.
         :param additional_args: Additional arguments to be passed to Playwright's context as additional settings, and it takes higher priority than Scrapling's settings.
         :return: A `Response` object.
@@ -199,6 +204,7 @@ class DynamicFetcher(BaseFetcher):
             extra_headers=extra_headers,
             wait_selector=wait_selector,
             disable_webgl=disable_webgl,
+            extra_flags=extra_flags,
             additional_args=additional_args,
             disable_resources=disable_resources,
             wait_selector_state=wait_selector_state,

setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = scrapling
-version = 0.3.7
+version = 0.3.8
 author = Karim Shoair
 author_email = karim.shoair@pm.me
 description = Scrapling is an undetectable, powerful, flexible, high-performance Python library that makes Web Scraping easy and effortless as it should be!