From a54ceb333590fb3d0d67518b460af3c92dd17911 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Sun, 19 Oct 2025 15:38:46 +0300 Subject: [PATCH 01/12] build: Pumping up the deps and the version --- pyproject.toml | 7 ++++--- scrapling/__init__.py | 2 +- setup.cfg | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 252f05d..c9d35f8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,8 +4,8 @@ build-backend = "setuptools.build_meta" [project] name = "scrapling" -# Static version instead of dynamic version so we can get better layer caching while building docker, check the docker file to understand -version = "0.3.7" +# Static version instead of a dynamic version so we can get better layer caching while building docker, check the docker file to understand +version = "0.3.8" description = "Scrapling is an undetectable, powerful, flexible, high-performance Python library that makes Web Scraping easy and effortless as it should be!" readme = {file = "README.md", content-type = "text/markdown"} license = {file = "LICENSE"} @@ -74,7 +74,7 @@ fetchers = [ "msgspec>=0.19.0", ] ai = [ - "mcp>=1.16.0", + "mcp>=1.18.0", "markdownify>=1.2.0", "scrapling[fetchers]", ] @@ -89,6 +89,7 @@ all = [ [project.urls] Homepage = "https://github.com/D4Vinci/Scrapling" +Changelog = "https://github.com/D4Vinci/Scrapling/releases" Documentation = "https://scrapling.readthedocs.io/en/latest/" Repository = "https://github.com/D4Vinci/Scrapling" "Bug Tracker" = "https://github.com/D4Vinci/Scrapling/issues" diff --git a/scrapling/__init__.py b/scrapling/__init__.py index cbc3a2c..2122396 100644 --- a/scrapling/__init__.py +++ b/scrapling/__init__.py @@ -1,5 +1,5 @@ __author__ = "Karim Shoair (karim.shoair@pm.me)" -__version__ = "0.3.7" +__version__ = "0.3.8" __copyright__ = "Copyright (c) 2024 Karim Shoair" from typing import Any, TYPE_CHECKING diff --git a/setup.cfg b/setup.cfg index cd31254..fddbe8f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = scrapling -version = 0.3.7 +version = 0.3.8 author = Karim Shoair author_email = karim.shoair@pm.me description = Scrapling is an undetectable, powerful, flexible, high-performance Python library that makes Web Scraping easy and effortless as it should be! From 43db79d1df50d2b5aa08b13a45d3d585c0d91985 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Sat, 25 Oct 2025 22:29:07 +0300 Subject: [PATCH 02/12] docs: adding cloud browser segment --- docs/fetching/dynamic.md | 38 +++++++++++++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/docs/fetching/dynamic.md b/docs/fetching/dynamic.md index 3075fdc..18c39d7 100644 --- a/docs/fetching/dynamic.md +++ b/docs/fetching/dynamic.md @@ -300,4 +300,40 @@ Use DynamicFetcher when: - Need custom browser config - Want flexible stealth options -If you want more stealth and control without much config, check out the [StealthyFetcher](stealthy.md). \ No newline at end of file +If you want more stealth and control without much config, check out the [StealthyFetcher](stealthy.md). + +## External Cloud Browser Version + +If you have issues with the browser installation, such as resource management, we recommend you try the Cloud Browser from [Scrapeless](https://www.scrapeless.com/en/product/scraping-browser) for free! + +The usage is straightforward: create an account and [get your API key](https://docs.scrapeless.com/en/scraping-browser/quickstart/getting-started/), then pass it to the `DynamicSession` like this: + +```python +from urllib.parse import urlencode + +from scrapling.fetchers import DynamicSession + +# Configure your browser session +config = { + "token": "YOUR_API_KEY", + "sessionName": "scrapling-session", + "sessionTTL": "300", # 5 minutes + "proxyCountry": "ANY", + "sessionRecording": "false", +} + +# Build WebSocket URL +ws_endpoint = f"wss://browser.scrapeless.com/api/v2/browser?{urlencode(config)}" +print('Connecting to Scrapeless...') + +with DynamicSession(cdp_url=ws_endpoint, disable_resources=True) as s: + print("Connected!") + page = s.fetch("https://httpbin.org/headers", network_idle=True) + print(f"Page loaded, content length: {len(page.body)}") + print(page.json()) +``` +The `DynamicSession` class instance will work as usual, so no further explanation is needed. + +However, the Scrapeless Cloud Browser can be configured with proxy options, like the proxy country in the config above, [custom fingerprint](https://docs.scrapeless.com/en/scraping-browser/features/advanced-privacy-anti-detection/custom-fingerprint/) configuration, [captcha solving](https://docs.scrapeless.com/en/scraping-browser/features/advanced-privacy-anti-detection/supported-captchas/), and more. + +Check out the [Scrapeless's browser documentation](https://docs.scrapeless.com/en/scraping-browser/quickstart/introduction/) for more details. \ No newline at end of file From 74fa45daea9e329231c80a22cf09b5e2ea80aeb2 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Sun, 26 Oct 2025 00:41:45 +0300 Subject: [PATCH 03/12] fix: Addressing the issue of collecting response after `page_action` in #100 --- scrapling/engines/_browsers/_base.py | 5 ++- scrapling/engines/_browsers/_camoufox.py | 46 ++++----------------- scrapling/engines/_browsers/_controllers.py | 4 +- scrapling/engines/toolbelt/convertor.py | 39 ++++++++++++++++- 4 files changed, 50 insertions(+), 44 deletions(-) diff --git a/scrapling/engines/_browsers/_base.py b/scrapling/engines/_browsers/_base.py index 9a6dfae..e01a820 100644 --- a/scrapling/engines/_browsers/_base.py +++ b/scrapling/engines/_browsers/_base.py @@ -53,7 +53,9 @@ class SyncSession: for script in _compiled_stealth_scripts(): page.add_init_script(script=script) - return self.page_pool.add_page(page) + page_info = self.page_pool.add_page(page) + page_info.mark_busy() + return page_info def get_pool_stats(self) -> Dict[str, int]: """Get statistics about the current page pool""" @@ -97,7 +99,6 @@ class AsyncSession: f"No pages finished to clear place in the pool within the {self._max_wait_for_page}s timeout period" ) - assert self.context is not None, "Browser context not initialized" page = await self.context.new_page() page.set_default_navigation_timeout(timeout) page.set_default_timeout(timeout) diff --git a/scrapling/engines/_browsers/_camoufox.py b/scrapling/engines/_browsers/_camoufox.py index 94e8d37..e66a826 100644 --- a/scrapling/engines/_browsers/_camoufox.py +++ b/scrapling/engines/_browsers/_camoufox.py @@ -206,21 +206,6 @@ class StealthySession(StealthySessionMixin, SyncSession): self._closed = True - @staticmethod - def _get_page_content(page: Page) -> str: - """ - A workaround for the Playwright issue with `page.content()` on Windows. Ref.: https://github.com/microsoft/playwright/issues/16108 - :param page: The page to extract content from. - :return: - """ - while True: - try: - return page.content() or "" - except PlaywrightError: - page.wait_for_timeout(1000) - continue - return "" # pyright: ignore - def _solve_cloudflare(self, page: Page) -> None: # pragma: no cover """Solve the cloudflare challenge displayed on the playwright page passed @@ -231,14 +216,14 @@ class StealthySession(StealthySessionMixin, SyncSession): page.wait_for_load_state("networkidle", timeout=5000) except PlaywrightError: pass - challenge_type = self._detect_cloudflare(self._get_page_content(page)) + challenge_type = self._detect_cloudflare(ResponseFactory._get_page_content(page)) if not challenge_type: log.error("No Cloudflare challenge found.") return else: log.info(f'The turnstile version discovered is "{challenge_type}"') if challenge_type == "non-interactive": - while "Just a moment..." in (self._get_page_content(page)): + while "Just a moment..." in (ResponseFactory._get_page_content(page)): log.info("Waiting for Cloudflare wait page to disappear.") page.wait_for_timeout(1000) page.wait_for_load_state() @@ -249,7 +234,7 @@ class StealthySession(StealthySessionMixin, SyncSession): box_selector = "#cf_turnstile div, #cf-turnstile div, .turnstile>div>div" if challenge_type != "embedded": box_selector = ".main-content p+div>div>div" - while "Verifying you are human." in self._get_page_content(page): + while "Verifying you are human." in ResponseFactory._get_page_content(page): # Waiting for the verify spinner to disappear, checking every 1s if it disappeared page.wait_for_timeout(500) @@ -403,7 +388,7 @@ class StealthySession(StealthySessionMixin, SyncSession): page_info.page.wait_for_timeout(params.wait) response = ResponseFactory.from_playwright_response( - page_info.page, first_response, final_response, params.selector_config + page_info.page, first_response, final_response, params.selector_config, bool(params.page_action) ) # Close the page to free up resources @@ -550,21 +535,6 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession): self._closed = True - @staticmethod - async def _get_page_content(page: async_Page) -> str: - """ - A workaround for the Playwright issue with `page.content()` on Windows. Ref.: https://github.com/microsoft/playwright/issues/16108 - :param page: The page to extract content from. - :return: - """ - while True: - try: - return (await page.content()) or "" - except PlaywrightError: - await page.wait_for_timeout(1000) - continue - return "" # pyright: ignore - async def _solve_cloudflare(self, page: async_Page): """Solve the cloudflare challenge displayed on the playwright page passed. The async version @@ -575,14 +545,14 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession): await page.wait_for_load_state("networkidle", timeout=5000) except PlaywrightError: pass - challenge_type = self._detect_cloudflare(await self._get_page_content(page)) + challenge_type = self._detect_cloudflare(await ResponseFactory._get_async_page_content(page)) if not challenge_type: log.error("No Cloudflare challenge found.") return else: log.info(f'The turnstile version discovered is "{challenge_type}"') if challenge_type == "non-interactive": # pragma: no cover - while "Just a moment..." in (await self._get_page_content(page)): + while "Just a moment..." in (await ResponseFactory._get_async_page_content(page)): log.info("Waiting for Cloudflare wait page to disappear.") await page.wait_for_timeout(1000) await page.wait_for_load_state() @@ -593,7 +563,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession): box_selector = "#cf_turnstile div, #cf-turnstile div, .turnstile>div>div" if challenge_type != "embedded": box_selector = ".main-content p+div>div>div" - while "Verifying you are human." in (await self._get_page_content(page)): + while "Verifying you are human." in (await ResponseFactory._get_async_page_content(page)): # Waiting for the verify spinner to disappear, checking every 1s if it disappeared await page.wait_for_timeout(500) @@ -753,7 +723,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession): # Create response object response = await ResponseFactory.from_async_playwright_response( - page_info.page, first_response, final_response, params.selector_config + page_info.page, first_response, final_response, params.selector_config, bool(params.page_action) ) # Close the page to free up resources diff --git a/scrapling/engines/_browsers/_controllers.py b/scrapling/engines/_browsers/_controllers.py index ca8b45e..45a6938 100644 --- a/scrapling/engines/_browsers/_controllers.py +++ b/scrapling/engines/_browsers/_controllers.py @@ -306,7 +306,7 @@ class DynamicSession(DynamicSessionMixin, SyncSession): # Create response object response = ResponseFactory.from_playwright_response( - page_info.page, first_response, final_response, params.selector_config + page_info.page, first_response, final_response, params.selector_config, bool(params.page_action) ) # Close the page to free up resources @@ -563,7 +563,7 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession): # Create response object response = await ResponseFactory.from_async_playwright_response( - page_info.page, first_response, final_response, params.selector_config + page_info.page, first_response, final_response, params.selector_config, bool(params.page_action) ) # Close the page to free up resources diff --git a/scrapling/engines/toolbelt/convertor.py b/scrapling/engines/toolbelt/convertor.py index b66b518..5bb4a49 100644 --- a/scrapling/engines/toolbelt/convertor.py +++ b/scrapling/engines/toolbelt/convertor.py @@ -2,6 +2,7 @@ from functools import lru_cache from re import compile as re_compile from curl_cffi.requests import Response as CurlResponse +from playwright._impl._errors import Error as PlaywrightError from playwright.sync_api import Page as SyncPage, Response as SyncResponse from playwright.async_api import Page as AsyncPage, Response as AsyncResponse @@ -84,6 +85,7 @@ class ResponseFactory: first_response: SyncResponse, final_response: Optional[SyncResponse], parser_arguments: Dict, + automated_page: bool = False, ) -> Response: """ Transforms a Playwright response into an internal `Response` object, encapsulating @@ -99,6 +101,7 @@ class ResponseFactory: :param first_response: An earlier or initial Playwright `Response` object that may serve as a fallback response in the absence of the final one. :param parser_arguments: A dictionary containing additional arguments needed for parsing or further customization of the returned `Response`. These arguments are dynamically unpacked into the `Response` object. + :param automated_page: If True, it means the `page_action` argument was being used, so the response retrieving method changes to use Playwright's page instead of the final response. :return: A fully populated `Response` object containing the page's URL, content, status, headers, cookies, and other derived metadata. :rtype: Response @@ -114,7 +117,7 @@ class ResponseFactory: history = cls._process_response_history(first_response, parser_arguments) try: - page_content = final_response.text() + page_content = final_response.text() if not automated_page else cls._get_page_content(page) except Exception as e: # pragma: no cover log.error(f"Error getting page content: {e}") page_content = "" @@ -179,6 +182,36 @@ class ResponseFactory: return history + @classmethod + def _get_page_content(cls, page: SyncPage) -> str: + """ + A workaround for the Playwright issue with `page.content()` on Windows. Ref.: https://github.com/microsoft/playwright/issues/16108 + :param page: The page to extract content from. + :return: + """ + while True: + try: + return page.content() or "" + except PlaywrightError: + page.wait_for_timeout(500) + continue + return "" # pyright: ignore + + @classmethod + async def _get_async_page_content(cls, page: AsyncPage) -> str: + """ + A workaround for the Playwright issue with `page.content()` on Windows. Ref.: https://github.com/microsoft/playwright/issues/16108 + :param page: The page to extract content from. + :return: + """ + while True: + try: + return (await page.content()) or "" + except PlaywrightError: + await page.wait_for_timeout(500) + continue + return "" # pyright: ignore + @classmethod async def from_async_playwright_response( cls, @@ -186,6 +219,7 @@ class ResponseFactory: first_response: AsyncResponse, final_response: Optional[AsyncResponse], parser_arguments: Dict, + automated_page: bool = False, ) -> Response: """ Transforms a Playwright response into an internal `Response` object, encapsulating @@ -201,6 +235,7 @@ class ResponseFactory: :param first_response: An earlier or initial Playwright `Response` object that may serve as a fallback response in the absence of the final one. :param parser_arguments: A dictionary containing additional arguments needed for parsing or further customization of the returned `Response`. These arguments are dynamically unpacked into the `Response` object. + :param automated_page: If True, it means the `page_action` argument was being used, so the response retrieving method changes to use Playwright's page instead of the final response. :return: A fully populated `Response` object containing the page's URL, content, status, headers, cookies, and other derived metadata. :rtype: Response @@ -216,7 +251,7 @@ class ResponseFactory: history = await cls._async_process_response_history(first_response, parser_arguments) try: - page_content = await final_response.text() + page_content = await (final_response.text() if not automated_page else cls._get_async_page_content(page)) except Exception as e: # pragma: no cover log.error(f"Error getting page content in async: {e}") page_content = "" From 59ec6be20108350fcc03ad0696545a49175d1c2c Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Sun, 26 Oct 2025 05:04:19 +0300 Subject: [PATCH 04/12] refactor(fetchers): Less duplicated code and better handling for unstable websites - Websites that never finish loading their requests won't crash the code now if you used `network_idle` with them - Fixed a typo in `load_dom` in DynamicSession's async_fetch - Removed dead code --- scrapling/engines/_browsers/_base.py | 146 +++++++++++++++++-- scrapling/engines/_browsers/_camoufox.py | 150 ++++---------------- scrapling/engines/_browsers/_controllers.py | 111 +++------------ scrapling/engines/toolbelt/custom.py | 12 -- scrapling/engines/toolbelt/fingerprints.py | 14 +- 5 files changed, 185 insertions(+), 248 deletions(-) diff --git a/scrapling/engines/_browsers/_base.py b/scrapling/engines/_browsers/_base.py index e01a820..4157494 100644 --- a/scrapling/engines/_browsers/_base.py +++ b/scrapling/engines/_browsers/_base.py @@ -2,17 +2,27 @@ from time import time from asyncio import sleep as asyncio_sleep, Lock from camoufox import DefaultAddons -from playwright.sync_api import BrowserContext, Playwright -from playwright.async_api import ( - BrowserContext as AsyncBrowserContext, - Playwright as AsyncPlaywright, +from playwright.sync_api import ( + Page, + Frame, + BrowserContext, + Playwright, + Response as SyncPlaywrightResponse, ) +from playwright.async_api import ( + Page as AsyncPage, + Frame as AsyncFrame, + Playwright as AsyncPlaywright, + Response as AsyncPlaywrightResponse, + BrowserContext as AsyncBrowserContext, +) +from playwright._impl._errors import Error as PlaywrightError from camoufox.pkgman import installed_verstr as camoufox_version from camoufox.utils import launch_options as generate_launch_options from ._page import PageInfo, PagePool from scrapling.parser import Selector -from scrapling.core._types import Any, cast, Dict, Optional, TYPE_CHECKING +from scrapling.core._types import Any, cast, Dict, List, Optional, Callable, TYPE_CHECKING from scrapling.engines.toolbelt.fingerprints import get_os_name from ._validators import validate, PlaywrightConfig, CamoufoxConfig from ._config_tools import _compiled_stealth_scripts, _launch_kwargs, _context_kwargs @@ -26,10 +36,35 @@ class SyncSession: self.max_pages = max_pages self.page_pool = PagePool(max_pages) self._max_wait_for_page = 60 - self.playwright: Optional[Playwright] = None - self.context: Optional[BrowserContext] = None + self.playwright: Playwright | Any = None + self.context: BrowserContext | Any = None self._closed = False + def __create__(self): + pass + + def close(self): # pragma: no cover + """Close all resources""" + if self._closed: + return + + if self.context: + self.context.close() + self.context = None + + if self.playwright: + self.playwright.stop() + self.playwright = None # pyright: ignore + + self._closed = True + + def __enter__(self): + self.__create__() + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.close() + def _get_page( self, timeout: int | float, @@ -65,17 +100,76 @@ class SyncSession: "max_pages": self.max_pages, } + @staticmethod + def _wait_for_networkidle(page: Page | Frame, timeout: Optional[int] = None): + """Wait for the page to become idle (no network activity) even if there are never-ending requests.""" + try: + page.wait_for_load_state("networkidle", timeout=timeout) + except PlaywrightError: + pass + + def _wait_for_page_stability(self, page: Page | Frame, load_dom: bool, network_idle: bool): + page.wait_for_load_state(state="load") + if load_dom: + page.wait_for_load_state(state="domcontentloaded") + if network_idle: + self._wait_for_networkidle(page) + + @staticmethod + def _create_response_handler(page_info: PageInfo, response_container: List) -> Callable: + """Create a response handler that captures the final navigation response. + + :param page_info: The PageInfo object containing the page + :param response_container: A list to store the final response (mutable container) + :return: A callback function for page.on("response", ...) + """ + + def handle_response(finished_response: SyncPlaywrightResponse): + if ( + finished_response.request.resource_type == "document" + and finished_response.request.is_navigation_request() + and finished_response.request.frame == page_info.page.main_frame + ): + response_container[0] = finished_response + + return handle_response + class AsyncSession: def __init__(self, max_pages: int = 1): self.max_pages = max_pages self.page_pool = PagePool(max_pages) self._max_wait_for_page = 60 - self.playwright: Optional[AsyncPlaywright] = None - self.context: Optional[AsyncBrowserContext] = None + self.playwright: AsyncPlaywright | Any = None + self.context: AsyncBrowserContext | Any = None self._closed = False self._lock = Lock() + async def __create__(self): + pass + + async def close(self): + """Close all resources""" + if self._closed: # pragma: no cover + return + + if self.context: + await self.context.close() + self.context = None # pyright: ignore + + if self.playwright: + await self.playwright.stop() + self.playwright = None # pyright: ignore + + self._closed = True + + async def __aenter__(self): + await self.__create__() + return self + + async def __aexit__(self, exc_type, exc_val, exc_tb): + await self.close() + async def _get_page( self, timeout: int | float, @@ -122,6 +216,40 @@ class AsyncSession: "max_pages": self.max_pages, } + @staticmethod + async def _wait_for_networkidle(page: AsyncPage | AsyncFrame, timeout: Optional[int] = None): + """Wait for the page to become idle (no network activity) even if there are never-ending requests.""" + try: + await page.wait_for_load_state("networkidle", timeout=timeout) + except PlaywrightError: + pass + + async def _wait_for_page_stability(self, page: AsyncPage | AsyncFrame, load_dom: bool, network_idle: bool): + await page.wait_for_load_state(state="load") + if load_dom: + await page.wait_for_load_state(state="domcontentloaded") + if network_idle: + await self._wait_for_networkidle(page) + + @staticmethod + def _create_response_handler(page_info: PageInfo, response_container: List) -> Callable: + """Create an async response handler that captures the final navigation response. + + :param page_info: The PageInfo object containing the page + :param response_container: A list to store the final response (mutable container) + :return: A callback function for page.on("response", ...) + """ + + async def handle_response(finished_response: AsyncPlaywrightResponse): + if ( + finished_response.request.resource_type == "document" + and finished_response.request.is_navigation_request() + and finished_response.request.frame == page_info.page.main_frame + ): + response_container[0] = finished_response + + return handle_response + class DynamicSessionMixin: def __validate__(self, **params): diff --git a/scrapling/engines/_browsers/_camoufox.py b/scrapling/engines/_browsers/_camoufox.py index e66a826..ac66870 100644 --- a/scrapling/engines/_browsers/_camoufox.py +++ b/scrapling/engines/_browsers/_camoufox.py @@ -2,18 +2,16 @@ from random import randint from re import compile as re_compile from playwright.sync_api import ( - Response as SyncPlaywrightResponse, - sync_playwright, - Locator, Page, + Locator, + sync_playwright, ) from playwright.async_api import ( async_playwright, - Response as AsyncPlaywrightResponse, - BrowserContext as AsyncBrowserContext, - Playwright as AsyncPlaywright, - Locator as AsyncLocator, Page as async_Page, + Locator as AsyncLocator, + Playwright as AsyncPlaywright, + BrowserContext as AsyncBrowserContext, ) from playwright._impl._errors import Error as PlaywrightError @@ -184,38 +182,13 @@ class StealthySession(StealthySessionMixin, SyncSession): if self.cookies: # pragma: no cover self.context.add_cookies(self.cookies) - def __enter__(self): # pragma: no cover - self.__create__() - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - self.close() - - def close(self): # pragma: no cover - """Close all resources""" - if self._closed: # pragma: no cover - return - - if self.context: - self.context.close() - self.context = None - - if self.playwright: - self.playwright.stop() - self.playwright = None - - self._closed = True - def _solve_cloudflare(self, page: Page) -> None: # pragma: no cover """Solve the cloudflare challenge displayed on the playwright page passed :param page: The targeted page :return: """ - try: - page.wait_for_load_state("networkidle", timeout=5000) - except PlaywrightError: - pass + self._wait_for_networkidle(page, timeout=5000) challenge_type = self._detect_cloudflare(ResponseFactory._get_page_content(page)) if not challenge_type: log.error("No Cloudflare challenge found.") @@ -241,8 +214,7 @@ class StealthySession(StealthySessionMixin, SyncSession): outer_box = {} iframe = page.frame(url=__CF_PATTERN__) if iframe is not None: - iframe.wait_for_load_state(state="domcontentloaded") - iframe.wait_for_load_state("networkidle") + self._wait_for_page_stability(iframe, True, True) if challenge_type != "embedded": while not iframe.frame_element().is_visible(): @@ -258,7 +230,7 @@ class StealthySession(StealthySessionMixin, SyncSession): # Move the mouse to the center of the window, then press and hold the left mouse button page.mouse.click(captcha_x, captcha_y, delay=60, button="left") - page.wait_for_load_state("networkidle") + self._wait_for_networkidle(page) if iframe is not None: # Wait for the frame to be removed from the page while iframe in page.frames: @@ -266,8 +238,7 @@ class StealthySession(StealthySessionMixin, SyncSession): if challenge_type != "embedded": page.locator(box_selector).last.wait_for(state="detached") page.locator(".zone-name-title").wait_for(state="hidden") - page.wait_for_load_state(state="load") - page.wait_for_load_state(state="domcontentloaded") + self._wait_for_page_stability(page, True, False) log.info("Cloudflare captcha is solved") return @@ -328,32 +299,19 @@ class StealthySession(StealthySessionMixin, SyncSession): if self._closed: # pragma: no cover raise RuntimeError("Context manager has been closed") - final_response = None referer = ( generate_convincing_referer(url) if (params.google_search and "referer" not in self._headers_keys) else None ) - def handle_response(finished_response: SyncPlaywrightResponse): - nonlocal final_response - if ( - finished_response.request.resource_type == "document" - and finished_response.request.is_navigation_request() - and finished_response.request.frame == page_info.page.main_frame - ): - final_response = finished_response - page_info = self._get_page(params.timeout, params.extra_headers, params.disable_resources) - page_info.mark_busy(url=url) + final_response = [None] + handle_response = self._create_response_handler(page_info, final_response) try: # pragma: no cover # Navigate to URL and wait for a specified state page_info.page.on("response", handle_response) first_response = page_info.page.goto(url, referer=referer) - if params.load_dom: - page_info.page.wait_for_load_state(state="domcontentloaded") - - if params.network_idle: - page_info.page.wait_for_load_state("networkidle") + self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle) if not first_response: raise RuntimeError(f"Failed to get response for {url}") @@ -361,11 +319,7 @@ class StealthySession(StealthySessionMixin, SyncSession): if params.solve_cloudflare: self._solve_cloudflare(page_info.page) # Make sure the page is fully loaded after the captcha - page_info.page.wait_for_load_state(state="load") - if params.load_dom: - page_info.page.wait_for_load_state(state="domcontentloaded") - if params.network_idle: - page_info.page.wait_for_load_state("networkidle") + self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle) if params.page_action: try: @@ -378,17 +332,13 @@ class StealthySession(StealthySessionMixin, SyncSession): waiter: Locator = page_info.page.locator(params.wait_selector) waiter.first.wait_for(state=params.wait_selector_state) # Wait again after waiting for the selector, helpful with protections like Cloudflare - page_info.page.wait_for_load_state(state="load") - if params.load_dom: - page_info.page.wait_for_load_state(state="domcontentloaded") - if params.network_idle: - page_info.page.wait_for_load_state("networkidle") + self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle) except Exception as e: log.error(f"Error waiting for selector {params.wait_selector}: {e}") page_info.page.wait_for_timeout(params.wait) response = ResponseFactory.from_playwright_response( - page_info.page, first_response, final_response, params.selector_config, bool(params.page_action) + page_info.page, first_response, final_response[0], params.selector_config, bool(params.page_action) ) # Close the page to free up resources @@ -513,38 +463,13 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession): if self.cookies: await self.context.add_cookies(self.cookies) # pyright: ignore [reportArgumentType] - async def __aenter__(self): - await self.__create__() - return self - - async def __aexit__(self, exc_type, exc_val, exc_tb): - await self.close() - - async def close(self): - """Close all resources""" - if self._closed: # pragma: no cover - return - - if self.context: - await self.context.close() - self.context = None # pyright: ignore - - if self.playwright: - await self.playwright.stop() - self.playwright = None # pyright: ignore - - self._closed = True - - async def _solve_cloudflare(self, page: async_Page): + async def _solve_cloudflare(self, page: async_Page): # pragma: no cover """Solve the cloudflare challenge displayed on the playwright page passed. The async version :param page: The async targeted page :return: """ - try: - await page.wait_for_load_state("networkidle", timeout=5000) - except PlaywrightError: - pass + await self._wait_for_networkidle(page, timeout=5000) challenge_type = self._detect_cloudflare(await ResponseFactory._get_async_page_content(page)) if not challenge_type: log.error("No Cloudflare challenge found.") @@ -570,8 +495,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession): outer_box = {} iframe = page.frame(url=__CF_PATTERN__) if iframe is not None: - await iframe.wait_for_load_state(state="domcontentloaded") - await iframe.wait_for_load_state("networkidle") + await self._wait_for_page_stability(iframe, True, True) if challenge_type != "embedded": while not await (await iframe.frame_element()).is_visible(): @@ -587,7 +511,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession): # Move the mouse to the center of the window, then press and hold the left mouse button await page.mouse.click(captcha_x, captcha_y, delay=60, button="left") - await page.wait_for_load_state("networkidle") + await self._wait_for_networkidle(page) if iframe is not None: # Wait for the frame to be removed from the page while iframe in page.frames: @@ -595,8 +519,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession): if challenge_type != "embedded": await page.locator(box_selector).wait_for(state="detached") await page.locator(".zone-name-title").wait_for(state="hidden") - await page.wait_for_load_state(state="load") - await page.wait_for_load_state(state="domcontentloaded") + await self._wait_for_page_stability(page, True, False) log.info("Cloudflare captcha is solved") return @@ -657,22 +580,13 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession): if self._closed: # pragma: no cover raise RuntimeError("Context manager has been closed") - final_response = None referer = ( generate_convincing_referer(url) if (params.google_search and "referer" not in self._headers_keys) else None ) - async def handle_response(finished_response: AsyncPlaywrightResponse): - nonlocal final_response - if ( - finished_response.request.resource_type == "document" - and finished_response.request.is_navigation_request() - and finished_response.request.frame == page_info.page.main_frame - ): - final_response = finished_response - page_info = await self._get_page(params.timeout, params.extra_headers, params.disable_resources) - page_info.mark_busy(url=url) + final_response = [None] + handle_response = self._create_response_handler(page_info, final_response) if TYPE_CHECKING: if not isinstance(page_info.page, async_Page): @@ -682,11 +596,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession): # Navigate to URL and wait for a specified state page_info.page.on("response", handle_response) first_response = await page_info.page.goto(url, referer=referer) - if params.load_dom: - await page_info.page.wait_for_load_state(state="domcontentloaded") - - if params.network_idle: - await page_info.page.wait_for_load_state("networkidle") + await self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle) if not first_response: raise RuntimeError(f"Failed to get response for {url}") @@ -694,11 +604,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession): if params.solve_cloudflare: await self._solve_cloudflare(page_info.page) # Make sure the page is fully loaded after the captcha - await page_info.page.wait_for_load_state(state="load") - if params.load_dom: - await page_info.page.wait_for_load_state(state="domcontentloaded") - if params.network_idle: - await page_info.page.wait_for_load_state("networkidle") + await self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle) if params.page_action: try: @@ -711,11 +617,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession): waiter: AsyncLocator = page_info.page.locator(params.wait_selector) await waiter.first.wait_for(state=params.wait_selector_state) # Wait again after waiting for the selector, helpful with protections like Cloudflare - await page_info.page.wait_for_load_state(state="load") - if params.load_dom: - await page_info.page.wait_for_load_state(state="domcontentloaded") - if params.network_idle: - await page_info.page.wait_for_load_state("networkidle") + await self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle) except Exception as e: log.error(f"Error waiting for selector {params.wait_selector}: {e}") @@ -723,7 +625,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession): # Create response object response = await ResponseFactory.from_async_playwright_response( - page_info.page, first_response, final_response, params.selector_config, bool(params.page_action) + page_info.page, first_response, final_response[0], params.selector_config, bool(params.page_action) ) # Close the page to free up resources diff --git a/scrapling/engines/_browsers/_controllers.py b/scrapling/engines/_browsers/_controllers.py index 45a6938..155789a 100644 --- a/scrapling/engines/_browsers/_controllers.py +++ b/scrapling/engines/_browsers/_controllers.py @@ -1,16 +1,13 @@ from playwright.sync_api import ( - Response as SyncPlaywrightResponse, - sync_playwright, - Playwright, Locator, + Playwright, + sync_playwright, ) from playwright.async_api import ( async_playwright, - Response as AsyncPlaywrightResponse, - BrowserContext as AsyncBrowserContext, - Playwright as AsyncPlaywright, Locator as AsyncLocator, - Page as async_Page, + Playwright as AsyncPlaywright, + BrowserContext as AsyncBrowserContext, ) from patchright.sync_api import sync_playwright as sync_patchright from patchright.async_api import async_playwright as async_patchright @@ -178,28 +175,6 @@ class DynamicSession(DynamicSessionMixin, SyncSession): if self.cookies: # pragma: no cover self.context.add_cookies(self.cookies) - def __enter__(self): - self.__create__() - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - self.close() - - def close(self): # pragma: no cover - """Close all resources""" - if self._closed: - return - - if self.context: - self.context.close() - self.context = None - - if self.playwright: - self.playwright.stop() - self.playwright = None # pyright: ignore - - self._closed = True - def fetch( self, url: str, @@ -253,32 +228,19 @@ class DynamicSession(DynamicSessionMixin, SyncSession): if self._closed: # pragma: no cover raise RuntimeError("Context manager has been closed") - final_response = None referer = ( generate_convincing_referer(url) if (params.google_search and "referer" not in self._headers_keys) else None ) - def handle_response(finished_response: SyncPlaywrightResponse): - nonlocal final_response - if ( - finished_response.request.resource_type == "document" - and finished_response.request.is_navigation_request() - and finished_response.request.frame == page_info.page.main_frame - ): - final_response = finished_response - page_info = self._get_page(params.timeout, params.extra_headers, params.disable_resources) - page_info.mark_busy(url=url) + final_response = [None] + handle_response = self._create_response_handler(page_info, final_response) try: # pragma: no cover # Navigate to URL and wait for a specified state page_info.page.on("response", handle_response) first_response = page_info.page.goto(url, referer=referer) - if params.load_dom: - page_info.page.wait_for_load_state(state="domcontentloaded") - - if params.network_idle: - page_info.page.wait_for_load_state("networkidle") + self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle) if not first_response: raise RuntimeError(f"Failed to get response for {url}") @@ -294,11 +256,7 @@ class DynamicSession(DynamicSessionMixin, SyncSession): waiter: Locator = page_info.page.locator(params.wait_selector) waiter.first.wait_for(state=params.wait_selector_state) # Wait again after waiting for the selector, helpful with protections like Cloudflare - page_info.page.wait_for_load_state(state="load") - if params.load_dom: - page_info.page.wait_for_load_state(state="domcontentloaded") - if params.network_idle: - page_info.page.wait_for_load_state("networkidle") + self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle) except Exception as e: # pragma: no cover log.error(f"Error waiting for selector {params.wait_selector}: {e}") @@ -306,7 +264,7 @@ class DynamicSession(DynamicSessionMixin, SyncSession): # Create response object response = ResponseFactory.from_playwright_response( - page_info.page, first_response, final_response, params.selector_config, bool(params.page_action) + page_info.page, first_response, final_response[0], params.selector_config, bool(params.page_action) ) # Close the page to free up resources @@ -431,28 +389,6 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession): if self.cookies: await self.context.add_cookies(self.cookies) # pyright: ignore - async def __aenter__(self): - await self.__create__() - return self - - async def __aexit__(self, exc_type, exc_val, exc_tb): - await self.close() - - async def close(self): - """Close all resources""" - if self._closed: # pragma: no cover - return - - if self.context: - await self.context.close() - self.context = None # pyright: ignore - - if self.playwright: - await self.playwright.stop() - self.playwright = None # pyright: ignore - - self._closed = True - async def fetch( self, url: str, @@ -506,24 +442,17 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession): if self._closed: # pragma: no cover raise RuntimeError("Context manager has been closed") - final_response = None referer = ( generate_convincing_referer(url) if (params.google_search and "referer" not in self._headers_keys) else None ) - async def handle_response(finished_response: AsyncPlaywrightResponse): - nonlocal final_response - if ( - finished_response.request.resource_type == "document" - and finished_response.request.is_navigation_request() - and finished_response.request.frame == page_info.page.main_frame - ): - final_response = finished_response - page_info = await self._get_page(params.timeout, params.extra_headers, params.disable_resources) - page_info.mark_busy(url=url) + final_response = [None] + handle_response = self._create_response_handler(page_info, final_response) if TYPE_CHECKING: + from playwright.async_api import Page as async_Page + if not isinstance(page_info.page, async_Page): raise TypeError @@ -531,11 +460,7 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession): # Navigate to URL and wait for a specified state page_info.page.on("response", handle_response) first_response = await page_info.page.goto(url, referer=referer) - if self.load_dom: - await page_info.page.wait_for_load_state(state="domcontentloaded") - - if params.network_idle: - await page_info.page.wait_for_load_state("networkidle") + await self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle) if not first_response: raise RuntimeError(f"Failed to get response for {url}") @@ -551,11 +476,7 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession): waiter: AsyncLocator = page_info.page.locator(params.wait_selector) await waiter.first.wait_for(state=params.wait_selector_state) # Wait again after waiting for the selector, helpful with protections like Cloudflare - await page_info.page.wait_for_load_state(state="load") - if self.load_dom: - await page_info.page.wait_for_load_state(state="domcontentloaded") - if params.network_idle: - await page_info.page.wait_for_load_state("networkidle") + await self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle) except Exception as e: log.error(f"Error waiting for selector {params.wait_selector}: {e}") @@ -563,7 +484,7 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession): # Create response object response = await ResponseFactory.from_async_playwright_response( - page_info.page, first_response, final_response, params.selector_config, bool(params.page_action) + page_info.page, first_response, final_response[0], params.selector_config, bool(params.page_action) ) # Close the page to free up resources diff --git a/scrapling/engines/toolbelt/custom.py b/scrapling/engines/toolbelt/custom.py index 43ef61c..1f20038 100644 --- a/scrapling/engines/toolbelt/custom.py +++ b/scrapling/engines/toolbelt/custom.py @@ -209,15 +209,3 @@ class StatusText: def get(cls, status_code: int) -> str: """Get the phrase for a given HTTP status code.""" return cls._phrases.get(status_code, "Unknown Status Code") - - -def get_variable_name(var: Any) -> Optional[str]: - """Get the name of a variable using global and local scopes. - :param var: The variable to find the name for - :return: The name of the variable if found, None otherwise - """ - for scope in [globals(), locals()]: - for name, value in scope.items(): - if value is var: - return name - return None diff --git a/scrapling/engines/toolbelt/fingerprints.py b/scrapling/engines/toolbelt/fingerprints.py index 7bcad8a..c006ec2 100644 --- a/scrapling/engines/toolbelt/fingerprints.py +++ b/scrapling/engines/toolbelt/fingerprints.py @@ -7,8 +7,9 @@ from platform import system as platform_system from tldextract import extract from browserforge.headers import Browser, HeaderGenerator +from browserforge.headers.generator import SUPPORTED_OPERATING_SYSTEMS -from scrapling.core._types import Dict, Literal +from scrapling.core._types import Dict, Literal, Tuple __OS_NAME__ = platform_system() OSName = Literal["linux", "macos", "windows"] @@ -29,12 +30,12 @@ def generate_convincing_referer(url: str) -> str: @lru_cache(1, typed=True) -def get_os_name() -> OSName | None: +def get_os_name() -> OSName | Tuple: """Get the current OS name in the same format needed for browserforge, if the OS is Unknown, return None so browserforge uses all. :return: Current OS name or `None` otherwise """ - match __OS_NAME__: + match __OS_NAME__: # pragma: no cover case "Linux": return "linux" case "Darwin": @@ -42,7 +43,7 @@ def get_os_name() -> OSName | None: case "Windows": return "windows" case _: - return None + return SUPPORTED_OPERATING_SYSTEMS def generate_headers(browser_mode: bool = False) -> Dict: @@ -63,10 +64,7 @@ def generate_headers(browser_mode: bool = False) -> Dict: Browser(name="edge", min_version=130), ] ) - if os_name: - return HeaderGenerator(browser=browsers, os=os_name, device="desktop").generate() - else: - return HeaderGenerator(browser=browsers, device="desktop").generate() + return HeaderGenerator(browser=browsers, os=os_name, device="desktop").generate() __default_useragent__ = generate_headers(browser_mode=False).get("User-Agent") From f9a514cc64ac121532e6656e5175417feab8ab06 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Sun, 26 Oct 2025 05:04:53 +0300 Subject: [PATCH 05/12] build: Pump up deps --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index c9d35f8..2dbbb92 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -59,7 +59,7 @@ classifiers = [ dependencies = [ "lxml>=6.0.2", "cssselect>=1.3.0", - "orjson>=3.11.3", + "orjson>=3.11.4", "tldextract>=5.3.0", ] @@ -74,7 +74,7 @@ fetchers = [ "msgspec>=0.19.0", ] ai = [ - "mcp>=1.18.0", + "mcp>=1.19.0", "markdownify>=1.2.0", "scrapling[fetchers]", ] From 13b9a4668cc2983466195f1e0d2689c62b9060dc Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Sun, 26 Oct 2025 16:53:27 +0300 Subject: [PATCH 06/12] feat(DynamicSession): New option to add extra browser flags --- docs/fetching/dynamic.md | 1 + scrapling/engines/_browsers/_base.py | 2 ++ scrapling/engines/_browsers/_config_tools.py | 10 ++++++++-- scrapling/engines/_browsers/_controllers.py | 6 ++++++ scrapling/engines/_browsers/_validators.py | 3 +++ scrapling/fetchers/chrome.py | 6 ++++++ 6 files changed, 26 insertions(+), 2 deletions(-) diff --git a/docs/fetching/dynamic.md b/docs/fetching/dynamic.md index 18c39d7..1054d54 100644 --- a/docs/fetching/dynamic.md +++ b/docs/fetching/dynamic.md @@ -89,6 +89,7 @@ Scrapling provides many options with this fetcher and its session classes. To ma | locale | Set the locale for the browser if wanted. The default value is `en-US`. | ✔️ | | cdp_url | Instead of launching a new browser instance, connect to this CDP URL to control real browsers through CDP. | ✔️ | | user_data_dir | Path to a User Data Directory, which stores browser session data like cookies and local storage. The default is to create a temporary directory. **Only Works with sessions** | ✔️ | +| extra_flags | A list of additional browser flags to pass to the browser on launch. | ✔️ | | additional_args | Additional arguments to be passed to Playwright's context as additional settings, and they take higher priority than Scrapling's settings. | ✔️ | | selector_config | A dictionary of custom parsing arguments to be used when creating the final `Selector`/`Response` class. | ✔️ | diff --git a/scrapling/engines/_browsers/_base.py b/scrapling/engines/_browsers/_base.py index 4157494..f670e4c 100644 --- a/scrapling/engines/_browsers/_base.py +++ b/scrapling/engines/_browsers/_base.py @@ -276,6 +276,7 @@ class DynamicSessionMixin: self.wait_selector = config.wait_selector self.init_script = config.init_script self.wait_selector_state = config.wait_selector_state + self.extra_flags = config.extra_flags self.selector_config = config.selector_config self.additional_args = config.additional_args self.page_action = config.page_action @@ -300,6 +301,7 @@ class DynamicSessionMixin: self.stealth, self.hide_canvas, self.disable_webgl, + tuple(self.extra_flags) if self.extra_flags else tuple(), ) ) self.launch_options["extra_http_headers"] = dict(self.launch_options["extra_http_headers"]) diff --git a/scrapling/engines/_browsers/_config_tools.py b/scrapling/engines/_browsers/_config_tools.py index 322d7ad..57e2f17 100644 --- a/scrapling/engines/_browsers/_config_tools.py +++ b/scrapling/engines/_browsers/_config_tools.py @@ -70,12 +70,17 @@ def _launch_kwargs( stealth, hide_canvas, disable_webgl, + extra_flags: Tuple, ) -> Tuple: """Creates the arguments we will use while launching playwright's browser""" + base_args = DEFAULT_FLAGS + if extra_flags: + base_args = base_args + extra_flags + launch_kwargs = { "locale": locale, "headless": headless, - "args": DEFAULT_FLAGS, + "args": base_args, "color_scheme": "dark", # Bypasses the 'prefersLightColor' check in creepjs "proxy": proxy or tuple(), "device_scale_factor": 2, @@ -85,9 +90,10 @@ def _launch_kwargs( "user_agent": useragent or __default_useragent__, } if stealth: + stealth_args = base_args + _set_flags(hide_canvas, disable_webgl) launch_kwargs.update( { - "args": DEFAULT_FLAGS + _set_flags(hide_canvas, disable_webgl), + "args": stealth_args, "chromium_sandbox": True, "is_mobile": False, "has_touch": False, diff --git a/scrapling/engines/_browsers/_controllers.py b/scrapling/engines/_browsers/_controllers.py index 155789a..d8f841b 100644 --- a/scrapling/engines/_browsers/_controllers.py +++ b/scrapling/engines/_browsers/_controllers.py @@ -95,6 +95,7 @@ class DynamicSession(DynamicSessionMixin, SyncSession): load_dom: bool = True, wait_selector_state: SelectorWaitStates = "attached", user_data_dir: str = "", + extra_flags: Optional[List[str]] = None, selector_config: Optional[Dict] = None, additional_args: Optional[Dict] = None, ): @@ -124,6 +125,7 @@ class DynamicSession(DynamicSessionMixin, SyncSession): :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only. :param user_data_dir: Path to a User Data Directory, which stores browser session data like cookies and local storage. The default is to create a temporary directory. + :param extra_flags: A list of additional browser flags to pass to the browser on launch. :param selector_config: The arguments that will be passed in the end while creating the final Selector's class. :param additional_args: Additional arguments to be passed to Playwright's context as additional settings, and it takes higher priority than Scrapling's settings. """ @@ -149,6 +151,7 @@ class DynamicSession(DynamicSessionMixin, SyncSession): extra_headers=extra_headers, wait_selector=wait_selector, disable_webgl=disable_webgl, + extra_flags=extra_flags, selector_config=selector_config, additional_args=additional_args, disable_resources=disable_resources, @@ -306,6 +309,7 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession): load_dom: bool = True, wait_selector_state: SelectorWaitStates = "attached", user_data_dir: str = "", + extra_flags: Optional[List[str]] = None, selector_config: Optional[Dict] = None, additional_args: Optional[Dict] = None, ): @@ -336,6 +340,7 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession): :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only. :param max_pages: The maximum number of tabs to be opened at the same time. It will be used in rotation through a PagePool. :param user_data_dir: Path to a User Data Directory, which stores browser session data like cookies and local storage. The default is to create a temporary directory. + :param extra_flags: A list of additional browser flags to pass to the browser on launch. :param selector_config: The arguments that will be passed in the end while creating the final Selector's class. :param additional_args: Additional arguments to be passed to Playwright's context as additional settings, and it takes higher priority than Scrapling's settings. """ @@ -362,6 +367,7 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession): extra_headers=extra_headers, wait_selector=wait_selector, disable_webgl=disable_webgl, + extra_flags=extra_flags, selector_config=selector_config, additional_args=additional_args, disable_resources=disable_resources, diff --git a/scrapling/engines/_browsers/_validators.py b/scrapling/engines/_browsers/_validators.py index 0d7d236..831aacb 100644 --- a/scrapling/engines/_browsers/_validators.py +++ b/scrapling/engines/_browsers/_validators.py @@ -88,6 +88,7 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False): load_dom: bool = True wait_selector_state: SelectorWaitStates = "attached" user_data_dir: str = "" + extra_flags: Optional[List[str]] = None selector_config: Optional[Dict] = {} additional_args: Optional[Dict] = {} @@ -102,6 +103,8 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False): if not self.cookies: self.cookies = [] + if not self.extra_flags: + self.extra_flags = [] if not self.selector_config: self.selector_config = {} if not self.additional_args: diff --git a/scrapling/fetchers/chrome.py b/scrapling/fetchers/chrome.py index 9cc980d..0c2ab84 100644 --- a/scrapling/fetchers/chrome.py +++ b/scrapling/fetchers/chrome.py @@ -50,6 +50,7 @@ class DynamicFetcher(BaseFetcher): network_idle: bool = False, load_dom: bool = True, wait_selector_state: SelectorWaitStates = "attached", + extra_flags: Optional[List[str]] = None, additional_args: Optional[Dict] = None, custom_config: Optional[Dict] = None, ) -> Response: @@ -79,6 +80,7 @@ class DynamicFetcher(BaseFetcher): :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name. :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only. + :param extra_flags: A list of additional browser flags to pass to the browser on launch. :param custom_config: A dictionary of custom parser arguments to use with this request. Any argument passed will override any class parameters values. :param additional_args: Additional arguments to be passed to Playwright's context as additional settings, and it takes higher priority than Scrapling's settings. :return: A `Response` object. @@ -108,6 +110,7 @@ class DynamicFetcher(BaseFetcher): extra_headers=extra_headers, wait_selector=wait_selector, disable_webgl=disable_webgl, + extra_flags=extra_flags, additional_args=additional_args, disable_resources=disable_resources, wait_selector_state=wait_selector_state, @@ -140,6 +143,7 @@ class DynamicFetcher(BaseFetcher): network_idle: bool = False, load_dom: bool = True, wait_selector_state: SelectorWaitStates = "attached", + extra_flags: Optional[List[str]] = None, additional_args: Optional[Dict] = None, custom_config: Optional[Dict] = None, ) -> Response: @@ -169,6 +173,7 @@ class DynamicFetcher(BaseFetcher): :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name. :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._ :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only. + :param extra_flags: A list of additional browser flags to pass to the browser on launch. :param custom_config: A dictionary of custom parser arguments to use with this request. Any argument passed will override any class parameters values. :param additional_args: Additional arguments to be passed to Playwright's context as additional settings, and it takes higher priority than Scrapling's settings. :return: A `Response` object. @@ -199,6 +204,7 @@ class DynamicFetcher(BaseFetcher): extra_headers=extra_headers, wait_selector=wait_selector, disable_webgl=disable_webgl, + extra_flags=extra_flags, additional_args=additional_args, disable_resources=disable_resources, wait_selector_state=wait_selector_state, From d4a250766685e68849dcaa85578ff9463f968e3c Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Mon, 27 Oct 2025 04:48:45 +0300 Subject: [PATCH 07/12] refactor(validation): All browser based fetchers are ~8-15% faster now in most cases --- scrapling/engines/_browsers/_camoufox.py | 5 +- scrapling/engines/_browsers/_controllers.py | 4 +- scrapling/engines/_browsers/_validators.py | 130 +++++++++++--------- 3 files changed, 75 insertions(+), 64 deletions(-) diff --git a/scrapling/engines/_browsers/_camoufox.py b/scrapling/engines/_browsers/_camoufox.py index ac66870..c0b2f2c 100644 --- a/scrapling/engines/_browsers/_camoufox.py +++ b/scrapling/engines/_browsers/_camoufox.py @@ -13,9 +13,8 @@ from playwright.async_api import ( Playwright as AsyncPlaywright, BrowserContext as AsyncBrowserContext, ) -from playwright._impl._errors import Error as PlaywrightError -from ._validators import validate_fetch as _validate +from ._validators import validate_fetch as _validate, CamoufoxConfig from ._base import SyncSession, AsyncSession, StealthySessionMixin from scrapling.core.utils import log from scrapling.core._types import ( @@ -293,6 +292,7 @@ class StealthySession(StealthySessionMixin, SyncSession): ("solve_cloudflare", solve_cloudflare, self.solve_cloudflare), ("selector_config", selector_config, self.selector_config), ], + CamoufoxConfig, _UNSET, ) @@ -574,6 +574,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession): ("solve_cloudflare", solve_cloudflare, self.solve_cloudflare), ("selector_config", selector_config, self.selector_config), ], + CamoufoxConfig, _UNSET, ) diff --git a/scrapling/engines/_browsers/_controllers.py b/scrapling/engines/_browsers/_controllers.py index d8f841b..bcfd0ac 100644 --- a/scrapling/engines/_browsers/_controllers.py +++ b/scrapling/engines/_browsers/_controllers.py @@ -14,7 +14,7 @@ from patchright.async_api import async_playwright as async_patchright from scrapling.core.utils import log from ._base import SyncSession, AsyncSession, DynamicSessionMixin -from ._validators import validate_fetch as _validate +from ._validators import validate_fetch as _validate, PlaywrightConfig from scrapling.core._types import ( Any, Dict, @@ -225,6 +225,7 @@ class DynamicSession(DynamicSessionMixin, SyncSession): ("load_dom", load_dom, self.load_dom), ("selector_config", selector_config, self.selector_config), ], + PlaywrightConfig, _UNSET, ) @@ -442,6 +443,7 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession): ("load_dom", load_dom, self.load_dom), ("selector_config", selector_config, self.selector_config), ], + PlaywrightConfig, _UNSET, ) diff --git a/scrapling/engines/_browsers/_validators.py b/scrapling/engines/_browsers/_validators.py index 831aacb..edeaa2f 100644 --- a/scrapling/engines/_browsers/_validators.py +++ b/scrapling/engines/_browsers/_validators.py @@ -1,7 +1,8 @@ from pathlib import Path from typing import Annotated -from dataclasses import dataclass +from functools import lru_cache from urllib.parse import urlparse +from dataclasses import dataclass, fields from msgspec import Struct, Meta, convert, ValidationError @@ -19,18 +20,20 @@ from scrapling.engines.toolbelt.navigation import construct_proxy_dict # Custom validators for msgspec -def _validate_file_path(value: str): +@lru_cache(8) +def _is_invalid_file_path(value: str) -> bool | str: """Fast file path validation""" path = Path(value) if not path.exists(): - raise ValueError(f"Init script path not found: {value}") + return f"Init script path not found: {value}" if not path.is_file(): - raise ValueError(f"Init script is not a file: {value}") + return f"Init script is not a file: {value}" if not path.is_absolute(): - raise ValueError(f"Init script is not a absolute path: {value}") + return f"Init script is not a absolute path: {value}" + return False -def _validate_addon_path(value: str): +def _validate_addon_path(value: str) -> None: """Fast addon path validation""" path = Path(value) if not path.exists(): @@ -39,22 +42,16 @@ def _validate_addon_path(value: str): raise ValueError(f"Addon path must be a directory of the extracted addon: {value}") -def _validate_cdp_url(cdp_url: str): +@lru_cache(2) +def _is_invalid_cdp_url(cdp_url: str) -> bool | str: """Fast CDP URL validation""" - try: - # Check the scheme - if not cdp_url.startswith(("ws://", "wss://")): - raise ValueError("CDP URL must use 'ws://' or 'wss://' scheme") + if not cdp_url.startswith(("ws://", "wss://")): + return "CDP URL must use 'ws://' or 'wss://' scheme" - # Validate hostname and port - if not urlparse(cdp_url).netloc: - raise ValueError("Invalid hostname for the CDP URL") - - except AttributeError as e: - raise ValueError(f"Malformed CDP URL: {cdp_url}: {str(e)}") - - except Exception as e: - raise ValueError(f"Invalid CDP URL '{cdp_url}': {str(e)}") + netloc = urlparse(cdp_url).netloc + if not netloc: + return "Invalid hostname for the CDP URL" + return False # Type aliases for cleaner annotations @@ -62,7 +59,7 @@ PagesCount = Annotated[int, Meta(ge=1, le=50)] Seconds = Annotated[int, float, Meta(ge=0)] -class PlaywrightConfig(Struct, kw_only=True, frozen=False): +class PlaywrightConfig(Struct, kw_only=True, frozen=False, weakref=True): """Configuration struct for validation""" max_pages: PagesCount = 1 @@ -99,7 +96,9 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False): if self.proxy: self.proxy = construct_proxy_dict(self.proxy, as_tuple=True) if self.cdp_url: - _validate_cdp_url(self.cdp_url) + cdp_msg = _is_invalid_cdp_url(self.cdp_url) + if cdp_msg: + raise ValueError(cdp_msg) if not self.cookies: self.cookies = [] @@ -111,10 +110,12 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False): self.additional_args = {} if self.init_script is not None: - _validate_file_path(self.init_script) + validation_msg = _is_invalid_file_path(self.init_script) + if validation_msg: + raise ValueError(validation_msg) -class CamoufoxConfig(Struct, kw_only=True, frozen=False): +class CamoufoxConfig(Struct, kw_only=True, frozen=False, weakref=True): """Configuration struct for validation""" max_pages: PagesCount = 1 @@ -152,14 +153,16 @@ class CamoufoxConfig(Struct, kw_only=True, frozen=False): if self.proxy: self.proxy = construct_proxy_dict(self.proxy, as_tuple=True) - if self.addons and isinstance(self.addons, list): + if self.addons: for addon in self.addons: _validate_addon_path(addon) else: self.addons = [] if self.init_script is not None: - _validate_file_path(self.init_script) + validation_msg = _is_invalid_file_path(self.init_script) + if validation_msg: + raise ValueError(validation_msg) if not self.cookies: self.cookies = [] @@ -172,27 +175,6 @@ class CamoufoxConfig(Struct, kw_only=True, frozen=False): self.additional_args = {} -# Code parts to validate `fetch` in the least possible numbers of lines overall -class FetchConfig(Struct, kw_only=True): - """Configuration struct for `fetch` calls validation""" - - google_search: bool = True - timeout: Seconds = 30000 - wait: Seconds = 0 - page_action: Optional[Callable] = None - extra_headers: Optional[Dict[str, str]] = None - disable_resources: bool = False - wait_selector: Optional[str] = None - wait_selector_state: SelectorWaitStates = "attached" - network_idle: bool = False - load_dom: bool = True - solve_cloudflare: bool = False - selector_config: Dict = {} - - def to_dict(self): - return {f: getattr(self, f) for f in self.__struct_fields__} - - @dataclass class _fetch_params: """A dataclass of all parameters used by `fetch` calls""" @@ -211,7 +193,9 @@ class _fetch_params: selector_config: Dict -def validate_fetch(params: List[Tuple], sentinel=None) -> _fetch_params: +def validate_fetch( + params: List[Tuple], model: type[PlaywrightConfig] | type[CamoufoxConfig], sentinel=None +) -> _fetch_params: result = {} overrides = {} @@ -222,16 +206,44 @@ def validate_fetch(params: List[Tuple], sentinel=None) -> _fetch_params: result[arg] = session_value if overrides: - overrides = validate(overrides, FetchConfig).to_dict() - overrides.update(result) - return _fetch_params(**overrides) + validated_config = validate(overrides, model) + # Extract only the fields that _fetch_params needs from validated_config + validated_dict = { + f.name: getattr(validated_config, f.name) + for f in fields(_fetch_params) + if hasattr(validated_config, f.name) + } + # solve_cloudflare defaults to False for models that don't have it (PlaywrightConfig) + validated_dict.setdefault("solve_cloudflare", False) - if not result.get("solve_cloudflare"): - result["solve_cloudflare"] = False + validated_dict.update(result) + return _fetch_params(**validated_dict) + + result.setdefault("solve_cloudflare", False) return _fetch_params(**result) +# Cache default values for each model to reduce validation overhead +models_default_values = {} + +for _model in (CamoufoxConfig, PlaywrightConfig): + _defaults = {} + if hasattr(_model, "__struct_defaults__") and hasattr(_model, "__struct_fields__"): + for field_name, default_value in zip(_model.__struct_fields__, _model.__struct_defaults__): # type: ignore + # Skip factory defaults - these are msgspec._core.Factory instances + if type(default_value).__name__ != "Factory": + _defaults[field_name] = default_value + + models_default_values[_model.__name__] = _defaults.copy() + + +def _filter_defaults(params: Dict, model: str) -> Dict: + """Filter out parameters that match their default values to reduce validation overhead.""" + defaults = models_default_values[model] + return {k: v for k, v in params.items() if k not in defaults or v != defaults[k]} + + @overload def validate(params: Dict, model: type[PlaywrightConfig]) -> PlaywrightConfig: ... @@ -240,14 +252,10 @@ def validate(params: Dict, model: type[PlaywrightConfig]) -> PlaywrightConfig: . def validate(params: Dict, model: type[CamoufoxConfig]) -> CamoufoxConfig: ... -@overload -def validate(params: Dict, model: type[FetchConfig]) -> FetchConfig: ... - - -def validate( - params: Dict, model: type[PlaywrightConfig] | type[CamoufoxConfig] | type[FetchConfig] -) -> PlaywrightConfig | CamoufoxConfig | FetchConfig: +def validate(params: Dict, model: type[PlaywrightConfig] | type[CamoufoxConfig]) -> PlaywrightConfig | CamoufoxConfig: try: - return convert(params, model) + # Filter out params with the default values (no need to validate them) to speed up validation + filtered = _filter_defaults(params, model.__name__) + return convert(filtered, model) except ValidationError as e: raise TypeError(f"Invalid argument type: {e}") from e From fc4a809985dde28e7fec88f03a9f74adab130e91 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Mon, 27 Oct 2025 05:13:12 +0300 Subject: [PATCH 08/12] fix(cloudflare): Limit the waiting period for websites that captcha doesn't disappear after solving Also solves #100 --- scrapling/engines/_browsers/_camoufox.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/scrapling/engines/_browsers/_camoufox.py b/scrapling/engines/_browsers/_camoufox.py index c0b2f2c..4ac0558 100644 --- a/scrapling/engines/_browsers/_camoufox.py +++ b/scrapling/engines/_browsers/_camoufox.py @@ -231,9 +231,14 @@ class StealthySession(StealthySessionMixin, SyncSession): page.mouse.click(captcha_x, captcha_y, delay=60, button="left") self._wait_for_networkidle(page) if iframe is not None: - # Wait for the frame to be removed from the page + # Wait for the frame to be removed from the page (with 30s timeout = 300 iterations * 100 ms) + attempts = 0 while iframe in page.frames: + if attempts >= 300: + log.info("Cloudflare iframe didn't disappear after 30s, continuing...") + break page.wait_for_timeout(100) + attempts += 1 if challenge_type != "embedded": page.locator(box_selector).last.wait_for(state="detached") page.locator(".zone-name-title").wait_for(state="hidden") @@ -513,9 +518,14 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession): await page.mouse.click(captcha_x, captcha_y, delay=60, button="left") await self._wait_for_networkidle(page) if iframe is not None: - # Wait for the frame to be removed from the page + # Wait for the frame to be removed from the page (with 30s timeout = 300 iterations * 100 ms) + attempts = 0 while iframe in page.frames: + if attempts >= 300: + log.info("Cloudflare iframe didn't disappear after 30s, continuing...") + break await page.wait_for_timeout(100) + attempts += 1 if challenge_type != "embedded": await page.locator(box_selector).wait_for(state="detached") await page.locator(".zone-name-title").wait_for(state="hidden") From 339ec4334169622b9d7a07db6dd329f7cdc210bd Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Mon, 27 Oct 2025 06:32:31 +0300 Subject: [PATCH 09/12] docs: adding separate doc for external browser --- docs/fetching/dynamic.md | 36 ------------------------------------ docs/tutorials/external.md | 35 +++++++++++++++++++++++++++++++++++ mkdocs.yml | 1 + 3 files changed, 36 insertions(+), 36 deletions(-) create mode 100644 docs/tutorials/external.md diff --git a/docs/fetching/dynamic.md b/docs/fetching/dynamic.md index 1054d54..fbdb387 100644 --- a/docs/fetching/dynamic.md +++ b/docs/fetching/dynamic.md @@ -302,39 +302,3 @@ Use DynamicFetcher when: - Want flexible stealth options If you want more stealth and control without much config, check out the [StealthyFetcher](stealthy.md). - -## External Cloud Browser Version - -If you have issues with the browser installation, such as resource management, we recommend you try the Cloud Browser from [Scrapeless](https://www.scrapeless.com/en/product/scraping-browser) for free! - -The usage is straightforward: create an account and [get your API key](https://docs.scrapeless.com/en/scraping-browser/quickstart/getting-started/), then pass it to the `DynamicSession` like this: - -```python -from urllib.parse import urlencode - -from scrapling.fetchers import DynamicSession - -# Configure your browser session -config = { - "token": "YOUR_API_KEY", - "sessionName": "scrapling-session", - "sessionTTL": "300", # 5 minutes - "proxyCountry": "ANY", - "sessionRecording": "false", -} - -# Build WebSocket URL -ws_endpoint = f"wss://browser.scrapeless.com/api/v2/browser?{urlencode(config)}" -print('Connecting to Scrapeless...') - -with DynamicSession(cdp_url=ws_endpoint, disable_resources=True) as s: - print("Connected!") - page = s.fetch("https://httpbin.org/headers", network_idle=True) - print(f"Page loaded, content length: {len(page.body)}") - print(page.json()) -``` -The `DynamicSession` class instance will work as usual, so no further explanation is needed. - -However, the Scrapeless Cloud Browser can be configured with proxy options, like the proxy country in the config above, [custom fingerprint](https://docs.scrapeless.com/en/scraping-browser/features/advanced-privacy-anti-detection/custom-fingerprint/) configuration, [captcha solving](https://docs.scrapeless.com/en/scraping-browser/features/advanced-privacy-anti-detection/supported-captchas/), and more. - -Check out the [Scrapeless's browser documentation](https://docs.scrapeless.com/en/scraping-browser/quickstart/introduction/) for more details. \ No newline at end of file diff --git a/docs/tutorials/external.md b/docs/tutorials/external.md new file mode 100644 index 0000000..ba61093 --- /dev/null +++ b/docs/tutorials/external.md @@ -0,0 +1,35 @@ +## External Cloud Browser Version + +If you have issues with the browser installation, such as resource management, we recommend you try the Cloud Browser from [Scrapeless](https://www.scrapeless.com/en/product/scraping-browser) for free! + +The usage is straightforward: create an account and [get your API key](https://docs.scrapeless.com/en/scraping-browser/quickstart/getting-started/), then pass it to the `DynamicSession` like this: + +```python +from urllib.parse import urlencode + +from scrapling.fetchers import DynamicSession + +# Configure your browser session +config = { + "token": "YOUR_API_KEY", + "sessionName": "scrapling-session", + "sessionTTL": "300", # 5 minutes + "proxyCountry": "ANY", + "sessionRecording": "false", +} + +# Build WebSocket URL +ws_endpoint = f"wss://browser.scrapeless.com/api/v2/browser?{urlencode(config)}" +print('Connecting to Scrapeless...') + +with DynamicSession(cdp_url=ws_endpoint, disable_resources=True) as s: + print("Connected!") + page = s.fetch("https://httpbin.org/headers", network_idle=True) + print(f"Page loaded, content length: {len(page.body)}") + print(page.json()) +``` +The `DynamicSession` class instance will work as usual, so no further explanation is needed. + +However, the Scrapeless Cloud Browser can be configured with proxy options, like the proxy country in the config above, [custom fingerprint](https://docs.scrapeless.com/en/scraping-browser/features/advanced-privacy-anti-detection/custom-fingerprint/) configuration, [captcha solving](https://docs.scrapeless.com/en/scraping-browser/features/advanced-privacy-anti-detection/supported-captchas/), and more. + +Check out the [Scrapeless's browser documentation](https://docs.scrapeless.com/en/scraping-browser/quickstart/introduction/) for more details. \ No newline at end of file diff --git a/mkdocs.yml b/mkdocs.yml index d455c4a..6a16483 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -83,6 +83,7 @@ nav: - Tutorials: - A Free Alternative to AI for Robust Web Scraping: tutorials/replacing_ai.md - Migrating from BeautifulSoup: tutorials/migrating_from_beautifulsoup.md + - Using Scrapeless browser: tutorials/external.md # - Migrating from AutoScraper: tutorials/migrating_from_autoscraper.md - Development: - API Reference: From 37cb5097c07d55e827d8bafbb3def17193369960 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Mon, 27 Oct 2025 06:33:45 +0300 Subject: [PATCH 10/12] docs: update external browser tutorial --- docs/tutorials/external.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/tutorials/external.md b/docs/tutorials/external.md index ba61093..797496c 100644 --- a/docs/tutorials/external.md +++ b/docs/tutorials/external.md @@ -1,4 +1,3 @@ -## External Cloud Browser Version If you have issues with the browser installation, such as resource management, we recommend you try the Cloud Browser from [Scrapeless](https://www.scrapeless.com/en/product/scraping-browser) for free! From 3c8d5c68e88e753333c58ba1cc6ec4c42cc14228 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Mon, 27 Oct 2025 16:42:03 +0300 Subject: [PATCH 11/12] docs: updating external links --- docs/tutorials/external.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/tutorials/external.md b/docs/tutorials/external.md index 797496c..15a5f9b 100644 --- a/docs/tutorials/external.md +++ b/docs/tutorials/external.md @@ -1,7 +1,7 @@ -If you have issues with the browser installation, such as resource management, we recommend you try the Cloud Browser from [Scrapeless](https://www.scrapeless.com/en/product/scraping-browser) for free! +If you have issues with the browser installation, such as resource management, we recommend you try the Cloud Browser from [Scrapeless](https://www.scrapeless.com/en/product/scraping-browser?utm_source=official&utm_term=scrapling) for free! -The usage is straightforward: create an account and [get your API key](https://docs.scrapeless.com/en/scraping-browser/quickstart/getting-started/), then pass it to the `DynamicSession` like this: +The usage is straightforward: create an account and [get your API key](https://docs.scrapeless.com/en/scraping-browser/quickstart/getting-started/?utm_source=official&utm_term=scrapling), then pass it to the `DynamicSession` like this: ```python from urllib.parse import urlencode @@ -29,6 +29,6 @@ with DynamicSession(cdp_url=ws_endpoint, disable_resources=True) as s: ``` The `DynamicSession` class instance will work as usual, so no further explanation is needed. -However, the Scrapeless Cloud Browser can be configured with proxy options, like the proxy country in the config above, [custom fingerprint](https://docs.scrapeless.com/en/scraping-browser/features/advanced-privacy-anti-detection/custom-fingerprint/) configuration, [captcha solving](https://docs.scrapeless.com/en/scraping-browser/features/advanced-privacy-anti-detection/supported-captchas/), and more. +However, the Scrapeless Cloud Browser can be configured with proxy options, like the proxy country in the config above, [custom fingerprint](https://docs.scrapeless.com/en/scraping-browser/features/advanced-privacy-anti-detection/custom-fingerprint/?utm_source=official&utm_term=scrapling) configuration, [captcha solving](https://docs.scrapeless.com/en/scraping-browser/features/advanced-privacy-anti-detection/supported-captchas/?utm_source=official&utm_term=scrapling), and more. -Check out the [Scrapeless's browser documentation](https://docs.scrapeless.com/en/scraping-browser/quickstart/introduction/) for more details. \ No newline at end of file +Check out the [Scrapeless's browser documentation](https://docs.scrapeless.com/en/scraping-browser/quickstart/introduction/?utm_source=official&utm_term=scrapling) for more details. \ No newline at end of file From 1e4d1e73a02a9f7fe3adb130d651471895c83c8a Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Mon, 27 Oct 2025 17:36:44 +0300 Subject: [PATCH 12/12] ops: Make container image pushed to Github registry too --- .github/workflows/docker-build.yml | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 62338f3..3c8ddcd 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -13,8 +13,8 @@ on: default: 'latest' env: - REGISTRY: docker.io - IMAGE_NAME: pyd4vinci/scrapling + DOCKERHUB_IMAGE: pyd4vinci/scrapling + GHCR_IMAGE: ghcr.io/${{ github.repository_owner }}/scrapling jobs: build-and-push: @@ -35,15 +35,24 @@ jobs: - name: Log in to Docker Hub uses: docker/login-action@v3 with: - registry: ${{ env.REGISTRY }} + registry: docker.io username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} + - name: Log in to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.CONTAINER_TOKEN }} + - name: Extract metadata id: meta uses: docker/metadata-action@v5 with: - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + images: | + ${{ env.DOCKERHUB_IMAGE }} + ${{ env.GHCR_IMAGE }} tags: | type=ref,event=branch type=ref,event=pr @@ -51,6 +60,14 @@ jobs: type=semver,pattern={{major}}.{{minor}} type=semver,pattern={{major}} type=raw,value=latest,enable={{is_default_branch}} + labels: | + org.opencontainers.image.title=Scrapling + org.opencontainers.image.description=An undetectable, powerful, flexible, high-performance Python library that makes Web Scraping easy and effortless as it should be! + org.opencontainers.image.vendor=D4Vinci + org.opencontainers.image.licenses=BSD + org.opencontainers.image.url=https://scrapling.readthedocs.io/en/latest/ + org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }} + org.opencontainers.image.documentation=https://scrapling.readthedocs.io/en/latest/ - name: Build and push Docker image uses: docker/build-push-action@v5