diff --git a/.gitignore b/.gitignore index f27bc08..77d1210 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,7 @@ +# local files site/* +local_tests/* +.mcpregistry_* # AI related files .claude/* diff --git a/MANIFEST.in b/MANIFEST.in index 8c168dd..aa9bf23 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,7 +1,6 @@ include LICENSE include *.db include *.js -include scrapling/engines/toolbelt/bypasses/*.js include scrapling/*.db include scrapling/*.db* include scrapling/*.db-* diff --git a/README.md b/README.md index aae6045..5b4ca2c 100644 --- a/README.md +++ b/README.md @@ -78,6 +78,7 @@ MySpider().start() # Platinum Sponsors +Do you want to be the first company to show up here? Click [here](https://github.com/sponsors/D4Vinci/sponsorships?tier_id=586646) # Sponsors @@ -366,11 +367,20 @@ This installation only includes the parser engine and its dependencies, without ```bash pip install "scrapling[fetchers]" - scrapling install + scrapling install # normal install + scrapling install --force # force reinstall ``` This downloads all browsers, along with their system dependencies and fingerprint manipulation dependencies. + Or you can install them from the code instead of running a command like this: + ```python + from scrapling.cli import install + + install([], standalone_mode=False) # normal install + install(["--force"], standalone_mode=False) # force reinstall + ``` + 2. Extra features: - Install the MCP server feature: ```bash diff --git a/docs/README_AR.md b/docs/README_AR.md index dcccfa0..d5f525c 100644 --- a/docs/README_AR.md +++ b/docs/README_AR.md @@ -1,3 +1,5 @@ + +

@@ -73,6 +75,7 @@ MySpider().start() # الرعاة البلاتينيون +هل تريد أن تكون أول شركة تظهر هنا؟ انقر [هنا](https://github.com/sponsors/D4Vinci/sponsorships?tier_id=586646) # الرعاة @@ -361,11 +364,20 @@ pip install scrapling ```bash pip install "scrapling[fetchers]" - scrapling install + scrapling install # normal install + scrapling install --force # force reinstall ``` يقوم هذا بتنزيل جميع المتصفحات، إلى جانب تبعيات النظام وتبعيات معالجة fingerprint الخاصة بها. + أو يمكنك تثبيتها من الكود بدلاً من تشغيل أمر كالتالي: + ```python + from scrapling.cli import install + + install([], standalone_mode=False) # normal install + install(["--force"], standalone_mode=False) # force reinstall + ``` + 2. ميزات إضافية: - تثبيت ميزة خادم MCP: ```bash diff --git a/docs/README_CN.md b/docs/README_CN.md index d9ec033..b18bff3 100644 --- a/docs/README_CN.md +++ b/docs/README_CN.md @@ -1,3 +1,5 @@ + +

@@ -73,6 +75,7 @@ MySpider().start() # 铂金赞助商 +想成为第一个出现在这里的公司吗？点击[这里](https://github.com/sponsors/D4Vinci/sponsorships?tier_id=586646) # 赞助商 @@ -361,11 +364,20 @@ pip install scrapling ```bash pip install "scrapling[fetchers]" - scrapling install + scrapling install # normal install + scrapling install --force # force reinstall ``` 这会下载所有浏览器，以及它们的系统依赖项和fingerprint操作依赖项。 + 或者你可以从代码中安装，而不是运行命令： + ```python + from scrapling.cli import install + + install([], standalone_mode=False) # normal install + install(["--force"], standalone_mode=False) # force reinstall + ``` + 2. 额外功能： - 安装MCP服务器功能： ```bash diff --git a/docs/README_DE.md b/docs/README_DE.md index cec019c..4772d7f 100644 --- a/docs/README_DE.md +++ b/docs/README_DE.md @@ -1,3 +1,5 @@ + +

@@ -73,6 +75,7 @@ MySpider().start() # Platin-Sponsoren +Möchten Sie das erste Unternehmen sein, das hier erscheint? Klicken Sie [hier](https://github.com/sponsors/D4Vinci/sponsorships?tier_id=586646) # Sponsoren @@ -361,11 +364,20 @@ Diese Installation enthält nur die Parser-Engine und ihre Abhängigkeiten, ohne ```bash pip install "scrapling[fetchers]" - scrapling install + scrapling install # normal install + scrapling install --force # force reinstall ``` Dies lädt alle Browser zusammen mit ihren Systemabhängigkeiten und Fingerprint-Manipulationsabhängigkeiten herunter. + Oder Sie können sie aus dem Code heraus installieren, anstatt einen Befehl auszuführen: + ```python + from scrapling.cli import install + + install([], standalone_mode=False) # normal install + install(["--force"], standalone_mode=False) # force reinstall + ``` + 2. Zusätzliche Funktionen: - MCP-Server-Funktion installieren: ```bash diff --git a/docs/README_ES.md b/docs/README_ES.md index 851fa40..7037dd5 100644 --- a/docs/README_ES.md +++ b/docs/README_ES.md @@ -1,3 +1,5 @@ + +

@@ -73,6 +75,7 @@ MySpider().start() # Patrocinadores Platino +¿Quieres ser la primera empresa en aparecer aquí? Haz clic [aquí](https://github.com/sponsors/D4Vinci/sponsorships?tier_id=586646) # Patrocinadores @@ -361,11 +364,20 @@ Esta instalación solo incluye el motor de análisis y sus dependencias, sin nin ```bash pip install "scrapling[fetchers]" - scrapling install + scrapling install # normal install + scrapling install --force # force reinstall ``` Esto descarga todos los navegadores, junto con sus dependencias del sistema y dependencias de manipulación de fingerprint. + O puedes instalarlos desde el código en lugar de ejecutar un comando: + ```python + from scrapling.cli import install + + install([], standalone_mode=False) # normal install + install(["--force"], standalone_mode=False) # force reinstall + ``` + 2. Características adicionales: - Instalar la característica del servidor MCP: ```bash diff --git a/docs/README_JP.md b/docs/README_JP.md index 43afc62..890e049 100644 --- a/docs/README_JP.md +++ b/docs/README_JP.md @@ -1,3 +1,5 @@ + +

@@ -73,6 +75,7 @@ MySpider().start() # プラチナスポンサー +ここに最初に表示される企業になりませんか？[こちら](https://github.com/sponsors/D4Vinci/sponsorships?tier_id=586646)をクリック # スポンサー @@ -361,11 +364,20 @@ pip install scrapling ```bash pip install "scrapling[fetchers]" - scrapling install + scrapling install # normal install + scrapling install --force # force reinstall ``` これにより、すべてのブラウザ、およびそれらのシステム依存関係とfingerprint操作依存関係がダウンロードされます。 + または、コマンドを実行する代わりにコードからインストールすることもできます： + ```python + from scrapling.cli import install + + install([], standalone_mode=False) # normal install + install(["--force"], standalone_mode=False) # force reinstall + ``` + 2. 追加機能： - MCPサーバー機能をインストール： ```bash diff --git a/docs/README_RU.md b/docs/README_RU.md index 5a3151b..091244c 100644 --- a/docs/README_RU.md +++ b/docs/README_RU.md @@ -1,3 +1,5 @@ + +

@@ -73,6 +75,7 @@ MySpider().start() # Платиновые спонсоры +Хотите стать первой компанией, которая появится здесь? Нажмите [здесь](https://github.com/sponsors/D4Vinci/sponsorships?tier_id=586646) # Спонсоры @@ -361,11 +364,20 @@ pip install scrapling ```bash pip install "scrapling[fetchers]" - scrapling install + scrapling install # normal install + scrapling install --force # force reinstall ``` Это загрузит все браузеры вместе с их системными зависимостями и зависимостями для манипуляции fingerprint'ами. + Или вы можете установить их из кода вместо выполнения команды: + ```python + from scrapling.cli import install + + install([], standalone_mode=False) # normal install + install(["--force"], standalone_mode=False) # force reinstall + ``` + 2. Дополнительные возможности: - Установить функцию MCP-сервера: ```bash diff --git a/docs/ai/mcp-server.md b/docs/ai/mcp-server.md index 316f455..bf59d66 100644 --- a/docs/ai/mcp-server.md +++ b/docs/ai/mcp-server.md @@ -125,7 +125,7 @@ If you are using the Docker image, then it would be something like } ``` -The same logic applies to [Cursor](https://docs.cursor.com/en/context/mcp), [WindSurf](https://windsurf.com/university/tutorials/configuring-first-mcp-server), and others. +The same logic applies to [Cursor](https://cursor.com/docs/context/mcp), [WindSurf](https://windsurf.com/university/tutorials/configuring-first-mcp-server), and others. ### Claude Code Here it's much simpler to do. If you have [Claude Code](https://www.anthropic.com/claude-code) installed, open the terminal and execute the following command: diff --git a/docs/index.md b/docs/index.md index 15f7b5d..2bafd80 100644 --- a/docs/index.md +++ b/docs/index.md @@ -154,11 +154,20 @@ This installation only includes the parser engine and its dependencies, without ```bash pip install "scrapling[fetchers]" - scrapling install + scrapling install # normal install + scrapling install --force # force reinstall ``` This downloads all browsers, along with their system dependencies and fingerprint manipulation dependencies. + Or you can install them from the code instead of running a command like this: + ```python + from scrapling.cli import install + + install([], standalone_mode=False) # normal install + install(["--force"], standalone_mode=False) # force reinstall + ``` + 2. Extra features: diff --git a/docs/requirements.txt b/docs/requirements.txt index ff7edb0..3495486 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,8 +1,8 @@ -zensical>=0.0.23 +zensical>=0.0.24 mkdocstrings>=1.0.3 -mkdocstrings-python>=2.0.2 -griffe-inherited-docstrings -griffe-runtime-objects -griffe-sphinx +mkdocstrings-python>=2.0.3 +griffe-inherited-docstrings>=1.1.3 +griffe-runtime-objects>=0.3.1 +griffe-sphinx>=0.2.1 black>=26.1.0 pngquant \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 90457e9..e16db98 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta" [project] name = "scrapling" # Static version instead of a dynamic version so we can get better layer caching while building docker, check the docker file to understand -version = "0.4" +version = "0.4.1" description = "Scrapling is an undetectable, powerful, flexible, high-performance Python library that makes Web Scraping easy and effortless as it should be!" readme = {file = "README.md", content-type = "text/markdown"} license = {file = "LICENSE"} @@ -80,7 +80,7 @@ fetchers = [ "anyio>=4.12.1" ] ai = [ - "mcp>=1.24.0", + "mcp>=1.26.0", "markdownify>=1.2.0", "scrapling[fetchers]", ] diff --git a/scrapling/__init__.py b/scrapling/__init__.py index 60c859c..1b35dbe 100644 --- a/scrapling/__init__.py +++ b/scrapling/__init__.py @@ -1,5 +1,5 @@ __author__ = "Karim Shoair (karim.shoair@pm.me)" -__version__ = "0.4" +__version__ = "0.4.1" __copyright__ = "Copyright (c) 2024 Karim Shoair" from typing import Any, TYPE_CHECKING diff --git a/scrapling/core/ai.py b/scrapling/core/ai.py index 171359a..f637437 100644 --- a/scrapling/core/ai.py +++ b/scrapling/core/ai.py @@ -37,11 +37,25 @@ class ResponseModel(BaseModel): url: str = Field(description="The URL given by the user that resulted in this response.") -def _ContentTranslator(content: Generator[str, None, None], page: _ScraplingResponse) -> ResponseModel: +def _content_translator(content: Generator[str, None, None], page: _ScraplingResponse) -> ResponseModel: """Convert a content generator to a list of ResponseModel objects.""" return ResponseModel(status=page.status, content=[result for result in content], url=page.url) +def _normalize_credentials(credentials: Optional[Dict[str, str]]) -> Optional[Tuple[str, str]]: + """Convert a credentials dictionary to a tuple accepted by fetchers.""" + if not credentials: + return None + + username = credentials.get("username") + password = credentials.get("password") + + if username is None or password is None: + raise ValueError("Credentials dictionary must contain both 'username' and 'password' keys") + + return username, password + + class ScraplingMCPServer: @staticmethod def get( @@ -50,17 +64,17 @@ class ScraplingMCPServer: extraction_type: extraction_types = "markdown", css_selector: Optional[str] = None, main_content_only: bool = True, - params: Optional[Dict | List | Tuple] = None, + params: Optional[Dict] = None, headers: Optional[Mapping[str, Optional[str]]] = None, - cookies: Optional[Dict[str, str] | list[tuple[str, str]]] = None, + cookies: Optional[Dict[str, str]] = None, timeout: Optional[int | float] = 30, follow_redirects: bool = True, max_redirects: int = 30, retries: Optional[int] = 3, retry_delay: Optional[int] = 1, proxy: Optional[str] = None, - proxy_auth: Optional[Tuple[str, str]] = None, - auth: Optional[Tuple[str, str]] = None, + proxy_auth: Optional[Dict[str, str]] = None, + auth: Optional[Dict[str, str]] = None, verify: Optional[bool] = True, http3: Optional[bool] = False, stealthy_headers: Optional[bool] = True, @@ -87,20 +101,23 @@ class ScraplingMCPServer: :param retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second. :param proxy: Proxy URL to use. Format: "http://username:password@localhost:8030". Cannot be used together with the `proxies` parameter. - :param proxy_auth: HTTP basic auth for proxy, tuple of (username, password). - :param auth: HTTP basic auth tuple of (username, password). Only basic auth is supported. + :param proxy_auth: HTTP basic auth for proxy in dictionary format with `username` and `password` keys. + :param auth: HTTP basic auth in dictionary format with `username` and `password` keys. :param verify: Whether to verify HTTPS certificates. :param http3: Whether to use HTTP3. Defaults to False. It might be problematic if used it with `impersonate`. :param stealthy_headers: If enabled (default), it creates and adds real browser headers. It also sets the referer header as if this request came from a Google search of URL's domain. """ + normalized_proxy_auth = _normalize_credentials(proxy_auth) + normalized_auth = _normalize_credentials(auth) + page = Fetcher.get( url, - auth=auth, + auth=normalized_auth, proxy=proxy, http3=http3, verify=verify, params=params, - proxy_auth=proxy_auth, + proxy_auth=normalized_proxy_auth, retry_delay=retry_delay, stealthy_headers=stealthy_headers, impersonate=impersonate, @@ -111,7 +128,7 @@ class ScraplingMCPServer: max_redirects=max_redirects, follow_redirects=follow_redirects, ) - return _ContentTranslator( + return _content_translator( Convertor._extract_content( page, css_selector=css_selector, @@ -123,22 +140,22 @@ class ScraplingMCPServer: @staticmethod async def bulk_get( - urls: Tuple[str, ...], + urls: List[str], impersonate: ImpersonateType = "chrome", extraction_type: extraction_types = "markdown", css_selector: Optional[str] = None, main_content_only: bool = True, - params: Optional[Dict | List | Tuple] = None, + params: Optional[Dict] = None, headers: Optional[Mapping[str, Optional[str]]] = None, - cookies: Optional[Dict[str, str] | list[tuple[str, str]]] = None, + cookies: Optional[Dict[str, str]] = None, timeout: Optional[int | float] = 30, follow_redirects: bool = True, max_redirects: int = 30, retries: Optional[int] = 3, retry_delay: Optional[int] = 1, proxy: Optional[str] = None, - proxy_auth: Optional[Tuple[str, str]] = None, - auth: Optional[Tuple[str, str]] = None, + proxy_auth: Optional[Dict[str, str]] = None, + auth: Optional[Dict[str, str]] = None, verify: Optional[bool] = True, http3: Optional[bool] = False, stealthy_headers: Optional[bool] = True, @@ -147,7 +164,7 @@ class ScraplingMCPServer: Note: This is only suitable for low-mid protection levels. For high-protection levels or websites that require JS loading, use the other tools directly. Note: If the `css_selector` resolves to more than one element, all the elements will be returned. - :param urls: A tuple of the URLs to request. + :param urls: A list of the URLs to request. :param impersonate: Browser version to impersonate its fingerprint. It's using the latest chrome version by default. :param extraction_type: The type of content to extract from the page. Defaults to "markdown". Options are: - Markdown will convert the page content to Markdown format. @@ -165,17 +182,20 @@ class ScraplingMCPServer: :param retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second. :param proxy: Proxy URL to use. Format: "http://username:password@localhost:8030". Cannot be used together with the `proxies` parameter. - :param proxy_auth: HTTP basic auth for proxy, tuple of (username, password). - :param auth: HTTP basic auth tuple of (username, password). Only basic auth is supported. + :param proxy_auth: HTTP basic auth for proxy in dictionary format with `username` and `password` keys. + :param auth: HTTP basic auth in dictionary format with `username` and `password` keys. :param verify: Whether to verify HTTPS certificates. :param http3: Whether to use HTTP3. Defaults to False. It might be problematic if used it with `impersonate`. :param stealthy_headers: If enabled (default), it creates and adds real browser headers. It also sets the referer header as if this request came from a Google search of URL's domain. """ + normalized_proxy_auth = _normalize_credentials(proxy_auth) + normalized_auth = _normalize_credentials(auth) + async with FetcherSession() as session: tasks: List[Any] = [ session.get( url, - auth=auth, + auth=normalized_auth, proxy=proxy, http3=http3, verify=verify, @@ -184,7 +204,7 @@ class ScraplingMCPServer: cookies=cookies, timeout=timeout, retries=retries, - proxy_auth=proxy_auth, + proxy_auth=normalized_proxy_auth, retry_delay=retry_delay, impersonate=impersonate, max_redirects=max_redirects, @@ -195,7 +215,7 @@ class ScraplingMCPServer: ] responses = await gather(*tasks) return [ - _ContentTranslator( + _content_translator( Convertor._extract_content( page, css_selector=css_selector, @@ -279,7 +299,7 @@ class ScraplingMCPServer: disable_resources=disable_resources, wait_selector_state=wait_selector_state, ) - return _ContentTranslator( + return _content_translator( Convertor._extract_content( page, css_selector=css_selector, @@ -291,7 +311,7 @@ class ScraplingMCPServer: @staticmethod async def bulk_fetch( - urls: Tuple[str, ...], + urls: List[str], extraction_type: extraction_types = "markdown", css_selector: Optional[str] = None, main_content_only: bool = True, @@ -316,7 +336,7 @@ class ScraplingMCPServer: Note: This is only suitable for low-mid protection levels. Note: If the `css_selector` resolves to more than one element, all the elements will be returned. - :param urls: A tuple of the URLs to request. + :param urls: A list of the URLs to request. :param extraction_type: The type of content to extract from the page. Defaults to "markdown". Options are: - Markdown will convert the page content to Markdown format. - HTML will return the raw HTML content of the page. @@ -364,7 +384,7 @@ class ScraplingMCPServer: tasks = [session.fetch(url) for url in urls] responses = await gather(*tasks) return [ - _ContentTranslator( + _content_translator( Convertor._extract_content( page, css_selector=css_selector, @@ -463,7 +483,7 @@ class ScraplingMCPServer: disable_resources=disable_resources, wait_selector_state=wait_selector_state, ) - return _ContentTranslator( + return _content_translator( Convertor._extract_content( page, css_selector=css_selector, @@ -475,7 +495,7 @@ class ScraplingMCPServer: @staticmethod async def bulk_stealthy_fetch( - urls: Tuple[str, ...], + urls: List[str], extraction_type: extraction_types = "markdown", css_selector: Optional[str] = None, main_content_only: bool = True, @@ -505,7 +525,7 @@ class ScraplingMCPServer: Note: This is the only suitable fetcher for high protection levels. Note: If the `css_selector` resolves to more than one element, all the elements will be returned. - :param urls: A tuple of the URLs to request. + :param urls: A list of the URLs to request. :param extraction_type: The type of content to extract from the page. Defaults to "markdown". Options are: - Markdown will convert the page content to Markdown format. - HTML will return the raw HTML content of the page. @@ -562,7 +582,7 @@ class ScraplingMCPServer: tasks = [session.fetch(url) for url in urls] responses = await gather(*tasks) return [ - _ContentTranslator( + _content_translator( Convertor._extract_content( page, css_selector=css_selector, diff --git a/scrapling/core/shell.py b/scrapling/core/shell.py index 0bd6efc..80caa99 100644 --- a/scrapling/core/shell.py +++ b/scrapling/core/shell.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- from sys import stderr +from copy import deepcopy from functools import wraps from re import sub as re_sub from collections import namedtuple @@ -571,6 +572,14 @@ class Convertor: return markdownify(body) + @classmethod + def _strip_noise_tags(cls, page: Selector) -> Selector: + """Return a copy of the Selector with noise tags removed.""" + clean_root = deepcopy(page._root) + for element in clean_root.iter(*{"script", "style", "noscript", "svg"}): + element.drop_tree() + return Selector(root=clean_root, url=page.url) + @classmethod def _extract_content( cls, @@ -587,6 +596,7 @@ class Convertor: else: if main_content_only: page = cast(Selector, page.css("body").first) or page + page = cls._strip_noise_tags(page) pages = [page] if not css_selector else cast(Selectors, page.css(css_selector)) for page in pages: @@ -596,7 +606,9 @@ class Convertor: case "html": yield page.html_content case "text": - txt_content = page.get_all_text(strip=True) + txt_content = page.get_all_text( + strip=True, ignore_tags=("script", "style", "noscript", "svg", "iframe") + ) for s in ( "\n", "\r", diff --git a/scrapling/engines/_browsers/_config_tools.py b/scrapling/engines/_browsers/_config_tools.py index 2d7691b..5b9ee2e 100644 --- a/scrapling/engines/_browsers/_config_tools.py +++ b/scrapling/engines/_browsers/_config_tools.py @@ -1,38 +1,4 @@ -from functools import lru_cache - -from scrapling.engines.toolbelt.navigation import js_bypass_path from scrapling.engines.toolbelt.fingerprints import generate_headers __default_useragent__ = generate_headers(browser_mode=True).get("User-Agent") __default_chrome_useragent__ = generate_headers(browser_mode="chrome").get("User-Agent") - - -@lru_cache(1) -def _compiled_stealth_scripts(): - """Pre-read and compile stealth scripts""" - # Basic bypasses nothing fancy as I'm still working on it - # But with adding these bypasses to the above config, it bypasses many online tests like - # https://bot.sannysoft.com/ - # https://kaliiiiiiiiii.github.io/brotector/ - # https://pixelscan.net/ - # https://iphey.com/ - # https://www.browserscan.net/bot-detection <== this one also checks for the CDP runtime fingerprint - # https://arh.antoinevastel.com/bots/areyouheadless/ - # https://prescience-data.github.io/execution-monitor.html - stealth_scripts_paths = tuple( - js_bypass_path(script) - for script in ( - # Order is important - "webdriver_fully.js", - "window_chrome.js", - "navigator_plugins.js", - "notification_permission.js", - "screen_props.js", - "playwright_fingerprint.js", - ) - ) - scripts = [] - for script_path in stealth_scripts_paths: - with open(script_path, "r") as f: - scripts.append(f.read()) - return tuple(scripts) diff --git a/scrapling/engines/_browsers/_stealth.py b/scrapling/engines/_browsers/_stealth.py index 3c9ea58..797a0c1 100644 --- a/scrapling/engines/_browsers/_stealth.py +++ b/scrapling/engines/_browsers/_stealth.py @@ -17,12 +17,11 @@ from scrapling.core._types import Any, Optional, ProxyType, Unpack from scrapling.engines.toolbelt.proxy_rotation import is_proxy_error from scrapling.engines.toolbelt.convertor import Response, ResponseFactory from scrapling.engines.toolbelt.fingerprints import generate_convincing_referer -from scrapling.engines._browsers._config_tools import _compiled_stealth_scripts from scrapling.engines._browsers._types import StealthSession, StealthFetchParams from scrapling.engines._browsers._base import SyncSession, AsyncSession, StealthySessionMixin from scrapling.engines._browsers._validators import validate_fetch as _validate, StealthConfig -__CF_PATTERN__ = re_compile("challenges.cloudflare.com/cdn-cgi/challenge-platform/.*") +__CF_PATTERN__ = re_compile(r"^https?://challenges\.cloudflare\.com/cdn-cgi/challenge-platform/.*") class StealthySession(SyncSession, StealthySessionMixin): @@ -109,14 +108,6 @@ class StealthySession(SyncSession, StealthySessionMixin): else: raise RuntimeError("Session has been already started") - def _initialize_context(self, config, ctx: BrowserContext) -> BrowserContext: - """Initialize the browser context.""" - for script in _compiled_stealth_scripts(): - ctx.add_init_script(script=script) - - ctx = super()._initialize_context(config, ctx) - return ctx - def _cloudflare_solver(self, page: Page) -> None: # pragma: no cover """Solve the cloudflare challenge displayed on the playwright page passed @@ -127,7 +118,7 @@ class StealthySession(SyncSession, StealthySessionMixin): challenge_type = self._detect_cloudflare(ResponseFactory._get_page_content(page)) if not challenge_type: log.error("No Cloudflare challenge found.") - return + return None else: log.info(f'The turnstile version discovered is "{challenge_type}"') if challenge_type == "non-interactive": @@ -136,7 +127,7 @@ class StealthySession(SyncSession, StealthySessionMixin): page.wait_for_timeout(1000) page.wait_for_load_state() log.info("Cloudflare captcha is solved") - return + return None else: box_selector = "#cf_turnstile div, #cf-turnstile div, .turnstile>div>div" @@ -161,7 +152,7 @@ class StealthySession(SyncSession, StealthySessionMixin): if not iframe or not outer_box: if "Just a moment..." not in (ResponseFactory._get_page_content(page)): log.info("Cloudflare captcha is solved") - return + return None outer_box = page.locator(box_selector).last.bounding_box() @@ -171,22 +162,28 @@ class StealthySession(SyncSession, StealthySessionMixin): # Move the mouse to the center of the window, then press and hold the left mouse button page.mouse.click(captcha_x, captcha_y, delay=randint(100, 200), button="left") self._wait_for_networkidle(page) - if iframe is not None: - # Wait for the frame to be removed from the page (with 30s timeout = 300 iterations * 100 ms) + + if challenge_type != "embedded": attempts = 0 - while iframe in page.frames: - if attempts >= 300: - log.info("Cloudflare iframe didn't disappear after 30s, continuing...") + while "Just a moment..." in ResponseFactory._get_page_content(page): + # Wait for the page + if attempts >= 100: + log.info("Cloudflare page didn't disappear after 10s, continuing...") break page.wait_for_timeout(100) attempts += 1 - if challenge_type != "embedded": - page.locator(box_selector).last.wait_for(state="detached") - page.locator(".zone-name-title").wait_for(state="hidden") + + # page.locator(box_selector).last.wait_for(state="detached") + # page.locator(".zone-name-title").wait_for(state="hidden") + self._wait_for_page_stability(page, True, False) - log.info("Cloudflare captcha is solved") - return + if "Just a moment..." not in (ResponseFactory._get_page_content(page)): + log.info("Cloudflare captcha is solved") + return None + else: + log.info("Looks like Cloudflare captcha is still present, solving again") + return self._cloudflare_solver(page) def fetch(self, url: str, **kwargs: Unpack[StealthFetchParams]) -> Response: """Opens up the browser and do your request based on your chosen options. @@ -366,14 +363,6 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin): else: raise RuntimeError("Session has been already started") - async def _initialize_context(self, config: Any, ctx: AsyncBrowserContext) -> AsyncBrowserContext: - """Initialize the browser context.""" - for script in _compiled_stealth_scripts(): - await ctx.add_init_script(script=script) - - ctx = await super()._initialize_context(config, ctx) - return ctx - async def _cloudflare_solver(self, page: async_Page) -> None: # pragma: no cover """Solve the cloudflare challenge displayed on the playwright page passed @@ -384,7 +373,7 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin): challenge_type = self._detect_cloudflare(await ResponseFactory._get_async_page_content(page)) if not challenge_type: log.error("No Cloudflare challenge found.") - return + return None else: log.info(f'The turnstile version discovered is "{challenge_type}"') if challenge_type == "non-interactive": @@ -393,7 +382,7 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin): await page.wait_for_timeout(1000) await page.wait_for_load_state() log.info("Cloudflare captcha is solved") - return + return None else: box_selector = "#cf_turnstile div, #cf-turnstile div, .turnstile>div>div" @@ -418,7 +407,7 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin): if not iframe or not outer_box: if "Just a moment..." not in (await ResponseFactory._get_async_page_content(page)): log.info("Cloudflare captcha is solved") - return + return None outer_box = await page.locator(box_selector).last.bounding_box() @@ -428,22 +417,28 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin): # Move the mouse to the center of the window, then press and hold the left mouse button await page.mouse.click(captcha_x, captcha_y, delay=randint(100, 200), button="left") await self._wait_for_networkidle(page) - if iframe is not None: - # Wait for the frame to be removed from the page (with 30s timeout = 300 iterations * 100 ms) + + if challenge_type != "embedded": attempts = 0 - while iframe in page.frames: - if attempts >= 300: - log.info("Cloudflare iframe didn't disappear after 30s, continuing...") + while "Just a moment..." in (await ResponseFactory._get_async_page_content(page)): + # Wait for the page + if attempts >= 100: + log.info("Cloudflare page didn't disappear after 10s, continuing...") break await page.wait_for_timeout(100) attempts += 1 - if challenge_type != "embedded": - await page.locator(box_selector).last.wait_for(state="detached") - await page.locator(".zone-name-title").wait_for(state="hidden") + + # await page.locator(box_selector).last.wait_for(state="detached") + # await page.locator(".zone-name-title").wait_for(state="hidden") + await self._wait_for_page_stability(page, True, False) - log.info("Cloudflare captcha is solved") - return + if "Just a moment..." not in (await ResponseFactory._get_async_page_content(page)): + log.info("Cloudflare captcha is solved") + return None + else: + log.info("Looks like Cloudflare captcha is still present, solving again") + return await self._cloudflare_solver(page) async def fetch(self, url: str, **kwargs: Unpack[StealthFetchParams]) -> Response: """Opens up the browser and do your request based on your chosen options. diff --git a/scrapling/engines/toolbelt/bypasses/navigator_plugins.js b/scrapling/engines/toolbelt/bypasses/navigator_plugins.js deleted file mode 100644 index 653fa5f..0000000 --- a/scrapling/engines/toolbelt/bypasses/navigator_plugins.js +++ /dev/null @@ -1,40 +0,0 @@ -if(navigator.plugins.length == 0){ - Object.defineProperty(navigator, 'plugins', { - get: () => { - const PDFViewerPlugin = Object.create(Plugin.prototype, { - description: { value: 'Portable Document Format', enumerable: false }, - filename: { value: 'internal-pdf-viewer', enumerable: false }, - name: { value: 'PDF Viewer', enumerable: false }, - }); - const ChromePDFViewer = Object.create(Plugin.prototype, { - description: { value: 'Portable Document Format', enumerable: false }, - filename: { value: 'internal-pdf-viewer', enumerable: false }, - name: { value: 'Chrome PDF Viewer', enumerable: false }, - }); - const ChromiumPDFViewer = Object.create(Plugin.prototype, { - description: { value: 'Portable Document Format', enumerable: false }, - filename: { value: 'internal-pdf-viewer', enumerable: false }, - name: { value: 'Chromium PDF Viewer', enumerable: false }, - }); - const EdgePDFViewer = Object.create(Plugin.prototype, { - description: { value: 'Portable Document Format', enumerable: false }, - filename: { value: 'internal-pdf-viewer', enumerable: false }, - name: { value: 'Microsoft Edge PDF Viewer', enumerable: false }, - }); - const WebKitPDFPlugin = Object.create(Plugin.prototype, { - description: { value: 'Portable Document Format', enumerable: false }, - filename: { value: 'internal-pdf-viewer', enumerable: false }, - name: { value: 'WebKit built-in PDF', enumerable: false }, - }); - - return Object.create(PluginArray.prototype, { - length: { value: 5 }, - 0: { value: PDFViewerPlugin }, - 1: { value: ChromePDFViewer }, - 2: { value: ChromiumPDFViewer }, - 3: { value: EdgePDFViewer }, - 4: { value: WebKitPDFPlugin }, - }); - }, - }); -} \ No newline at end of file diff --git a/scrapling/engines/toolbelt/bypasses/notification_permission.js b/scrapling/engines/toolbelt/bypasses/notification_permission.js deleted file mode 100644 index 0c9c676..0000000 --- a/scrapling/engines/toolbelt/bypasses/notification_permission.js +++ /dev/null @@ -1,5 +0,0 @@ -// Bypasses `notificationIsDenied` test in creepsjs's 'Like Headless' sections -const isSecure = document.location.protocol.startsWith('https') -if (isSecure){ - Object.defineProperty(Notification, 'permission', {get: () => 'default'}) -} \ No newline at end of file diff --git a/scrapling/engines/toolbelt/bypasses/playwright_fingerprint.js b/scrapling/engines/toolbelt/bypasses/playwright_fingerprint.js deleted file mode 100644 index e1b959d..0000000 --- a/scrapling/engines/toolbelt/bypasses/playwright_fingerprint.js +++ /dev/null @@ -1,3 +0,0 @@ -// Remove playwright fingerprint => https://github.com/microsoft/playwright/commit/c9e673c6dca746384338ab6bb0cf63c7e7caa9b2#diff-087773eea292da9db5a3f27de8f1a2940cdb895383ad750c3cd8e01772a35b40R915 -delete window.__pwInitScripts; -delete window.__playwright__binding__; \ No newline at end of file diff --git a/scrapling/engines/toolbelt/bypasses/screen_props.js b/scrapling/engines/toolbelt/bypasses/screen_props.js deleted file mode 100644 index 5056b4b..0000000 --- a/scrapling/engines/toolbelt/bypasses/screen_props.js +++ /dev/null @@ -1,27 +0,0 @@ -const windowScreenProps = { - // Dimensions - innerHeight: 0, - innerWidth: 0, - outerHeight: 754, - outerWidth: 1313, - - // Position - screenX: 19, - pageXOffset: 0, - pageYOffset: 0, - - // Display - devicePixelRatio: 2 -}; - -try { - for (const [prop, value] of Object.entries(windowScreenProps)) { - if (value > 0) { - // The 0 values are introduced by collecting in the hidden iframe. - // They are document sizes anyway so no need to test them or inject them. - window[prop] = value; - } - } -} catch (e) { - console.warn(e); -}; \ No newline at end of file diff --git a/scrapling/engines/toolbelt/bypasses/webdriver_fully.js b/scrapling/engines/toolbelt/bypasses/webdriver_fully.js deleted file mode 100644 index 4bda260..0000000 --- a/scrapling/engines/toolbelt/bypasses/webdriver_fully.js +++ /dev/null @@ -1,27 +0,0 @@ -// Create a function that looks like a native getter -const nativeGetter = function get webdriver() { - return false; -}; - -// Copy over native function properties -Object.defineProperties(nativeGetter, { - name: { value: 'get webdriver', configurable: true }, - length: { value: 0, configurable: true }, - toString: { - value: function() { - return `function get webdriver() { [native code] }`; - }, - configurable: true - } -}); - -// Make it look native -Object.setPrototypeOf(nativeGetter, Function.prototype); - -// Apply the modified descriptor -Object.defineProperty(Navigator.prototype, 'webdriver', { - get: nativeGetter, - set: undefined, - enumerable: true, - configurable: true -}); \ No newline at end of file diff --git a/scrapling/engines/toolbelt/bypasses/window_chrome.js b/scrapling/engines/toolbelt/bypasses/window_chrome.js deleted file mode 100644 index ba63a9c..0000000 --- a/scrapling/engines/toolbelt/bypasses/window_chrome.js +++ /dev/null @@ -1,213 +0,0 @@ -// To escape `HEADCHR_CHROME_OBJ` test in headless mode => https://github.com/antoinevastel/fp-collect/blob/master/src/fpCollect.js#L322 -// Faking window.chrome fully - -if (!window.chrome) { - // First, save all existing properties - const originalKeys = Object.getOwnPropertyNames(window); - const tempObj = {}; - - // Recreate all properties in original order - for (const key of originalKeys) { - const descriptor = Object.getOwnPropertyDescriptor(window, key); - const value = window[key]; - // delete window[key]; - Object.defineProperty(tempObj, key, descriptor); - } - - // Use the exact property descriptor found in headful Chrome - // fetch it via `Object.getOwnPropertyDescriptor(window, 'chrome')` - const mockChrome = { - loadTimes: {}, - csi: {}, - app: { - isInstalled: false - }, - // Add other Chrome-specific properties - }; - - Object.defineProperty(tempObj, 'chrome', { - writable: true, - enumerable: true, - configurable: false, - value: mockChrome - }); - for (const key of Object.getOwnPropertyNames(tempObj)) { - try { - Object.defineProperty(window, key, - Object.getOwnPropertyDescriptor(tempObj, key)); - } catch (e) {} - }; - // todo: solve this - // Using line below bypasses the hasHighChromeIndex test in creepjs ==> https://github.com/abrahamjuliot/creepjs/blob/master/src/headless/index.ts#L121 - // Chrome object have to be in the end of the window properties - // Object.assign(window, tempObj); - // But makes window.chrome unreadable on 'https://bot.sannysoft.com/' -} - -// That means we're running headful and don't need to mock anything -if ('app' in window.chrome) { - return; // Nothing to do here -} -const makeError = { - ErrorInInvocation: fn => { - const err = new TypeError(`Error in invocation of app.${fn}()`); - return utils.stripErrorWithAnchor( - err, - `at ${fn} (eval at `, - ); - }, -}; -// check with: `JSON.stringify(window.chrome['app'])` -const STATIC_DATA = JSON.parse( - ` -{ - "isInstalled": false, - "InstallState": { - "DISABLED": "disabled", - "INSTALLED": "installed", - "NOT_INSTALLED": "not_installed" - }, - "RunningState": { - "CANNOT_RUN": "cannot_run", - "READY_TO_RUN": "ready_to_run", - "RUNNING": "running" - } -} - `.trim(), - ); -window.chrome.app = { - ...STATIC_DATA, - - get isInstalled() { - return false; - }, - - getDetails: function getDetails() { - if (arguments.length) { - throw makeError.ErrorInInvocation(`getDetails`); - } - return null; - }, - getIsInstalled: function getDetails() { - if (arguments.length) { - throw makeError.ErrorInInvocation(`getIsInstalled`); - } - return false; - }, - runningState: function getDetails() { - if (arguments.length) { - throw makeError.ErrorInInvocation(`runningState`); - } - return 'cannot_run'; - }, -}; -// Check that the Navigation Timing API v1 is available, we need that -if (!window.performance || !window.performance.timing) { - return; -} -const {timing} = window.performance; -window.chrome.csi = function () { - return { - onloadT: timing.domContentLoadedEventEnd, - startE: timing.navigationStart, - pageT: Date.now() - timing.navigationStart, - tran: 15, // Transition type or something - }; -}; -if (!window.PerformancePaintTiming){ - return; -} -const {performance} = window; -// Some stuff is not available on about:blank as it requires a navigation to occur, -// let's harden the code to not fail then: -const ntEntryFallback = { - nextHopProtocol: 'h2', - type: 'other', -}; - -// The API exposes some funky info regarding the connection -const protocolInfo = { - get connectionInfo() { - const ntEntry = - performance.getEntriesByType('navigation')[0] || ntEntryFallback; - return ntEntry.nextHopProtocol; - }, - get npnNegotiatedProtocol() { - // NPN is deprecated in favor of ALPN, but this implementation returns the - // HTTP/2 or HTTP2+QUIC/39 requests negotiated via ALPN. - const ntEntry = - performance.getEntriesByType('navigation')[0] || ntEntryFallback; - return ['h2', 'hq'].includes(ntEntry.nextHopProtocol) - ? ntEntry.nextHopProtocol - : 'unknown'; - }, - get navigationType() { - const ntEntry = - performance.getEntriesByType('navigation')[0] || ntEntryFallback; - return ntEntry.type; - }, - get wasAlternateProtocolAvailable() { - // The Alternate-Protocol header is deprecated in favor of Alt-Svc - // (https://www.mnot.net/blog/2016/03/09/alt-svc), so technically this - // should always return false. - return false; - }, - get wasFetchedViaSpdy() { - // SPDY is deprecated in favor of HTTP/2, but this implementation returns - // true for HTTP/2 or HTTP2+QUIC/39 as well. - const ntEntry = - performance.getEntriesByType('navigation')[0] || ntEntryFallback; - return ['h2', 'hq'].includes(ntEntry.nextHopProtocol); - }, - get wasNpnNegotiated() { - // NPN is deprecated in favor of ALPN, but this implementation returns true - // for HTTP/2 or HTTP2+QUIC/39 requests negotiated via ALPN. - const ntEntry = - performance.getEntriesByType('navigation')[0] || ntEntryFallback; - return ['h2', 'hq'].includes(ntEntry.nextHopProtocol); - }, -}; - -// Truncate number to specific number of decimals, most of the `loadTimes` stuff has 3 -function toFixed(num, fixed) { - var re = new RegExp('^-?\\d+(?:.\\d{0,' + (fixed || -1) + '})?'); - return num.toString().match(re)[0]; -} - -const timingInfo = { - get firstPaintAfterLoadTime() { - // This was never actually implemented and always returns 0. - return 0; - }, - get requestTime() { - return timing.navigationStart / 1000; - }, - get startLoadTime() { - return timing.navigationStart / 1000; - }, - get commitLoadTime() { - return timing.responseStart / 1000; - }, - get finishDocumentLoadTime() { - return timing.domContentLoadedEventEnd / 1000; - }, - get finishLoadTime() { - return timing.loadEventEnd / 1000; - }, - get firstPaintTime() { - const fpEntry = performance.getEntriesByType('paint')[0] || { - startTime: timing.loadEventEnd / 1000, // Fallback if no navigation occured (`about:blank`) - }; - return toFixed( - (fpEntry.startTime + performance.timeOrigin) / 1000, - 3, - ); - }, -}; - -window.chrome.loadTimes = function () { - return { - ...protocolInfo, - ...timingInfo, - }; -}; \ No newline at end of file diff --git a/scrapling/engines/toolbelt/navigation.py b/scrapling/engines/toolbelt/navigation.py index 5c19427..9b233d8 100644 --- a/scrapling/engines/toolbelt/navigation.py +++ b/scrapling/engines/toolbelt/navigation.py @@ -2,8 +2,6 @@ Functions related to files and URLs """ -from pathlib import Path -from functools import lru_cache from urllib.parse import urlparse from playwright.async_api import Route as async_Route @@ -14,8 +12,6 @@ from scrapling.core.utils import log from scrapling.core._types import Dict, Set, Tuple, Optional, Callable from scrapling.engines.constants import EXTRA_RESOURCES -__BYPASSES_DIR__ = Path(__file__).parent / "bypasses" - class ProxyDict(Struct): server: str @@ -111,13 +107,3 @@ def construct_proxy_dict(proxy_string: str | Dict[str, str] | Tuple) -> Dict: raise TypeError(f"Invalid proxy dictionary: {e}") raise TypeError(f"Invalid proxy string: {proxy_string}") - - -@lru_cache(10, typed=True) -def js_bypass_path(filename: str) -> str: - """Takes the base filename of a JS file inside the `bypasses` folder, then return the full path of it - - :param filename: The base filename of the JS file. - :return: The full path of the JS file. - """ - return str(__BYPASSES_DIR__ / filename) diff --git a/server.json b/server.json new file mode 100644 index 0000000..fa9a08e --- /dev/null +++ b/server.json @@ -0,0 +1,50 @@ +{ + "$schema": "https://static.modelcontextprotocol.io/schemas/2025-12-11/server.schema.json", + "name": "io.github.D4Vinci/Scrapling", + "title": "Scrapling MCP Server", + "description": "Web scraping with stealth HTTP, real browsers, and Cloudflare bypass. CSS selectors supported.", + "websiteUrl": "https://scrapling.readthedocs.io/en/latest/ai/mcp-server.html", + "repository": { + "url": "https://github.com/D4Vinci/Scrapling", + "source": "github" + }, + "icons": [ + { + "src": "https://raw.githubusercontent.com/D4Vinci/Scrapling/main/docs/assets/logo.png", + "mimeType": "image/png" + } + ], + "version": "0.4.1", + "packages": [ + { + "registryType": "pypi", + "identifier": "scrapling", + "version": "0.4.1", + "runtimeHint": "uvx", + "packageArguments": [ + { + "type": "positional", + "valueHint": "mcp", + "isFixed": true + } + ], + "transport": { + "type": "stdio" + } + }, + { + "registryType": "oci", + "identifier": "ghcr.io/d4vinci/scrapling", + "packageArguments": [ + { + "type": "positional", + "valueHint": "mcp", + "isFixed": true + } + ], + "transport": { + "type": "stdio" + } + } + ] +} \ No newline at end of file diff --git a/setup.cfg b/setup.cfg index d8aedf7..bad13d4 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = scrapling -version = 0.4 +version = 0.4.1 author = Karim Shoair author_email = karim.shoair@pm.me description = Scrapling is an undetectable, powerful, flexible, high-performance Python library that makes Web Scraping easy and effortless as it should be! diff --git a/tests/fetchers/test_utils.py b/tests/fetchers/test_utils.py index 4637a63..0a4be9f 100644 --- a/tests/fetchers/test_utils.py +++ b/tests/fetchers/test_utils.py @@ -1,12 +1,10 @@ import pytest -from pathlib import Path from scrapling.engines.toolbelt.custom import StatusText, Response from scrapling.engines.toolbelt.navigation import ( construct_proxy_dict, create_intercept_handler, create_async_intercept_handler, - js_bypass_path, ) from scrapling.engines.toolbelt.fingerprints import ( generate_convincing_referer, @@ -203,25 +201,6 @@ class TestConstructProxyDict: construct_proxy_dict({"invalid": "structure"}) -class TestJsBypassPath: - """Test JavaScript bypass path utility""" - - def test_js_bypass_path(self): - """Test getting JavaScript bypass file path""" - result = js_bypass_path("webdriver_fully.js") - - assert isinstance(result, str) - assert result.endswith("webdriver_fully.js") - assert Path(result).exists() - - def test_js_bypass_path_caching(self): - """Test that js_bypass_path is cached""" - result1 = js_bypass_path("webdriver_fully.js") - result2 = js_bypass_path("webdriver_fully.js") - - assert result1 == result2 - - class TestFingerprintFunctions: """Test fingerprint generation functions"""