freedak/Scrapling
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: better implementation for the mcp mode

Browse Source
This commit is contained in:
Karim shoair
2025-09-28 20:14:04 +03:00
parent 0b729176bc
commit 74f20d2a0e
2 changed files with 28 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scrapling/cli.py
+2 -2
View File
@@ -154,8 +154,8 @@ def install(force): # pragma: no cover
def mcp(http, host, port): def mcp(http, host, port):
from scrapling.core.ai import ScraplingMCPServer from scrapling.core.ai import ScraplingMCPServer
server = ScraplingMCPServer(host, port) server = ScraplingMCPServer()
server.run(transport="stdio" if not http else "streamable-http") server.serve(http, host, port)
@command(help="Interactive scraping console") @command(help="Interactive scraping console")
scrapling/core/ai.py
+26 -10
View File
@@ -41,10 +41,8 @@ def _ContentTranslator(content: Generator[str, None, None], page: _ScraplingResp
return ResponseModel(status=page.status, content=[result for result in content], url=page.url) return ResponseModel(status=page.status, content=[result for result in content], url=page.url)
def ScraplingMCPServer(host: str, port: int) -> FastMCP: class ScraplingMCPServer:
_server = FastMCP(name="Scrapling", host=host, port=port) @staticmethod
@_server.tool()
def get( def get(
url: str, url: str,
impersonate: Optional[BrowserTypeLiteral] = "chrome", impersonate: Optional[BrowserTypeLiteral] = "chrome",
@@ -122,7 +120,7 @@ def ScraplingMCPServer(host: str, port: int) -> FastMCP:
page, page,
) )
@_server.tool() @staticmethod
async def bulk_get( async def bulk_get(
urls: Tuple[str, ...], urls: Tuple[str, ...],
impersonate: Optional[BrowserTypeLiteral] = "chrome", impersonate: Optional[BrowserTypeLiteral] = "chrome",
@@ -208,7 +206,7 @@ def ScraplingMCPServer(host: str, port: int) -> FastMCP:
for page in responses for page in responses
] ]
@_server.tool() @staticmethod
async def fetch( async def fetch(
url: str, url: str,
extraction_type: extraction_types = "markdown", extraction_type: extraction_types = "markdown",
@@ -296,7 +294,7 @@ def ScraplingMCPServer(host: str, port: int) -> FastMCP:
page, page,
) )
@_server.tool() @staticmethod
async def bulk_fetch( async def bulk_fetch(
urls: Tuple[str, ...], urls: Tuple[str, ...],
extraction_type: extraction_types = "markdown", extraction_type: extraction_types = "markdown",
@@ -389,7 +387,7 @@ def ScraplingMCPServer(host: str, port: int) -> FastMCP:
for page in responses for page in responses
] ]
@_server.tool() @staticmethod
async def stealthy_fetch( async def stealthy_fetch(
url: str, url: str,
extraction_type: extraction_types = "markdown", extraction_type: extraction_types = "markdown",
@@ -488,7 +486,7 @@ def ScraplingMCPServer(host: str, port: int) -> FastMCP:
page, page,
) )
@_server.tool() @staticmethod
async def bulk_stealthy_fetch( async def bulk_stealthy_fetch(
urls: Tuple[str, ...], urls: Tuple[str, ...],
extraction_type: extraction_types = "markdown", extraction_type: extraction_types = "markdown",
@@ -592,4 +590,22 @@ def ScraplingMCPServer(host: str, port: int) -> FastMCP:
for page in responses for page in responses
] ]
return _server def serve(self, http: bool, host: str, port: int):
"""Serve the MCP server."""
server = FastMCP(name="Scrapling", host=host, port=port)
server.add_tool(self.get, title="get", description=self.get.__doc__, structured_output=True)
server.add_tool(self.bulk_get, title="bulk_get", description=self.bulk_get.__doc__, structured_output=True)
server.add_tool(self.fetch, title="fetch", description=self.fetch.__doc__, structured_output=True)
server.add_tool(
self.bulk_fetch, title="bulk_fetch", description=self.bulk_fetch.__doc__, structured_output=True
)
server.add_tool(
self.stealthy_fetch, title="stealthy_fetch", description=self.stealthy_fetch.__doc__, structured_output=True
)
server.add_tool(
self.bulk_stealthy_fetch,
title="bulk_stealthy_fetch",
description=self.bulk_stealthy_fetch.__doc__,
structured_output=True,
)
server.run(transport="stdio" if not http else "streamable-http")