feat: Make mcp able to use http transport

scrapling/cli.py

@@ -136,10 +136,26 @@ def install(force):  # pragma: no cover
 
 
 @command(help="Run Scrapling's MCP server (Check the docs for more info).")
-def mcp():
+@option(
+    "--http",
+    type=bool,
+    default=False,
+    help="Whether to run the MCP server in streamable-http transport or leave it as stdio (Default: False)",
+)
+@option(
+    "--host",
+    type=str,
+    default="0.0.0.0",
+    help="The host to use if streamable-http transport is enabled (Default: '0.0.0.0')",
+)
+@option(
+    "--port", type=int, default=8000, help="The port to use if streamable-http transport is enabled (Default: 8000)"
+)
+def mcp(http, host, port):
     from scrapling.core.ai import ScraplingMCPServer
 
-    ScraplingMCPServer().serve()
+    server = ScraplingMCPServer(host, port)
+    server.run(transport="stdio" if not http else "streamable-http")
 
 
 @command(help="Interactive scraping console")

scrapling/core/ai.py

@@ -41,10 +41,9 @@ def _ContentTranslator(content: Generator[str, None, None], page: _ScraplingResp
     return ResponseModel(status=page.status, content=[result for result in content], url=page.url)
 
 
-class ScraplingMCPServer:
-    _server = FastMCP(name="Scrapling")
+def ScraplingMCPServer(host: str, port: int) -> FastMCP:
+    _server = FastMCP(name="Scrapling", host=host, port=port)
 
-    @staticmethod
     @_server.tool()
     def get(
         url: str,
@@ -123,7 +122,6 @@ class ScraplingMCPServer:
             page,
         )
 
-    @staticmethod
     @_server.tool()
     async def bulk_get(
         urls: Tuple[str, ...],
@@ -210,7 +208,6 @@ class ScraplingMCPServer:
                 for page in responses
             ]
 
-    @staticmethod
     @_server.tool()
     async def fetch(
         url: str,
@@ -299,7 +296,6 @@ class ScraplingMCPServer:
             page,
         )
 
-    @staticmethod
     @_server.tool()
     async def bulk_fetch(
         urls: Tuple[str, ...],
@@ -393,7 +389,6 @@ class ScraplingMCPServer:
                 for page in responses
             ]
 
-    @staticmethod
     @_server.tool()
     async def stealthy_fetch(
         url: str,
@@ -493,7 +488,6 @@ class ScraplingMCPServer:
             page,
         )
 
-    @staticmethod
     @_server.tool()
     async def bulk_stealthy_fetch(
         urls: Tuple[str, ...],
@@ -598,6 +592,4 @@ class ScraplingMCPServer:
                 for page in responses
             ]
 
-    def serve(self):
-        """Serve the MCP server."""
-        self._server.run(transport="stdio")
+    return _server