comfyui: add native ComfyUI provider for image, video, and music generation

Adds three new BaseTool providers that delegate GPU work to a running
ComfyUI server via its REST API.  This avoids the need to install
PyTorch/diffusers directly, which is critical on hardware where the
ecosystem hasn't caught up (e.g. NVIDIA Blackwell / DGX Spark, aarch64
+ CUDA 13.0).

New files:
- tools/_comfyui/client.py — shared REST client (submit/poll/download)
- tools/_comfyui/workflows/ — 4 bundled workflow templates
- tools/graphics/comfyui_image.py — FLUX 2 Dev NVFP4 text-to-image
- tools/video/comfyui_video.py — WAN 2.2 14B t2v + i2v (4-step LightX2V)
- tools/audio/comfyui_music.py — ACE-Step 3.5B music generation
- tests/contracts/test_comfyui_tools.py — 41 contract tests
- docs/comfyui-adapter-plan.md — design document

Zero changes to existing tools, selectors, registry, or pipelines.
Tools are auto-discovered and selectors pick them up via capability match.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
martimramos
2026-04-16 23:59:33 +01:00
committed by Alastair Beal
parent 80e51fd618
commit 6ec2bbb090
11 changed files with 1793 additions and 0 deletions
+173
View File
@@ -0,0 +1,173 @@
"""Contract tests for ComfyUI provider tools.
These tests verify that the tools satisfy the BaseTool contract without
requiring a running ComfyUI server. They check class attributes,
schemas, status reporting, and cost estimates.
"""
import json
from pathlib import Path
import pytest
from tools.base_tool import (
BaseTool,
ToolRuntime,
ToolStability,
ToolStatus,
ToolTier,
)
from tools.graphics.comfyui_image import ComfyUIImage
from tools.video.comfyui_video import ComfyUIVideo
from tools.audio.comfyui_music import ComfyUIMusic
TOOLS = [ComfyUIImage, ComfyUIVideo, ComfyUIMusic]
WORKFLOW_DIR = Path(__file__).resolve().parent.parent.parent / "tools" / "_comfyui" / "workflows"
# ------------------------------------------------------------------
# Contract compliance
# ------------------------------------------------------------------
@pytest.mark.parametrize("cls", TOOLS, ids=lambda c: c.name)
class TestContract:
def test_inherits_base_tool(self, cls):
assert issubclass(cls, BaseTool)
def test_has_required_identity(self, cls):
tool = cls()
assert tool.name
assert tool.version
assert tool.capability
assert tool.provider == "comfyui"
assert tool.tier == ToolTier.GENERATE
assert tool.stability == ToolStability.EXPERIMENTAL
assert tool.runtime == ToolRuntime.LOCAL_GPU
def test_has_input_schema(self, cls):
tool = cls()
schema = tool.input_schema
assert schema.get("type") == "object"
assert "prompt" in schema.get("properties", {})
assert "prompt" in schema.get("required", [])
def test_has_capabilities(self, cls):
tool = cls()
assert len(tool.capabilities) > 0
def test_has_fallbacks(self, cls):
tool = cls()
assert tool.fallback or tool.fallback_tools
def test_cost_is_zero(self, cls):
tool = cls()
assert tool.estimate_cost({"prompt": "test"}) == 0.0
def test_runtime_estimate_positive(self, cls):
tool = cls()
assert tool.estimate_runtime({"prompt": "test"}) > 0
def test_get_info_returns_dict(self, cls):
tool = cls()
info = tool.get_info()
assert isinstance(info, dict)
assert info["name"] == tool.name
assert info["provider"] == "comfyui"
assert info["runtime"] == "local_gpu"
def test_status_unavailable_without_server(self, cls):
"""Without a running server, status should be UNAVAILABLE."""
tool = cls()
# Point to a port that's almost certainly not running ComfyUI
tool._client.server_url = "http://127.0.0.1:19999"
assert tool.get_status() == ToolStatus.UNAVAILABLE
def test_idempotency_key_fields(self, cls):
tool = cls()
assert len(tool.idempotency_key_fields) > 0
assert "prompt" in tool.idempotency_key_fields
# ------------------------------------------------------------------
# Workflow files
# ------------------------------------------------------------------
EXPECTED_WORKFLOWS = [
"flux2-txt2img.json",
"wan22-i2v-4step.json",
"wan22-t2v-4step.json",
"ace-step-music.json",
]
@pytest.mark.parametrize("filename", EXPECTED_WORKFLOWS)
def test_workflow_exists_and_valid_json(filename):
path = WORKFLOW_DIR / filename
assert path.exists(), f"Missing workflow: {path}"
with open(path) as f:
data = json.load(f)
assert isinstance(data, dict)
assert len(data) > 0
def test_flux2_workflow_has_templated_nodes():
with open(WORKFLOW_DIR / "flux2-txt2img.json") as f:
w = json.load(f)
assert "4" in w # CLIPTextEncode (prompt)
assert "7" in w # RandomNoise (seed)
assert "13" in w # SaveImage (output)
def test_i2v_workflow_has_templated_nodes():
with open(WORKFLOW_DIR / "wan22-i2v-4step.json") as f:
w = json.load(f)
assert "93" in w # CLIPTextEncode (prompt)
assert "97" in w # LoadImage (reference)
assert "86" in w # KSamplerAdvanced (seed)
assert "108" in w # SaveVideo (output)
def test_t2v_workflow_has_templated_nodes():
with open(WORKFLOW_DIR / "wan22-t2v-4step.json") as f:
w = json.load(f)
assert "2" in w # CLIPTextEncode (prompt)
assert "12" in w # KSamplerAdvanced (seed)
assert "16" in w # SaveVideo (output)
# ------------------------------------------------------------------
# Client unit tests
# ------------------------------------------------------------------
class TestClientHelpers:
def test_load_workflow(self):
from tools._comfyui.client import ComfyUIClient
w = ComfyUIClient.load_workflow(WORKFLOW_DIR / "flux2-txt2img.json")
assert isinstance(w, dict)
assert "1" in w
def test_patch_workflow(self):
from tools._comfyui.client import ComfyUIClient
w = ComfyUIClient.load_workflow(WORKFLOW_DIR / "flux2-txt2img.json")
patched = ComfyUIClient.patch_workflow(w, {
"4": {"text": "hello world"},
"7": {"noise_seed": 123},
})
assert patched["4"]["inputs"]["text"] == "hello world"
assert patched["7"]["inputs"]["noise_seed"] == 123
# Original unchanged
assert w["4"]["inputs"]["text"] == ""
def test_patch_workflow_bad_node(self):
from tools._comfyui.client import ComfyUIClient, ComfyUIError
w = {"1": {"inputs": {"x": 1}}}
with pytest.raises(ComfyUIError, match="not found"):
ComfyUIClient.patch_workflow(w, {"99": {"x": 2}})
def test_random_seed_range(self):
from tools._comfyui.client import ComfyUIClient
for _ in range(100):
s = ComfyUIClient.random_seed()
assert 0 <= s < 2**32