freedak/OpenMontage
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Initial release — OpenMontage: the first open-source agentic video production system

Browse Source 
11 production pipelines, 47 tools, 124 agent skills.
Supports cloud APIs (fal.ai, OpenAI, ElevenLabs, Suno, HeyGen, Runway) and
free local providers (diffusers, Piper TTS, WAN 2.1, Hunyuan, CogVideo).

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
calesthio
2026-03-29 08:25:17 -07:00
commit a3e735cc7a
1147 changed files with 240221 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/video/__init__.py
+1
View File
@@ -0,0 +1 @@
"""Video tools — generation providers, composition, stitching, and trimming."""
tools/video/_shared.py
+576
View File
@@ -0,0 +1,576 @@
"""Shared helpers for provider-specific video generation tools."""
from __future__ import annotations
import os
import shutil
import subprocess
import time
from pathlib import Path
from typing import Any
from tools.base_tool import ToolResult, ToolStatus
HEYGEN_PROVIDERS = {
"veo_3_1": {"name": "Google VEO 3.1", "quality": "highest", "speed": "slow"},
"veo_3_1_fast": {"name": "Google VEO 3.1 Fast", "quality": "high", "speed": "medium"},
"veo3": {"name": "Google VEO 3", "quality": "high", "speed": "slow"},
"veo3_fast": {"name": "Google VEO 3 Fast", "quality": "high", "speed": "medium"},
"veo2": {"name": "Google VEO 2", "quality": "medium", "speed": "medium"},
"kling_pro": {"name": "Kling Pro", "quality": "high", "speed": "medium"},
"kling_v2": {"name": "Kling v2", "quality": "medium", "speed": "fast"},
"sora_v2": {"name": "Sora v2", "quality": "high", "speed": "slow"},
"sora_v2_pro": {"name": "Sora v2 Pro", "quality": "highest", "speed": "slow"},
"runway_gen4": {"name": "Runway Gen-4", "quality": "high", "speed": "medium"},
"seedance_lite": {"name": "Seedance Lite", "quality": "medium", "speed": "fast"},
"seedance_pro": {"name": "Seedance Pro", "quality": "high", "speed": "medium"},
"ltx_distilled": {"name": "LTX Distilled", "quality": "low", "speed": "fastest"},
}
WAN_VARIANTS = {
"wan2.1-1.3b": {
"name": "Wan 2.1 (1.3B)",
"hf_id": "Wan-AI/Wan2.1-T2V-1.3B-Diffusers",
"hf_i2v_id": "Wan-AI/Wan2.1-I2V-14B-480P-Diffusers",
"pipeline_class": "WanPipeline",
"vram_mb": 8000,
"quality": "high",
"speed": "medium",
"t2v": True,
"i2v": True,
"license": "Apache-2.0",
"default_width": 832,
"default_height": 480,
"default_num_frames": 81,
"fps": 16,
},
"wan2.1-14b": {
"name": "Wan 2.1 (14B)",
"hf_id": "Wan-AI/Wan2.1-T2V-14B-Diffusers",
"hf_i2v_id": "Wan-AI/Wan2.1-I2V-14B-480P-Diffusers",
"pipeline_class": "WanPipeline",
"vram_mb": 24000,
"quality": "highest",
"speed": "slow",
"t2v": True,
"i2v": True,
"license": "Apache-2.0",
"default_width": 1280,
"default_height": 720,
"default_num_frames": 81,
"fps": 16,
},
}
HUNYUAN_VARIANTS = {
"hunyuan-1.5": {
"name": "HunyuanVideo 1.5",
"hf_id": "tencent/HunyuanVideo-1.5",
"pipeline_class": "HunyuanVideoPipeline",
"vram_mb": 14000,
"quality": "high",
"speed": "medium",
"t2v": True,
"i2v": True,
"license": "Apache-2.0",
"default_width": 848,
"default_height": 480,
"default_num_frames": 121,
"fps": 24,
},
}
LTX_LOCAL_VARIANTS = {
"ltx2-local": {
"name": "LTX-2 (Local)",
"hf_id": "Lightricks/LTX-2",
"pipeline_class": "LTXPipeline",
"vram_mb": 12000,
"quality": "high",
"speed": "medium",
"t2v": True,
"i2v": True,
"license": "LTX-2-Community",
"default_width": 768,
"default_height": 512,
"default_num_frames": 121,
"fps": 24,
},
}
COGVIDEO_VARIANTS = {
"cogvideo-5b": {
"name": "CogVideoX 1.5 (5B)",
"hf_id": "THUDM/CogVideoX-5b",
"pipeline_class": "CogVideoXPipeline",
"vram_mb": 12000,
"quality": "medium",
"speed": "medium",
"t2v": True,
"i2v": True,
"license": "Apache-2.0",
"default_width": 720,
"default_height": 480,
"default_num_frames": 49,
"fps": 8,
},
"cogvideo-2b": {
"name": "CogVideoX (2B)",
"hf_id": "THUDM/CogVideoX-2b",
"pipeline_class": "CogVideoXPipeline",
"vram_mb": 6000,
"quality": "medium",
"speed": "fast",
"t2v": True,
"i2v": False,
"license": "Apache-2.0",
"default_width": 720,
"default_height": 480,
"default_num_frames": 49,
"fps": 8,
},
}
LTX2_FRAME_COUNTS = {
"1s": 25,
"2s": 49,
"3s": 73,
"4s": 97,
"5s": 121,
"6.7s": 161,
"8s": 193,
}
def local_generation_enabled() -> bool:
return os.environ.get("VIDEO_GEN_LOCAL_ENABLED", "").lower() in {"true", "1", "yes"}
def local_generation_status() -> ToolStatus:
if not local_generation_enabled():
return ToolStatus.UNAVAILABLE
try:
import diffusers # noqa: F401
import torch # noqa: F401
except ImportError:
return ToolStatus.UNAVAILABLE
return ToolStatus.AVAILABLE
def local_install_instructions() -> str:
return (
"Enable local video generation and install the diffusers stack:\n"
" set VIDEO_GEN_LOCAL_ENABLED=true\n"
" pip install diffusers transformers accelerate torch pillow requests\n"
"Use a GPU with the VRAM profile listed on the selected tool."
)
def estimate_quality_cost(quality: str) -> float:
if quality == "highest":
return 0.50
if quality == "high":
return 0.35
if quality == "low":
return 0.15
return 0.20
def estimate_speed_runtime(speed: str) -> float:
return {"fastest": 30.0, "fast": 60.0, "medium": 120.0, "slow": 300.0}.get(speed, 120.0)
def estimate_local_runtime(speed: str) -> float:
return {"fast": 120.0, "medium": 240.0, "slow": 600.0}.get(speed, 240.0)
def load_diffusers_pipeline(pipeline_class: str, model_id: str, enable_offload: bool):
import diffusers
import torch
pipeline_map = {
"WanPipeline": "WanPipeline",
"HunyuanVideoPipeline": "HunyuanVideoPipeline",
"LTXPipeline": "LTXPipeline",
"CogVideoXPipeline": "CogVideoXPipeline",
}
pipeline_name = pipeline_map.get(pipeline_class, pipeline_class)
pipeline_class_obj = getattr(diffusers, pipeline_name)
dtype = torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float16
pipeline = pipeline_class_obj.from_pretrained(model_id, torch_dtype=dtype)
if enable_offload:
pipeline.enable_model_cpu_offload()
else:
pipeline = pipeline.to("cuda")
if hasattr(pipeline, "vae") and pipeline.vae is not None:
if hasattr(pipeline.vae, "enable_tiling"):
pipeline.vae.enable_tiling()
if hasattr(pipeline.vae, "enable_slicing"):
pipeline.vae.enable_slicing()
return pipeline
def load_reference_image(inputs: dict[str, Any], width: int, height: int):
from io import BytesIO
import requests
from PIL import Image
ref_path = inputs.get("reference_image_path")
ref_url = inputs.get("reference_image_url")
if ref_path:
image = Image.open(ref_path).convert("RGB")
elif ref_url:
response = requests.get(ref_url, timeout=60)
response.raise_for_status()
image = Image.open(BytesIO(response.content)).convert("RGB")
else:
return ToolResult(
success=False,
error="image_to_video requires reference_image_url or reference_image_path",
)
return image.resize((width, height), Image.LANCZOS)
def generate_local_video(
*,
tool_name: str,
variants: dict[str, dict[str, Any]],
default_variant: str,
inputs: dict[str, Any],
) -> ToolResult:
import torch
from diffusers.utils import export_to_video
variant = inputs.get("model_variant", default_variant)
if variant not in variants:
return ToolResult(
success=False,
error=f"Unknown model_variant: {variant}. Available: {', '.join(sorted(variants))}",
)
meta = variants[variant]
prompt = inputs["prompt"]
operation = inputs.get("operation", "text_to_video")
seed = inputs.get("seed")
enable_offload = inputs.get("enable_model_offload", True)
if operation == "image_to_video" and not meta.get("i2v"):
return ToolResult(
success=False,
error=f"{meta['name']} does not support image_to_video.",
)
width = inputs.get("width", meta["default_width"])
height = inputs.get("height", meta["default_height"])
num_frames = inputs.get("num_frames", meta["default_num_frames"])
fps = meta["fps"]
model_id = meta.get("hf_i2v_id") if operation == "image_to_video" and meta.get("hf_i2v_id") else meta["hf_id"]
pipeline = load_diffusers_pipeline(meta["pipeline_class"], model_id, enable_offload)
generation_args: dict[str, Any] = {
"prompt": prompt,
"num_frames": num_frames,
"width": width,
"height": height,
"num_inference_steps": inputs.get("num_inference_steps", 30),
}
if seed is not None:
generation_args["generator"] = torch.Generator(device="cpu").manual_seed(seed)
if operation == "image_to_video":
image = load_reference_image(inputs, width, height)
if isinstance(image, ToolResult):
return image
generation_args["image"] = image
if meta["pipeline_class"] == "CogVideoXPipeline":
generation_args["negative_prompt"] = "worst quality, low quality, blurry, distorted, watermark"
output = pipeline(**generation_args)
frames = output.frames[0] if hasattr(output, "frames") else output.images
output_path = Path(inputs.get("output_path", f"{tool_name}_{variant}.mp4"))
output_path.parent.mkdir(parents=True, exist_ok=True)
export_to_video(frames, str(output_path), fps=fps)
return ToolResult(
success=True,
data={
"provider": tool_name,
"model_variant": variant,
"provider_name": meta["name"],
"mode": "local",
"prompt": prompt,
"model_id": model_id,
"width": width,
"height": height,
"num_frames": num_frames,
"fps": fps,
"duration_seconds": round(num_frames / fps, 2),
"operation": operation,
"output": str(output_path),
"format": "mp4",
"license": meta["license"],
**probe_output(output_path),
},
artifacts=[str(output_path)],
seed=seed,
model=model_id,
)
def poll_heygen(execution_id: str, api_key: str, timeout: int = 600) -> str:
import requests
headers = {"X-Api-Key": api_key}
url = f"https://api.heygen.com/v1/workflows/executions/{execution_id}"
deadline = time.time() + timeout
interval = 5.0
while time.time() < deadline:
response = requests.get(url, headers=headers, timeout=30)
response.raise_for_status()
data = response.json().get("data", {})
status = data.get("status", "")
if status == "completed":
video_url = (
data.get("output", {}).get("video", {}).get("video_url")
or data.get("output", {}).get("video_url")
)
if video_url:
return video_url
raise RuntimeError(f"Completed but no video_url in output: {data}")
if status in {"failed", "error"}:
raise RuntimeError(f"HeyGen generation failed: {data.get('error', 'Unknown')}")
time.sleep(min(interval, max(0.0, deadline - time.time())))
interval = min(interval * 1.2, 30.0)
raise TimeoutError(f"HeyGen execution {execution_id} timed out after {timeout}s")
def upload_image_heygen(image_path: str, api_key: str) -> str:
import requests
path = Path(image_path)
if not path.exists():
raise FileNotFoundError(f"Image not found: {image_path}")
with path.open("rb") as handle:
response = requests.post(
"https://api.heygen.com/v1/asset",
headers={"X-Api-Key": api_key},
files={"file": (path.name, handle, "image/png")},
timeout=60,
)
response.raise_for_status()
return response.json().get("data", {}).get("url", "")
def generate_heygen_video(inputs: dict[str, Any]) -> ToolResult:
import requests
api_key = os.environ.get("HEYGEN_API_KEY")
if not api_key:
return ToolResult(success=False, error="HEYGEN_API_KEY not set.")
provider = inputs.get("provider_variant", "veo_3_1")
if provider not in HEYGEN_PROVIDERS:
return ToolResult(
success=False,
error=f"Unknown provider_variant: {provider}. Available: {', '.join(sorted(HEYGEN_PROVIDERS))}",
)
prompt = inputs["prompt"]
aspect_ratio = inputs.get("aspect_ratio", "16:9")
operation = inputs.get("operation", "text_to_video")
workflow_input: dict[str, Any] = {
"prompt": prompt,
"provider": provider,
"aspect_ratio": aspect_ratio,
}
if operation == "image_to_video":
ref_url = inputs.get("reference_image_url")
ref_path = inputs.get("reference_image_path")
if ref_path and not ref_url:
ref_url = upload_image_heygen(ref_path, api_key)
if not ref_url:
return ToolResult(
success=False,
error="image_to_video requires reference_image_url or reference_image_path",
)
workflow_input["reference_image_url"] = ref_url
response = requests.post(
"https://api.heygen.com/v1/workflows/executions",
headers={"X-Api-Key": api_key, "Content-Type": "application/json"},
json={"workflow_type": "GenerateVideoNode", "input": workflow_input},
timeout=30,
)
response.raise_for_status()
payload = response.json()
execution_id = payload.get("data", {}).get("execution_id")
if not execution_id:
return ToolResult(success=False, error=f"No execution_id in response: {payload}")
video_url = poll_heygen(execution_id, api_key, timeout=600)
output_path = Path(inputs.get("output_path", f"heygen_video_{execution_id}.mp4"))
output_path.parent.mkdir(parents=True, exist_ok=True)
download = requests.get(video_url, timeout=120)
download.raise_for_status()
output_path.write_bytes(download.content)
meta = HEYGEN_PROVIDERS[provider]
return ToolResult(
success=True,
data={
"provider": "heygen",
"provider_variant": provider,
"provider_name": meta["name"],
"mode": "api",
"prompt": prompt,
"aspect_ratio": aspect_ratio,
"operation": operation,
"execution_id": execution_id,
"output": str(output_path),
"format": "mp4",
},
artifacts=[str(output_path)],
model=provider,
)
def generate_ltx_modal_video(inputs: dict[str, Any]) -> ToolResult:
import base64
import requests
endpoint_url = os.environ.get("MODAL_LTX2_ENDPOINT_URL")
if not endpoint_url:
return ToolResult(success=False, error="MODAL_LTX2_ENDPOINT_URL not set.")
prompt = inputs["prompt"]
operation = inputs.get("operation", "text_to_video")
aspect = inputs.get("aspect_ratio", "16:9")
width = inputs.get("width")
height = inputs.get("height")
if width is None or height is None:
if aspect == "16:9":
width, height = 1024, 576
elif aspect == "9:16":
width, height = 576, 1024
else:
width, height = 512, 512
num_frames = inputs.get("num_frames", LTX2_FRAME_COUNTS.get(inputs.get("duration_hint", "5s"), 121))
if (num_frames - 1) % 8 != 0:
num_frames = ((num_frames - 1) // 8) * 8 + 1
payload: dict[str, Any] = {
"prompt": prompt,
"width": width,
"height": height,
"num_frames": num_frames,
"fps": 24,
"steps": inputs.get("num_inference_steps", 30),
"negative_prompt": "worst quality, low quality, blurry, distorted, watermark, text, logo",
}
if inputs.get("seed") is not None:
payload["seed"] = inputs["seed"]
if operation == "image_to_video":
ref_path = inputs.get("reference_image_path")
ref_url = inputs.get("reference_image_url")
if ref_path:
payload["input_image"] = base64.b64encode(Path(ref_path).read_bytes()).decode()
elif ref_url:
payload["input_image_url"] = ref_url
else:
return ToolResult(
success=False,
error="image_to_video requires reference_image_url or reference_image_path",
)
response = requests.post(endpoint_url, json=payload, timeout=300)
response.raise_for_status()
output_path = Path(inputs.get("output_path", "ltx_video_modal.mp4"))
output_path.parent.mkdir(parents=True, exist_ok=True)
content_type = response.headers.get("content-type", "")
if "video" in content_type or "octet-stream" in content_type:
output_path.write_bytes(response.content)
else:
response_payload = response.json()
video_url = response_payload.get("video_url") or response_payload.get("url")
if not video_url:
return ToolResult(success=False, error=f"No video data in response: {response_payload}")
download = requests.get(video_url, timeout=120)
download.raise_for_status()
output_path.write_bytes(download.content)
return ToolResult(
success=True,
data={
"provider": "ltx-modal",
"provider_name": "LTX-2 (Modal)",
"mode": "modal",
"prompt": prompt,
"width": width,
"height": height,
"num_frames": num_frames,
"fps": 24,
"duration_seconds": round(num_frames / 24, 2),
"operation": operation,
"output": str(output_path),
"format": "mp4",
},
artifacts=[str(output_path)],
seed=inputs.get("seed"),
model="ltx-2",
)
def probe_output(path: Path) -> dict[str, Any]:
info: dict[str, Any] = {"file_size_bytes": path.stat().st_size}
if not shutil.which("ffprobe"):
return info
import json
try:
proc = subprocess.run(
[
"ffprobe",
"-v",
"quiet",
"-print_format",
"json",
"-show_format",
"-show_streams",
str(path),
],
capture_output=True,
text=True,
timeout=10,
check=False,
)
if proc.returncode == 0:
probe = json.loads(proc.stdout)
fmt = probe.get("format", {})
info["duration_seconds"] = float(fmt.get("duration", 0))
info["file_size_mb"] = round(path.stat().st_size / (1024 * 1024), 2)
for stream in probe.get("streams", []):
if stream.get("codec_type") == "video":
info["video_width"] = int(stream.get("width", 0))
info["video_height"] = int(stream.get("height", 0))
info["video_codec"] = stream.get("codec_name", "")
break
except Exception:
pass
return info
tools/video/cogvideo_video.py
+97
View File
@@ -0,0 +1,97 @@
"""CogVideo local video generation."""
from __future__ import annotations
import time
from tools.base_tool import (
BaseTool,
Determinism,
ExecutionMode,
ResourceProfile,
RetryPolicy,
ToolResult,
ToolRuntime,
ToolStability,
ToolStatus,
ToolTier,
)
from tools.video._shared import COGVIDEO_VARIANTS, estimate_local_runtime, generate_local_video, local_generation_status, local_install_instructions
class CogVideoVideo(BaseTool):
name = "cogvideo_video"
version = "0.1.0"
tier = ToolTier.GENERATE
capability = "video_generation"
provider = "cogvideo"
stability = ToolStability.EXPERIMENTAL
execution_mode = ExecutionMode.SYNC
determinism = Determinism.STOCHASTIC
runtime = ToolRuntime.LOCAL_GPU
install_instructions = local_install_instructions()
fallback = "wan_video"
fallback_tools = ["wan_video", "hunyuan_video", "ltx_video_local", "image_selector"]
agent_skills = ["ltx2"]
capabilities = ["text_to_video", "image_to_video", "model_selection"]
supports = {
"reference_image": True,
"offline": True,
"native_audio": False,
"local_gpu": True,
}
best_for = [
"lower-VRAM local video experimentation",
"teams that want an explicit CogVideo family path in the registry",
]
not_good_for = ["best-in-class local quality targets"]
provider_matrix = {key: {"tool": "cogvideo_video", **value, "mode": "local_gpu"} for key, value in COGVIDEO_VARIANTS.items()}
input_schema = {
"type": "object",
"required": ["prompt"],
"properties": {
"prompt": {"type": "string"},
"operation": {"type": "string", "enum": ["text_to_video", "image_to_video"], "default": "text_to_video"},
"model_variant": {"type": "string", "enum": sorted(COGVIDEO_VARIANTS), "default": "cogvideo-5b"},
"reference_image_url": {"type": "string"},
"reference_image_path": {"type": "string"},
"width": {"type": "integer"},
"height": {"type": "integer"},
"num_frames": {"type": "integer"},
"num_inference_steps": {"type": "integer"},
"enable_model_offload": {"type": "boolean", "default": True},
"seed": {"type": "integer"},
"output_path": {"type": "string"},
},
}
resource_profile = ResourceProfile(cpu_cores=2, ram_mb=16000, vram_mb=6000, disk_mb=4000, network_required=False)
retry_policy = RetryPolicy(max_retries=1)
idempotency_key_fields = ["prompt", "model_variant", "operation", "seed"]
side_effects = ["writes video file to output_path", "may download model weights"]
user_visible_verification = ["Watch generated clip for motion coherence and artifacts"]
def get_status(self) -> ToolStatus:
return local_generation_status()
def estimate_cost(self, inputs: dict[str, object]) -> float:
return 0.0
def estimate_runtime(self, inputs: dict[str, object]) -> float:
variant = COGVIDEO_VARIANTS.get(inputs.get("model_variant", "cogvideo-5b"), COGVIDEO_VARIANTS["cogvideo-5b"])
return estimate_local_runtime(variant["speed"])
def execute(self, inputs: dict[str, object]) -> ToolResult:
if self.get_status() != ToolStatus.AVAILABLE:
return ToolResult(success=False, error="CogVideo local generation is unavailable. " + self.install_instructions)
start = time.time()
try:
result = generate_local_video(tool_name=self.name, variants=COGVIDEO_VARIANTS, default_variant="cogvideo-5b", inputs=inputs)
except Exception as exc:
return ToolResult(success=False, error=f"CogVideo generation failed: {exc}")
result.duration_seconds = round(time.time() - start, 2)
return result
tools/video/heygen_video.py
+117
View File
@@ -0,0 +1,117 @@
"""HeyGen-backed cloud video generation."""
from __future__ import annotations
import os
import time
from typing import Any
from tools.base_tool import (
BaseTool,
Determinism,
ExecutionMode,
ResourceProfile,
RetryPolicy,
ToolResult,
ToolRuntime,
ToolStability,
ToolStatus,
ToolTier,
)
from tools.video._shared import HEYGEN_PROVIDERS, estimate_quality_cost, estimate_speed_runtime, generate_heygen_video
class HeyGenVideo(BaseTool):
name = "heygen_video"
version = "0.1.0"
tier = ToolTier.GENERATE
capability = "video_generation"
provider = "heygen"
stability = ToolStability.EXPERIMENTAL
execution_mode = ExecutionMode.SYNC
determinism = Determinism.STOCHASTIC
runtime = ToolRuntime.API
install_instructions = (
"Set the HEYGEN_API_KEY environment variable:\n"
" set HEYGEN_API_KEY=your_key_here\n"
"Get a key at https://app.heygen.com/settings/api"
)
fallback = "wan_video"
fallback_tools = ["wan_video", "hunyuan_video", "ltx_video_local", "cogvideo_video", "ltx_video_modal", "image_selector"]
agent_skills = ["ai-video-gen", "create-video"]
capabilities = ["text_to_video", "image_to_video", "provider_selection"]
supports = {
"reference_image": True,
"offline": False,
"native_audio": False,
"cloud_generation": True,
}
best_for = [
"premium cloud video generation without local GPU setup",
"fast access to VEO, Sora, Kling, Runway, and Seedance providers",
]
not_good_for = [
"offline or privacy-constrained rendering",
"free local-first production",
]
provider_matrix = {
key: {"tool": "heygen_video", **value, "mode": "api"} for key, value in HEYGEN_PROVIDERS.items()
}
input_schema = {
"type": "object",
"required": ["prompt"],
"properties": {
"prompt": {"type": "string"},
"operation": {
"type": "string",
"enum": ["text_to_video", "image_to_video"],
"default": "text_to_video",
},
"provider_variant": {
"type": "string",
"enum": sorted(HEYGEN_PROVIDERS),
"default": "veo_3_1",
},
"reference_image_url": {"type": "string"},
"reference_image_path": {"type": "string"},
"aspect_ratio": {
"type": "string",
"enum": ["16:9", "9:16", "1:1"],
"default": "16:9",
},
"output_path": {"type": "string"},
},
}
resource_profile = ResourceProfile(cpu_cores=1, ram_mb=512, vram_mb=0, disk_mb=500, network_required=True)
retry_policy = RetryPolicy(max_retries=2, backoff_seconds=10.0, retryable_errors=["rate_limit", "timeout", "server_error"])
idempotency_key_fields = ["prompt", "provider_variant", "aspect_ratio"]
side_effects = ["writes video file to output_path", "calls HeyGen API"]
user_visible_verification = ["Watch generated clip for motion quality and prompt adherence"]
def get_status(self) -> ToolStatus:
return ToolStatus.AVAILABLE if os.environ.get("HEYGEN_API_KEY") else ToolStatus.UNAVAILABLE
def estimate_cost(self, inputs: dict[str, Any]) -> float:
meta = HEYGEN_PROVIDERS.get(inputs.get("provider_variant", "veo_3_1"), HEYGEN_PROVIDERS["veo_3_1"])
return estimate_quality_cost(meta["quality"])
def estimate_runtime(self, inputs: dict[str, Any]) -> float:
meta = HEYGEN_PROVIDERS.get(inputs.get("provider_variant", "veo_3_1"), HEYGEN_PROVIDERS["veo_3_1"])
return estimate_speed_runtime(meta["speed"])
def execute(self, inputs: dict[str, Any]) -> ToolResult:
if self.get_status() != ToolStatus.AVAILABLE:
return ToolResult(success=False, error="HeyGen video generation is unavailable. " + self.install_instructions)
start = time.time()
try:
result = generate_heygen_video(inputs)
except Exception as exc:
return ToolResult(success=False, error=f"HeyGen video generation failed: {exc}")
result.duration_seconds = round(time.time() - start, 2)
result.cost_usd = self.estimate_cost(inputs)
return result
tools/video/hunyuan_video.py
+95
View File
@@ -0,0 +1,95 @@
"""Hunyuan local video generation."""
from __future__ import annotations
import time
from tools.base_tool import (
BaseTool,
Determinism,
ExecutionMode,
ResourceProfile,
RetryPolicy,
ToolResult,
ToolRuntime,
ToolStability,
ToolStatus,
ToolTier,
)
from tools.video._shared import HUNYUAN_VARIANTS, estimate_local_runtime, generate_local_video, local_generation_status, local_install_instructions
class HunyuanVideo(BaseTool):
name = "hunyuan_video"
version = "0.1.0"
tier = ToolTier.GENERATE
capability = "video_generation"
provider = "hunyuan"
stability = ToolStability.EXPERIMENTAL
execution_mode = ExecutionMode.SYNC
determinism = Determinism.STOCHASTIC
runtime = ToolRuntime.LOCAL_GPU
install_instructions = local_install_instructions()
fallback = "wan_video"
fallback_tools = ["wan_video", "ltx_video_local", "cogvideo_video", "image_selector"]
agent_skills = ["ltx2"]
capabilities = ["text_to_video", "image_to_video"]
supports = {
"reference_image": True,
"offline": True,
"native_audio": False,
"local_gpu": True,
}
best_for = [
"local generation when Hunyuan motion behavior fits the brief",
"teams that want one known Hunyuan baseline instead of multiple variants",
]
not_good_for = ["CPU-only machines"]
provider_matrix = {key: {"tool": "hunyuan_video", **value, "mode": "local_gpu"} for key, value in HUNYUAN_VARIANTS.items()}
input_schema = {
"type": "object",
"required": ["prompt"],
"properties": {
"prompt": {"type": "string"},
"operation": {"type": "string", "enum": ["text_to_video", "image_to_video"], "default": "text_to_video"},
"model_variant": {"type": "string", "enum": ["hunyuan-1.5"], "default": "hunyuan-1.5"},
"reference_image_url": {"type": "string"},
"reference_image_path": {"type": "string"},
"width": {"type": "integer"},
"height": {"type": "integer"},
"num_frames": {"type": "integer"},
"num_inference_steps": {"type": "integer"},
"enable_model_offload": {"type": "boolean", "default": True},
"seed": {"type": "integer"},
"output_path": {"type": "string"},
},
}
resource_profile = ResourceProfile(cpu_cores=2, ram_mb=16000, vram_mb=14000, disk_mb=4000, network_required=False)
retry_policy = RetryPolicy(max_retries=1)
idempotency_key_fields = ["prompt", "model_variant", "operation", "seed"]
side_effects = ["writes video file to output_path", "may download model weights"]
user_visible_verification = ["Watch generated clip for motion coherence and artifacts"]
def get_status(self) -> ToolStatus:
return local_generation_status()
def estimate_cost(self, inputs: dict[str, Any]) -> float:
return 0.0
def estimate_runtime(self, inputs: dict[str, Any]) -> float:
return estimate_local_runtime(HUNYUAN_VARIANTS["hunyuan-1.5"]["speed"])
def execute(self, inputs: dict[str, Any]) -> ToolResult:
if self.get_status() != ToolStatus.AVAILABLE:
return ToolResult(success=False, error="Hunyuan local video generation is unavailable. " + self.install_instructions)
start = time.time()
try:
result = generate_local_video(tool_name=self.name, variants=HUNYUAN_VARIANTS, default_variant="hunyuan-1.5", inputs=inputs)
except Exception as exc:
return ToolResult(success=False, error=f"Hunyuan video generation failed: {exc}")
result.duration_seconds = round(time.time() - start, 2)
return result
tools/video/kling_video.py
+178
View File
@@ -0,0 +1,178 @@
"""Kling video generation via fal.ai API.
Best for cinematic B-roll with high visual fidelity and fluid motion.
"""
from __future__ import annotations
import os
import time
from pathlib import Path
from typing import Any
from tools.base_tool import (
BaseTool,
Determinism,
ExecutionMode,
ResourceProfile,
RetryPolicy,
ToolResult,
ToolRuntime,
ToolStability,
ToolStatus,
ToolTier,
)
class KlingVideo(BaseTool):
name = "kling_video"
version = "0.1.0"
tier = ToolTier.GENERATE
capability = "video_generation"
provider = "kling"
stability = ToolStability.EXPERIMENTAL
execution_mode = ExecutionMode.SYNC
determinism = Determinism.STOCHASTIC
runtime = ToolRuntime.API
dependencies = []
install_instructions = (
"Set FAL_KEY to your fal.ai API key.\n"
" Get one at https://fal.ai/dashboard/keys"
)
agent_skills = ["ai-video-gen"]
capabilities = ["text_to_video", "image_to_video"]
supports = {
"text_to_video": True,
"image_to_video": True,
"native_audio": True,
"cinematic_quality": True,
}
best_for = [
"cinematic B-roll with highest visual fidelity",
"fluid motion and camera direction",
"professional video clips",
]
not_good_for = ["budget-constrained projects", "offline generation", "quick iteration"]
fallback_tools = ["minimax_video", "veo_video", "wan_video"]
input_schema = {
"type": "object",
"required": ["prompt"],
"properties": {
"prompt": {"type": "string"},
"operation": {
"type": "string",
"enum": ["text_to_video", "image_to_video"],
"default": "text_to_video",
},
"model_variant": {
"type": "string",
"enum": ["v3/standard", "v2.1/master", "v2.1/pro", "v2.1/standard"],
"default": "v3/standard",
},
"duration": {
"type": "string",
"enum": ["5", "10"],
"default": "5",
"description": "Duration in seconds",
},
"aspect_ratio": {
"type": "string",
"enum": ["16:9", "9:16", "1:1"],
"default": "16:9",
},
"image_url": {"type": "string", "description": "Reference image URL for image_to_video"},
"output_path": {"type": "string"},
},
}
resource_profile = ResourceProfile(
cpu_cores=1, ram_mb=512, vram_mb=0, disk_mb=500, network_required=True
)
retry_policy = RetryPolicy(max_retries=2, retryable_errors=["rate_limit", "timeout"])
idempotency_key_fields = ["prompt", "model_variant", "operation", "duration"]
side_effects = ["writes video file to output_path", "calls fal.ai API"]
user_visible_verification = ["Watch generated clip for motion coherence and visual quality"]
def _get_api_key(self) -> str | None:
return os.environ.get("FAL_KEY") or os.environ.get("FAL_AI_API_KEY")
def get_status(self) -> ToolStatus:
if self._get_api_key():
return ToolStatus.AVAILABLE
return ToolStatus.UNAVAILABLE
def estimate_cost(self, inputs: dict[str, Any]) -> float:
variant = inputs.get("model_variant", "v3/standard")
duration = int(inputs.get("duration", "5"))
if "master" in variant:
return 0.30 * (duration / 5)
if "pro" in variant:
return 0.20 * (duration / 5)
return 0.10 * (duration / 5) # standard
def estimate_runtime(self, inputs: dict[str, Any]) -> float:
return 60.0 # ~1 minute typical
def execute(self, inputs: dict[str, Any]) -> ToolResult:
api_key = self._get_api_key()
if not api_key:
return ToolResult(
success=False,
error="FAL_KEY not set. " + self.install_instructions,
)
import requests
start = time.time()
operation = inputs.get("operation", "text_to_video")
variant = inputs.get("model_variant", "v3/standard")
model_path = f"kling-video/{variant}/{operation}"
payload: dict[str, Any] = {"prompt": inputs["prompt"]}
if inputs.get("duration"):
payload["duration"] = inputs["duration"]
if inputs.get("aspect_ratio"):
payload["aspect_ratio"] = inputs["aspect_ratio"]
if operation == "image_to_video" and inputs.get("image_url"):
payload["image_url"] = inputs["image_url"]
try:
response = requests.post(
f"https://fal.run/fal-ai/{model_path}",
headers={
"Authorization": f"Key {api_key}",
"Content-Type": "application/json",
},
json=payload,
timeout=300,
)
response.raise_for_status()
data = response.json()
video_url = data["video"]["url"]
video_response = requests.get(video_url, timeout=120)
video_response.raise_for_status()
output_path = Path(inputs.get("output_path", "kling_output.mp4"))
output_path.parent.mkdir(parents=True, exist_ok=True)
output_path.write_bytes(video_response.content)
except Exception as e:
return ToolResult(success=False, error=f"Kling video generation failed: {e}")
return ToolResult(
success=True,
data={
"provider": "kling",
"model": f"fal-ai/{model_path}",
"prompt": inputs["prompt"],
"output": str(output_path),
},
artifacts=[str(output_path)],
cost_usd=self.estimate_cost(inputs),
duration_seconds=round(time.time() - start, 2),
model=f"fal-ai/{model_path}",
)
tools/video/ltx_video_local.py
+96
View File
@@ -0,0 +1,96 @@
"""LTX local video generation."""
from __future__ import annotations
import time
from tools.base_tool import (
BaseTool,
Determinism,
ExecutionMode,
ResourceProfile,
RetryPolicy,
ToolResult,
ToolRuntime,
ToolStability,
ToolStatus,
ToolTier,
)
from tools.video._shared import LTX_LOCAL_VARIANTS, estimate_local_runtime, generate_local_video, local_generation_status, local_install_instructions
class LTXVideoLocal(BaseTool):
name = "ltx_video_local"
version = "0.1.0"
tier = ToolTier.GENERATE
capability = "video_generation"
provider = "ltx"
stability = ToolStability.EXPERIMENTAL
execution_mode = ExecutionMode.SYNC
determinism = Determinism.STOCHASTIC
runtime = ToolRuntime.LOCAL_GPU
install_instructions = local_install_instructions()
fallback = "wan_video"
fallback_tools = ["wan_video", "hunyuan_video", "cogvideo_video", "ltx_video_modal", "image_selector"]
agent_skills = ["ltx2"]
capabilities = ["text_to_video", "image_to_video"]
supports = {
"reference_image": True,
"offline": True,
"native_audio": False,
"local_gpu": True,
}
best_for = [
"local LTX workflows already tuned around LTX prompting",
"teams that want one dedicated LTX local path in the registry",
]
not_good_for = ["CPU-only machines"]
provider_matrix = {key: {"tool": "ltx_video_local", **value, "mode": "local_gpu"} for key, value in LTX_LOCAL_VARIANTS.items()}
input_schema = {
"type": "object",
"required": ["prompt"],
"properties": {
"prompt": {"type": "string"},
"operation": {"type": "string", "enum": ["text_to_video", "image_to_video"], "default": "text_to_video"},
"model_variant": {"type": "string", "enum": ["ltx2-local"], "default": "ltx2-local"},
"reference_image_url": {"type": "string"},
"reference_image_path": {"type": "string"},
"width": {"type": "integer"},
"height": {"type": "integer"},
"num_frames": {"type": "integer"},
"num_inference_steps": {"type": "integer"},
"enable_model_offload": {"type": "boolean", "default": True},
"seed": {"type": "integer"},
"output_path": {"type": "string"},
},
}
resource_profile = ResourceProfile(cpu_cores=2, ram_mb=16000, vram_mb=12000, disk_mb=4000, network_required=False)
retry_policy = RetryPolicy(max_retries=1)
idempotency_key_fields = ["prompt", "model_variant", "operation", "seed"]
side_effects = ["writes video file to output_path", "may download model weights"]
user_visible_verification = ["Watch generated clip for motion coherence and artifacts"]
def get_status(self) -> ToolStatus:
return local_generation_status()
def estimate_cost(self, inputs: dict[str, object]) -> float:
return 0.0
def estimate_runtime(self, inputs: dict[str, object]) -> float:
return estimate_local_runtime(LTX_LOCAL_VARIANTS["ltx2-local"]["speed"])
def execute(self, inputs: dict[str, object]) -> ToolResult:
if self.get_status() != ToolStatus.AVAILABLE:
return ToolResult(success=False, error="Local LTX video generation is unavailable. " + self.install_instructions)
start = time.time()
try:
result = generate_local_video(tool_name=self.name, variants=LTX_LOCAL_VARIANTS, default_variant="ltx2-local", inputs=inputs)
except Exception as exc:
return ToolResult(success=False, error=f"Local LTX video generation failed: {exc}")
result.duration_seconds = round(time.time() - start, 2)
return result
tools/video/ltx_video_modal.py
+106
View File
@@ -0,0 +1,106 @@
"""Modal-hosted LTX video generation."""
from __future__ import annotations
import os
import time
from tools.base_tool import (
BaseTool,
Determinism,
ExecutionMode,
ResourceProfile,
RetryPolicy,
ToolResult,
ToolRuntime,
ToolStability,
ToolStatus,
ToolTier,
)
from tools.video._shared import generate_ltx_modal_video
class LTXVideoModal(BaseTool):
name = "ltx_video_modal"
version = "0.1.0"
tier = ToolTier.GENERATE
capability = "video_generation"
provider = "ltx-modal"
stability = ToolStability.EXPERIMENTAL
execution_mode = ExecutionMode.SYNC
determinism = Determinism.STOCHASTIC
runtime = ToolRuntime.API
install_instructions = (
"Set the MODAL_LTX2_ENDPOINT_URL environment variable to your deployed LTX endpoint:\n"
" set MODAL_LTX2_ENDPOINT_URL=https://<your-modal-endpoint>"
)
fallback = "ltx_video_local"
fallback_tools = ["ltx_video_local", "wan_video", "hunyuan_video", "cogvideo_video", "image_selector"]
agent_skills = ["ltx2"]
capabilities = ["text_to_video", "image_to_video"]
supports = {
"reference_image": True,
"offline": False,
"native_audio": False,
"self_hosted_cloud": True,
}
best_for = ["self-hosted cloud GPU rendering for LTX without local workstation dependence"]
not_good_for = ["zero-setup local workflows"]
provider_matrix = {
"ltx2-modal": {
"tool": "ltx_video_modal",
"name": "LTX-2 (Modal)",
"mode": "api",
"quality": "high",
"speed": "medium",
}
}
input_schema = {
"type": "object",
"required": ["prompt"],
"properties": {
"prompt": {"type": "string"},
"operation": {"type": "string", "enum": ["text_to_video", "image_to_video"], "default": "text_to_video"},
"reference_image_url": {"type": "string"},
"reference_image_path": {"type": "string"},
"aspect_ratio": {"type": "string", "enum": ["16:9", "9:16", "1:1"], "default": "16:9"},
"duration_hint": {"type": "string"},
"width": {"type": "integer"},
"height": {"type": "integer"},
"num_frames": {"type": "integer"},
"num_inference_steps": {"type": "integer"},
"seed": {"type": "integer"},
"output_path": {"type": "string"},
},
}
resource_profile = ResourceProfile(cpu_cores=1, ram_mb=512, vram_mb=0, disk_mb=500, network_required=True)
retry_policy = RetryPolicy(max_retries=2, backoff_seconds=10.0, retryable_errors=["timeout", "server_error"])
idempotency_key_fields = ["prompt", "aspect_ratio", "num_frames", "seed"]
side_effects = ["writes video file to output_path", "calls modal endpoint"]
user_visible_verification = ["Watch generated clip for motion quality and prompt adherence"]
def get_status(self) -> ToolStatus:
return ToolStatus.AVAILABLE if os.environ.get("MODAL_LTX2_ENDPOINT_URL") else ToolStatus.UNAVAILABLE
def estimate_cost(self, inputs: dict[str, object]) -> float:
return 0.25
def estimate_runtime(self, inputs: dict[str, object]) -> float:
return 180.0
def execute(self, inputs: dict[str, object]) -> ToolResult:
if self.get_status() != ToolStatus.AVAILABLE:
return ToolResult(success=False, error="Modal LTX video generation is unavailable. " + self.install_instructions)
start = time.time()
try:
result = generate_ltx_modal_video(inputs)
except Exception as exc:
return ToolResult(success=False, error=f"Modal LTX video generation failed: {exc}")
result.duration_seconds = round(time.time() - start, 2)
result.cost_usd = self.estimate_cost(inputs)
return result
tools/video/minimax_video.py
+176
View File
@@ -0,0 +1,176 @@
"""MiniMax (Hailuo AI) video generation via fal.ai API.
Rewards prompt craft — follows camera directions well and produces high-texture footage.
"""
from __future__ import annotations
import os
import time
from pathlib import Path
from typing import Any
from tools.base_tool import (
BaseTool,
Determinism,
ExecutionMode,
ResourceProfile,
RetryPolicy,
ToolResult,
ToolRuntime,
ToolStability,
ToolStatus,
ToolTier,
)
class MiniMaxVideo(BaseTool):
name = "minimax_video"
version = "0.1.0"
tier = ToolTier.GENERATE
capability = "video_generation"
provider = "minimax"
stability = ToolStability.EXPERIMENTAL
execution_mode = ExecutionMode.SYNC
determinism = Determinism.STOCHASTIC
runtime = ToolRuntime.API
dependencies = []
install_instructions = (
"Set FAL_KEY to your fal.ai API key.\n"
" Get one at https://fal.ai/dashboard/keys"
)
agent_skills = ["ai-video-gen"]
capabilities = ["text_to_video", "image_to_video"]
supports = {
"text_to_video": True,
"image_to_video": True,
"camera_direction": True,
}
best_for = [
"prompt-following with camera directions (framing, motion, composition)",
"high-texture footage with minimal hallucination",
"cost-effective video generation",
]
not_good_for = ["offline generation", "very long clips"]
fallback_tools = ["kling_video", "veo_video", "wan_video"]
input_schema = {
"type": "object",
"required": ["prompt"],
"properties": {
"prompt": {"type": "string"},
"operation": {
"type": "string",
"enum": ["text_to_video", "image_to_video"],
"default": "text_to_video",
},
"model_variant": {
"type": "string",
"enum": [
"video-01", "hailuo-02/pro", "hailuo-02/standard",
"hailuo-2.3-fast/pro", "hailuo-2.3-fast/standard",
],
"default": "hailuo-02/pro",
},
"image_url": {"type": "string", "description": "Reference image URL for image_to_video"},
"output_path": {"type": "string"},
},
}
resource_profile = ResourceProfile(
cpu_cores=1, ram_mb=512, vram_mb=0, disk_mb=500, network_required=True
)
retry_policy = RetryPolicy(max_retries=2, retryable_errors=["rate_limit", "timeout"])
idempotency_key_fields = ["prompt", "model_variant", "operation"]
side_effects = ["writes video file to output_path", "calls fal.ai API"]
user_visible_verification = ["Watch generated clip for motion coherence and prompt adherence"]
def _get_api_key(self) -> str | None:
return os.environ.get("FAL_KEY") or os.environ.get("FAL_AI_API_KEY")
def get_status(self) -> ToolStatus:
if self._get_api_key():
return ToolStatus.AVAILABLE
return ToolStatus.UNAVAILABLE
def estimate_cost(self, inputs: dict[str, Any]) -> float:
variant = inputs.get("model_variant", "hailuo-02/pro")
if "pro" in variant:
return 0.15
if "fast" in variant:
return 0.08
return 0.10 # standard
def estimate_runtime(self, inputs: dict[str, Any]) -> float:
variant = inputs.get("model_variant", "hailuo-02/pro")
if "fast" in variant:
return 30.0
return 60.0
def execute(self, inputs: dict[str, Any]) -> ToolResult:
api_key = self._get_api_key()
if not api_key:
return ToolResult(
success=False,
error="FAL_KEY not set. " + self.install_instructions,
)
import requests
start = time.time()
operation = inputs.get("operation", "text_to_video")
variant = inputs.get("model_variant", "hailuo-02/pro")
# Build fal.ai model path
if operation == "text_to_video":
model_path = f"minimax/{variant}/text-to-video"
if variant == "video-01":
model_path = "minimax/video-01"
else:
model_path = f"minimax/{variant}/image-to-video"
if variant == "video-01":
model_path = "minimax/video-01/image-to-video"
payload: dict[str, Any] = {"prompt": inputs["prompt"]}
if operation == "image_to_video" and inputs.get("image_url"):
payload["image_url"] = inputs["image_url"]
try:
response = requests.post(
f"https://fal.run/fal-ai/{model_path}",
headers={
"Authorization": f"Key {api_key}",
"Content-Type": "application/json",
},
json=payload,
timeout=300,
)
response.raise_for_status()
data = response.json()
video_url = data["video"]["url"]
video_response = requests.get(video_url, timeout=120)
video_response.raise_for_status()
output_path = Path(inputs.get("output_path", "minimax_output.mp4"))
output_path.parent.mkdir(parents=True, exist_ok=True)
output_path.write_bytes(video_response.content)
except Exception as e:
return ToolResult(success=False, error=f"MiniMax video generation failed: {e}")
return ToolResult(
success=True,
data={
"provider": "minimax",
"model": f"fal-ai/{model_path}",
"prompt": inputs["prompt"],
"output": str(output_path),
},
artifacts=[str(output_path)],
cost_usd=self.estimate_cost(inputs),
duration_seconds=round(time.time() - start, 2),
model=f"fal-ai/{model_path}",
)
tools/video/pexels_video.py
+213
View File
@@ -0,0 +1,213 @@
"""Stock video acquisition from Pexels API (free)."""
from __future__ import annotations
import os
import time
from pathlib import Path
from typing import Any
from tools.base_tool import (
BaseTool,
Determinism,
ExecutionMode,
ResourceProfile,
RetryPolicy,
ToolResult,
ToolRuntime,
ToolStability,
ToolStatus,
ToolTier,
)
class PexelsVideo(BaseTool):
name = "pexels_video"
version = "0.1.0"
tier = ToolTier.SOURCE
capability = "video_generation"
provider = "pexels"
stability = ToolStability.BETA
execution_mode = ExecutionMode.SYNC
determinism = Determinism.DETERMINISTIC
runtime = ToolRuntime.API
dependencies = []
install_instructions = (
"Set PEXELS_API_KEY to your Pexels API key.\n"
" Get one free at https://www.pexels.com/api/"
)
agent_skills = []
capabilities = ["search_video", "download_video", "stock_video"]
supports = {
"orientation_filter": True,
"size_filter": True,
"free_commercial_use": True,
}
best_for = [
"real-world B-roll footage (cities, nature, people, offices)",
"establishing shots and transitions",
"free stock video — no cost, no attribution required",
]
not_good_for = [
"custom/specific scenes",
"animated or stylized content",
"offline use",
]
fallback_tools = ["pixabay_video"]
input_schema = {
"type": "object",
"required": ["query"],
"properties": {
"query": {"type": "string", "description": "Search term"},
"orientation": {
"type": "string",
"enum": ["landscape", "portrait", "square"],
},
"size": {
"type": "string",
"enum": ["large", "medium", "small"],
"description": "large=4K, medium=Full HD, small=HD",
},
"min_duration": {
"type": "integer",
"description": "Minimum duration in seconds",
},
"max_duration": {
"type": "integer",
"description": "Maximum duration in seconds",
},
"per_page": {"type": "integer", "default": 5, "minimum": 1, "maximum": 80},
"page": {"type": "integer", "default": 1},
"preferred_quality": {
"type": "string",
"enum": ["hd", "sd"],
"default": "hd",
},
"output_path": {"type": "string"},
},
}
resource_profile = ResourceProfile(
cpu_cores=1, ram_mb=256, vram_mb=0, disk_mb=200, network_required=True
)
retry_policy = RetryPolicy(max_retries=2, retryable_errors=["rate_limit", "timeout"])
idempotency_key_fields = ["query", "orientation", "size", "page"]
side_effects = ["writes video file to output_path", "calls Pexels API"]
user_visible_verification = ["Watch downloaded clip to verify it matches the intended scene"]
def get_status(self) -> ToolStatus:
if os.environ.get("PEXELS_API_KEY"):
return ToolStatus.AVAILABLE
return ToolStatus.UNAVAILABLE
def estimate_cost(self, inputs: dict[str, Any]) -> float:
return 0.0
def execute(self, inputs: dict[str, Any]) -> ToolResult:
api_key = os.environ.get("PEXELS_API_KEY")
if not api_key:
return ToolResult(
success=False,
error="PEXELS_API_KEY not set. " + self.install_instructions,
)
import requests
start = time.time()
query = inputs["query"]
params: dict[str, Any] = {
"query": query,
"per_page": inputs.get("per_page", 5),
"page": inputs.get("page", 1),
}
if inputs.get("orientation"):
params["orientation"] = inputs["orientation"]
if inputs.get("size"):
params["size"] = inputs["size"]
try:
search_response = requests.get(
"https://api.pexels.com/videos/search",
headers={"Authorization": api_key},
params=params,
timeout=30,
)
search_response.raise_for_status()
data = search_response.json()
videos = data.get("videos", [])
# Filter by duration if specified
min_dur = inputs.get("min_duration")
max_dur = inputs.get("max_duration")
if min_dur or max_dur:
filtered = []
for v in videos:
dur = v.get("duration", 0)
if min_dur and dur < min_dur:
continue
if max_dur and dur > max_dur:
continue
filtered.append(v)
videos = filtered
if not videos:
return ToolResult(
success=False,
error=f"No videos found for query: {query}",
data={"total_results": data.get("total_results", 0)},
)
video = videos[0]
preferred_quality = inputs.get("preferred_quality", "hd")
# Pick the best matching video file
video_files = video.get("video_files", [])
selected_file = None
for vf in sorted(video_files, key=lambda x: x.get("width", 0), reverse=True):
if vf.get("quality") == preferred_quality:
selected_file = vf
break
if not selected_file and video_files:
selected_file = video_files[0]
if not selected_file:
return ToolResult(success=False, error="No downloadable video file found.")
video_url = selected_file["link"]
video_response = requests.get(video_url, timeout=120)
video_response.raise_for_status()
output_path = Path(inputs.get("output_path", f"pexels_video_{video['id']}.mp4"))
output_path.parent.mkdir(parents=True, exist_ok=True)
output_path.write_bytes(video_response.content)
except Exception as e:
return ToolResult(success=False, error=f"Pexels video search failed: {e}")
return ToolResult(
success=True,
data={
"provider": "pexels",
"video_id": video["id"],
"user": video.get("user", {}).get("name", "Unknown"),
"duration_seconds": video.get("duration"),
"width": selected_file.get("width"),
"height": selected_file.get("height"),
"fps": selected_file.get("fps"),
"quality": selected_file.get("quality"),
"query": query,
"output": str(output_path),
"total_results": data.get("total_results", 0),
"results_returned": len(videos),
"license": "Pexels License (free, no attribution required)",
"pexels_url": video.get("url", ""),
},
artifacts=[str(output_path)],
cost_usd=0.0,
duration_seconds=round(time.time() - start, 2),
)
tools/video/pixabay_video.py
+221
View File
@@ -0,0 +1,221 @@
"""Stock video acquisition from Pixabay API (free)."""
from __future__ import annotations
import os
import time
from pathlib import Path
from typing import Any
from tools.base_tool import (
BaseTool,
Determinism,
ExecutionMode,
ResourceProfile,
RetryPolicy,
ToolResult,
ToolRuntime,
ToolStability,
ToolStatus,
ToolTier,
)
class PixabayVideo(BaseTool):
name = "pixabay_video"
version = "0.1.0"
tier = ToolTier.SOURCE
capability = "video_generation"
provider = "pixabay"
stability = ToolStability.BETA
execution_mode = ExecutionMode.SYNC
determinism = Determinism.DETERMINISTIC
runtime = ToolRuntime.API
dependencies = []
install_instructions = (
"Set PIXABAY_API_KEY to your Pixabay API key.\n"
" Get one free at https://pixabay.com/api/docs/"
)
agent_skills = []
capabilities = ["search_video", "download_video", "stock_video"]
supports = {
"video_type_filter": True,
"category_filter": True,
"editors_choice": True,
"free_commercial_use": True,
}
best_for = [
"large royalty-free video library",
"category-based filtering",
"free stock video — no cost, no attribution required",
]
not_good_for = [
"4K footage (max 1080p on standard API)",
"custom scenes",
"offline use",
]
fallback_tools = ["pexels_video"]
input_schema = {
"type": "object",
"required": ["query"],
"properties": {
"query": {"type": "string", "description": "Search term (max 100 chars)"},
"video_type": {
"type": "string",
"enum": ["all", "film", "animation"],
"default": "all",
},
"category": {
"type": "string",
"enum": [
"backgrounds", "fashion", "nature", "science", "education",
"feelings", "health", "people", "religion", "places",
"animals", "industry", "computer", "food", "sports",
"transportation", "travel", "buildings", "business", "music",
],
},
"min_duration": {
"type": "integer",
"description": "Minimum duration in seconds",
},
"max_duration": {
"type": "integer",
"description": "Maximum duration in seconds",
},
"editors_choice": {"type": "boolean", "default": False},
"safesearch": {"type": "boolean", "default": True},
"per_page": {"type": "integer", "default": 5, "minimum": 3, "maximum": 200},
"page": {"type": "integer", "default": 1},
"preferred_quality": {
"type": "string",
"enum": ["large", "medium", "small", "tiny"],
"default": "large",
},
"output_path": {"type": "string"},
},
}
resource_profile = ResourceProfile(
cpu_cores=1, ram_mb=256, vram_mb=0, disk_mb=200, network_required=True
)
retry_policy = RetryPolicy(max_retries=2, retryable_errors=["rate_limit", "timeout"])
idempotency_key_fields = ["query", "video_type", "category", "page"]
side_effects = ["writes video file to output_path", "calls Pixabay API"]
user_visible_verification = ["Watch downloaded clip to verify it matches the intended scene"]
def get_status(self) -> ToolStatus:
if os.environ.get("PIXABAY_API_KEY"):
return ToolStatus.AVAILABLE
return ToolStatus.UNAVAILABLE
def estimate_cost(self, inputs: dict[str, Any]) -> float:
return 0.0
def execute(self, inputs: dict[str, Any]) -> ToolResult:
api_key = os.environ.get("PIXABAY_API_KEY")
if not api_key:
return ToolResult(
success=False,
error="PIXABAY_API_KEY not set. " + self.install_instructions,
)
import requests
start = time.time()
query = inputs["query"]
params: dict[str, Any] = {
"key": api_key,
"q": query,
"per_page": inputs.get("per_page", 5),
"page": inputs.get("page", 1),
"safesearch": str(inputs.get("safesearch", True)).lower(),
}
if inputs.get("video_type") and inputs["video_type"] != "all":
params["video_type"] = inputs["video_type"]
if inputs.get("category"):
params["category"] = inputs["category"]
if inputs.get("editors_choice"):
params["editors_choice"] = "true"
try:
search_response = requests.get(
"https://pixabay.com/api/videos/",
params=params,
timeout=30,
)
search_response.raise_for_status()
data = search_response.json()
hits = data.get("hits", [])
# Filter by duration if specified
min_dur = inputs.get("min_duration")
max_dur = inputs.get("max_duration")
if min_dur or max_dur:
filtered = []
for h in hits:
dur = h.get("duration", 0)
if min_dur and dur < min_dur:
continue
if max_dur and dur > max_dur:
continue
filtered.append(h)
hits = filtered
if not hits:
return ToolResult(
success=False,
error=f"No videos found for query: {query}",
data={"total_results": data.get("total", 0)},
)
hit = hits[0]
preferred = inputs.get("preferred_quality", "large")
video_info = hit.get("videos", {}).get(preferred)
if not video_info:
# Fallback to best available
for quality in ["large", "medium", "small", "tiny"]:
video_info = hit.get("videos", {}).get(quality)
if video_info:
break
if not video_info:
return ToolResult(success=False, error="No downloadable video file found.")
# Download immediately — Pixabay URLs expire
video_url = video_info["url"]
video_response = requests.get(video_url, timeout=120)
video_response.raise_for_status()
output_path = Path(inputs.get("output_path", f"pixabay_video_{hit['id']}.mp4"))
output_path.parent.mkdir(parents=True, exist_ok=True)
output_path.write_bytes(video_response.content)
except Exception as e:
return ToolResult(success=False, error=f"Pixabay video search failed: {e}")
return ToolResult(
success=True,
data={
"provider": "pixabay",
"video_id": hit["id"],
"user": hit.get("user", "Unknown"),
"tags": hit.get("tags", ""),
"duration_seconds": hit.get("duration"),
"width": video_info.get("width"),
"height": video_info.get("height"),
"query": query,
"output": str(output_path),
"total_results": data.get("total", 0),
"results_returned": len(hits),
"license": "Pixabay Content License (free, no attribution required)",
"page_url": hit.get("pageURL", ""),
},
artifacts=[str(output_path)],
cost_usd=0.0,
duration_seconds=round(time.time() - start, 2),
)
tools/video/runway_video.py
+204
View File
@@ -0,0 +1,204 @@
"""Runway Gen-4 video generation via Runway API.
Highest Elo-rated video generation model — professional quality and control.
"""
from __future__ import annotations
import os
import time
from pathlib import Path
from typing import Any
from tools.base_tool import (
BaseTool,
Determinism,
ExecutionMode,
ResourceProfile,
RetryPolicy,
ToolResult,
ToolRuntime,
ToolStability,
ToolStatus,
ToolTier,
)
class RunwayVideo(BaseTool):
name = "runway_video"
version = "0.1.0"
tier = ToolTier.GENERATE
capability = "video_generation"
provider = "runway"
stability = ToolStability.EXPERIMENTAL
execution_mode = ExecutionMode.SYNC
determinism = Determinism.STOCHASTIC
runtime = ToolRuntime.API
dependencies = []
install_instructions = (
"Set RUNWAY_API_KEY to your Runway API key.\n"
" Get one at https://app.runwayml.com/settings/api-keys"
)
agent_skills = ["ai-video-gen"]
capabilities = ["text_to_video", "image_to_video"]
supports = {
"text_to_video": True,
"image_to_video": True,
"professional_control": True,
}
best_for = [
"highest overall video quality (#1 Elo rating)",
"professional video production",
"precise control over generation",
]
not_good_for = ["budget projects", "offline generation", "very long clips"]
fallback_tools = ["kling_video", "veo_video", "minimax_video", "wan_video"]
input_schema = {
"type": "object",
"required": ["prompt"],
"properties": {
"prompt": {"type": "string"},
"operation": {
"type": "string",
"enum": ["text_to_video", "image_to_video"],
"default": "text_to_video",
},
"model": {
"type": "string",
"enum": ["gen4_turbo", "gen4"],
"default": "gen4_turbo",
},
"duration": {
"type": "integer",
"enum": [5, 10],
"default": 5,
"description": "Duration in seconds",
},
"ratio": {
"type": "string",
"enum": ["16:9", "9:16", "1:1"],
"default": "16:9",
},
"image_url": {"type": "string", "description": "Reference image URL for image_to_video"},
"output_path": {"type": "string"},
},
}
resource_profile = ResourceProfile(
cpu_cores=1, ram_mb=512, vram_mb=0, disk_mb=500, network_required=True
)
retry_policy = RetryPolicy(max_retries=2, retryable_errors=["rate_limit", "timeout"])
idempotency_key_fields = ["prompt", "model", "operation", "duration"]
side_effects = ["writes video file to output_path", "calls Runway API"]
user_visible_verification = ["Watch generated clip for visual quality and motion coherence"]
def get_status(self) -> ToolStatus:
if os.environ.get("RUNWAY_API_KEY"):
return ToolStatus.AVAILABLE
return ToolStatus.UNAVAILABLE
def estimate_cost(self, inputs: dict[str, Any]) -> float:
duration = inputs.get("duration", 5)
# Runway charges per second of generated video
return 0.05 * duration # ~$0.25 for 5s, ~$0.50 for 10s
def estimate_runtime(self, inputs: dict[str, Any]) -> float:
model = inputs.get("model", "gen4_turbo")
if "turbo" in model:
return 30.0
return 60.0
def execute(self, inputs: dict[str, Any]) -> ToolResult:
api_key = os.environ.get("RUNWAY_API_KEY")
if not api_key:
return ToolResult(
success=False,
error="RUNWAY_API_KEY not set. " + self.install_instructions,
)
import requests
start = time.time()
model = inputs.get("model", "gen4_turbo")
operation = inputs.get("operation", "text_to_video")
# Runway API v1 — submit task
task_payload: dict[str, Any] = {
"model": model,
"promptText": inputs["prompt"],
"duration": inputs.get("duration", 5),
"ratio": inputs.get("ratio", "16:9"),
}
if operation == "image_to_video" and inputs.get("image_url"):
task_payload["promptImage"] = inputs["image_url"]
headers = {
"Authorization": f"Bearer {api_key}",
"Content-Type": "application/json",
"X-Runway-Version": "2024-11-06",
}
try:
# Submit generation task
submit_response = requests.post(
"https://api.dev.runwayml.com/v1/image_to_video" if operation == "image_to_video"
else "https://api.dev.runwayml.com/v1/text_to_video",
headers=headers,
json=task_payload,
timeout=30,
)
submit_response.raise_for_status()
task_id = submit_response.json()["id"]
# Poll for completion
video_url = None
for _ in range(60): # max 5 minutes
time.sleep(5)
poll_response = requests.get(
f"https://api.dev.runwayml.com/v1/tasks/{task_id}",
headers=headers,
timeout=15,
)
poll_response.raise_for_status()
task_data = poll_response.json()
if task_data["status"] == "SUCCEEDED":
video_url = task_data["output"][0]
break
if task_data["status"] == "FAILED":
return ToolResult(
success=False,
error=f"Runway generation failed: {task_data.get('failure', 'unknown error')}",
)
if not video_url:
return ToolResult(success=False, error="Runway generation timed out.")
# Download video
video_response = requests.get(video_url, timeout=120)
video_response.raise_for_status()
output_path = Path(inputs.get("output_path", "runway_output.mp4"))
output_path.parent.mkdir(parents=True, exist_ok=True)
output_path.write_bytes(video_response.content)
except Exception as e:
return ToolResult(success=False, error=f"Runway video generation failed: {e}")
return ToolResult(
success=True,
data={
"provider": "runway",
"model": model,
"prompt": inputs["prompt"],
"output": str(output_path),
"task_id": task_id,
},
artifacts=[str(output_path)],
cost_usd=self.estimate_cost(inputs),
duration_seconds=round(time.time() - start, 2),
model=model,
)
tools/video/veo_video.py
+198
View File
@@ -0,0 +1,198 @@
"""Google Veo 3 video generation via fal.ai API.
State-of-the-art video generation with native audio/dialogue synthesis.
"""
from __future__ import annotations
import os
import time
from pathlib import Path
from typing import Any
from tools.base_tool import (
BaseTool,
Determinism,
ExecutionMode,
ResourceProfile,
RetryPolicy,
ToolResult,
ToolRuntime,
ToolStability,
ToolStatus,
ToolTier,
)
class VeoVideo(BaseTool):
name = "veo_video"
version = "0.1.0"
tier = ToolTier.GENERATE
capability = "video_generation"
provider = "veo"
stability = ToolStability.EXPERIMENTAL
execution_mode = ExecutionMode.SYNC
determinism = Determinism.STOCHASTIC
runtime = ToolRuntime.API
dependencies = []
install_instructions = (
"Set FAL_KEY to your fal.ai API key.\n"
" Get one at https://fal.ai/dashboard/keys"
)
agent_skills = ["ai-video-gen"]
capabilities = ["text_to_video", "image_to_video"]
supports = {
"text_to_video": True,
"image_to_video": True,
"native_audio": True,
"dialogue_generation": True,
"ambient_sound": True,
}
best_for = [
"videos with synchronized dialogue and audio",
"cutting-edge quality from Google DeepMind",
"ambient sound and music generation built in",
]
not_good_for = ["budget projects", "offline generation", "quick iteration"]
fallback_tools = ["kling_video", "minimax_video", "wan_video"]
input_schema = {
"type": "object",
"required": ["prompt"],
"properties": {
"prompt": {"type": "string"},
"operation": {
"type": "string",
"enum": ["text_to_video", "image_to_video"],
"default": "text_to_video",
},
"model_variant": {
"type": "string",
"enum": ["veo3", "veo3/fast", "veo3.1", "veo3.1/fast"],
"default": "veo3",
},
"duration": {
"type": "string",
"enum": ["5", "8"],
"default": "8",
"description": "Duration in seconds",
},
"aspect_ratio": {
"type": "string",
"enum": ["16:9", "9:16"],
"default": "16:9",
},
"generate_audio": {
"type": "boolean",
"default": True,
"description": "Whether to generate synchronized audio",
},
"image_url": {"type": "string", "description": "Reference image URL for image_to_video"},
"output_path": {"type": "string"},
},
}
resource_profile = ResourceProfile(
cpu_cores=1, ram_mb=512, vram_mb=0, disk_mb=500, network_required=True
)
retry_policy = RetryPolicy(max_retries=2, retryable_errors=["rate_limit", "timeout"])
idempotency_key_fields = ["prompt", "model_variant", "operation", "duration"]
side_effects = ["writes video file to output_path", "calls fal.ai API"]
user_visible_verification = [
"Watch generated clip for visual quality and motion",
"Listen for audio synchronization and quality",
]
def _get_api_key(self) -> str | None:
return os.environ.get("FAL_KEY") or os.environ.get("FAL_AI_API_KEY")
def get_status(self) -> ToolStatus:
if self._get_api_key():
return ToolStatus.AVAILABLE
return ToolStatus.UNAVAILABLE
def estimate_cost(self, inputs: dict[str, Any]) -> float:
variant = inputs.get("model_variant", "veo3")
duration = int(inputs.get("duration", "8"))
if "fast" in variant:
per_second = 0.12
else:
per_second = 0.30
return per_second * duration
def estimate_runtime(self, inputs: dict[str, Any]) -> float:
variant = inputs.get("model_variant", "veo3")
if "fast" in variant:
return 45.0
return 120.0
def execute(self, inputs: dict[str, Any]) -> ToolResult:
api_key = self._get_api_key()
if not api_key:
return ToolResult(
success=False,
error="FAL_KEY not set. " + self.install_instructions,
)
import requests
start = time.time()
operation = inputs.get("operation", "text_to_video")
variant = inputs.get("model_variant", "veo3")
# Build fal.ai model path
if operation == "image_to_video":
model_path = f"{variant}/image-to-video"
else:
model_path = variant # text-to-video is the default endpoint
payload: dict[str, Any] = {"prompt": inputs["prompt"]}
if inputs.get("duration"):
payload["duration"] = inputs["duration"]
if inputs.get("aspect_ratio"):
payload["aspect_ratio"] = inputs["aspect_ratio"]
if inputs.get("generate_audio") is not None:
payload["generate_audio"] = inputs["generate_audio"]
if operation == "image_to_video" and inputs.get("image_url"):
payload["image_url"] = inputs["image_url"]
try:
response = requests.post(
f"https://fal.run/fal-ai/{model_path}",
headers={
"Authorization": f"Key {api_key}",
"Content-Type": "application/json",
},
json=payload,
timeout=300,
)
response.raise_for_status()
data = response.json()
video_url = data["video"]["url"]
video_response = requests.get(video_url, timeout=120)
video_response.raise_for_status()
output_path = Path(inputs.get("output_path", "veo_output.mp4"))
output_path.parent.mkdir(parents=True, exist_ok=True)
output_path.write_bytes(video_response.content)
except Exception as e:
return ToolResult(success=False, error=f"Veo video generation failed: {e}")
return ToolResult(
success=True,
data={
"provider": "veo",
"model": f"fal-ai/{model_path}",
"prompt": inputs["prompt"],
"output": str(output_path),
"has_audio": inputs.get("generate_audio", True),
},
artifacts=[str(output_path)],
cost_usd=self.estimate_cost(inputs),
duration_seconds=round(time.time() - start, 2),
model=f"fal-ai/{model_path}",
)
tools/video/video_compose.py
+787
View File
@@ -0,0 +1,787 @@
"""Video composition tool — FFmpeg + Remotion.
Final composition path that takes edit decisions, assets, and audio
and renders the complete output video. Supports subtitle burn-in,
overlay compositing, and platform-specific encoding profiles.
For compositions with still images, animated scenes, or component types
(text cards, stat cards, etc.), the render operation auto-routes to
Remotion for frame-accurate spring animations and React-based rendering.
For pure video cuts (talking-head, etc.), FFmpeg handles trimming and concat.
"""
from __future__ import annotations
import json
import time
from pathlib import Path
from typing import Any, Optional
from tools.base_tool import (
BaseTool,
Determinism,
ExecutionMode,
ResourceProfile,
RetryPolicy,
ResumeSupport,
ToolResult,
ToolStability,
ToolTier,
)
class VideoCompose(BaseTool):
name = "video_compose"
version = "0.1.0"
tier = ToolTier.CORE
capability = "video_post"
provider = "ffmpeg"
stability = ToolStability.EXPERIMENTAL
execution_mode = ExecutionMode.SYNC
determinism = Determinism.DETERMINISTIC
dependencies = ["cmd:ffmpeg"]
install_instructions = "Install FFmpeg: https://ffmpeg.org/download.html"
agent_skills = ["remotion-best-practices", "remotion", "ffmpeg"]
capabilities = [
"compose_cuts",
"burn_subtitles",
"overlay_assets",
"encode_profile",
"remotion_render",
]
input_schema = {
"type": "object",
"required": ["operation"],
"properties": {
"operation": {
"type": "string",
"enum": ["compose", "render", "remotion_render", "burn_subtitles", "overlay", "encode"],
"description": (
"compose: low-level concat cuts + audio + subtitles. "
"render: high-level — resolves asset IDs, auto-routes to Remotion "
"for images/animations or FFmpeg for video-only. Preferred for compose-director. "
"remotion_render: render via Remotion (Node.js). "
"burn_subtitles: burn subtitle file into existing video. "
"overlay: composite overlays onto base video. "
"encode: re-encode to a target profile/codec."
),
},
"input_path": {"type": "string"},
"output_path": {"type": "string"},
"edit_decisions": {
"type": "object",
"description": "Full edit_decisions artifact (required for compose/render)",
},
"asset_manifest": {
"type": "object",
"description": (
"Full asset_manifest artifact (required for render). "
"Used to resolve asset IDs in cuts[].source to file paths."
),
},
"subtitle_path": {"type": "string"},
"subtitle_style": {
"type": "object",
"description": "ASS subtitle styling. Also extracted from edit_decisions.subtitles if not provided.",
"properties": {
"font": {"type": "string", "default": "Arial"},
"font_size": {"type": "integer", "default": 24},
"primary_color": {"type": "string", "default": "&HFFFFFF"},
"outline_color": {"type": "string", "default": "&H000000"},
"outline_width": {"type": "number", "default": 2},
"margin_v": {"type": "integer", "default": 40},
"alignment": {"type": "integer", "default": 2},
},
},
"overlays": {
"type": "array",
"items": {
"type": "object",
"properties": {
"asset_path": {"type": "string"},
"x": {"type": "number"},
"y": {"type": "number"},
"width": {"type": "number"},
"height": {"type": "number"},
"start_seconds": {"type": "number"},
"end_seconds": {"type": "number"},
"opacity": {"type": "number", "minimum": 0, "maximum": 1},
},
},
},
"audio_path": {"type": "string", "description": "Mixed audio to mux into output"},
"profile": {
"type": "string",
"description": (
"Media profile name from media_profiles.py "
"(e.g. youtube_landscape, tiktok, instagram_reels). "
"Applied in render and encode operations."
),
},
"options": {
"type": "object",
"description": "Render options (used by the render operation)",
"properties": {
"subtitle_burn": {"type": "boolean", "default": True},
"two_pass_encode": {"type": "boolean", "default": False},
},
},
"codec": {"type": "string", "default": "libx264"},
"crf": {"type": "integer", "default": 23},
"preset": {"type": "string", "default": "medium"},
},
}
resource_profile = ResourceProfile(
cpu_cores=4, ram_mb=2048, vram_mb=0, disk_mb=5000, network_required=False
)
# Remotion scene types that trigger React-based rendering
_REMOTION_COMPONENTS = [
"text_card", "stat_card", "callout", "comparison",
"progress", "chart", "bar_chart", "line_chart", "pie_chart", "kpi_grid",
]
best_for = [
"Final render for explainer and animation pipelines",
"Image-to-video with spring animations (Remotion)",
"Animated text cards, stat cards, charts (Remotion)",
"Complex transitions between scenes (Remotion)",
"Pure video concat and trim (FFmpeg)",
]
retry_policy = RetryPolicy(max_retries=1, retryable_errors=["Conversion failed"])
resume_support = ResumeSupport.FROM_START
idempotency_key_fields = ["operation", "input_path", "edit_decisions"]
side_effects = ["writes video file to output_path"]
user_visible_verification = [
"Play the composed output and verify cuts, subtitles, and overlays",
]
def _remotion_available(self) -> bool:
"""Check if Remotion rendering is available (requires npx + composer project)."""
import shutil as _shutil
if not _shutil.which("npx"):
return False
composer_dir = Path(__file__).resolve().parent.parent.parent / "remotion-composer"
return composer_dir.exists() and (composer_dir / "package.json").exists()
def get_info(self) -> dict[str, Any]:
"""Extend base get_info to surface Remotion sub-capability.
This lets preflight report 'video_compose: AVAILABLE (FFmpeg + Remotion)'
so the agent knows Remotion is usable for motion graphics, animated text
cards, stat cards, charts, and image-to-video rendering — rather than
falling back to Ken Burns pan-and-zoom over still images.
"""
info = super().get_info()
remotion_ok = self._remotion_available()
info["render_engines"] = {
"ffmpeg": True,
"remotion": remotion_ok,
}
if remotion_ok:
info["remotion_components"] = self._REMOTION_COMPONENTS
info["remotion_note"] = (
"Remotion is available for React-based rendering. Use it for "
"image-to-video with spring animations, animated text/stat cards, "
"charts, callouts, comparisons, and complex transitions. "
"Prefer Remotion over Ken Burns pan-and-zoom for explainer "
"and motion-graphics pipelines."
)
else:
info["remotion_note"] = (
"Remotion is NOT available (needs Node.js/npx + remotion-composer). "
"Falling back to FFmpeg Ken Burns for image-based compositions."
)
return info
def execute(self, inputs: dict[str, Any]) -> ToolResult:
operation = inputs["operation"]
start = time.time()
try:
if operation == "compose":
result = self._compose(inputs)
elif operation == "render":
result = self._render(inputs)
elif operation == "remotion_render":
result = self._remotion_render(inputs)
elif operation == "burn_subtitles":
result = self._burn_subtitles(inputs)
elif operation == "overlay":
result = self._overlay(inputs)
elif operation == "encode":
result = self._encode(inputs)
else:
return ToolResult(success=False, error=f"Unknown operation: {operation}")
except Exception as e:
return ToolResult(success=False, error=str(e))
result.duration_seconds = round(time.time() - start, 2)
return result
_IMAGE_EXTENSIONS = {".png", ".jpg", ".jpeg", ".bmp", ".tiff", ".tif", ".webp"}
@staticmethod
def _is_image(path: Path) -> bool:
"""Check if a file is a still image (routes to Remotion, not FFmpeg)."""
return path.suffix.lower() in VideoCompose._IMAGE_EXTENSIONS
def _compose(self, inputs: dict[str, Any]) -> ToolResult:
"""FFmpeg composition: concat video cuts, add audio, burn subtitles.
Handles video sources only. Still images and animated scene types
are routed to Remotion via the render operation — call compose
directly only for pure video pipelines (e.g. talking-head).
"""
edit_decisions = inputs.get("edit_decisions")
if not edit_decisions:
return ToolResult(success=False, error="edit_decisions required for compose")
output_path = Path(inputs.get("output_path", "composed_output.mp4"))
output_path.parent.mkdir(parents=True, exist_ok=True)
audio_path = inputs.get("audio_path")
subtitle_path = inputs.get("subtitle_path")
codec = inputs.get("codec", "libx264")
crf = inputs.get("crf", 23)
preset = inputs.get("preset", "medium")
profile_name = inputs.get("profile")
# Resolve target resolution from profile or default
resolution = "1920x1080"
if profile_name:
try:
from lib.media_profiles import get_profile
p = get_profile(profile_name)
resolution = f"{p.width}x{p.height}"
except (ImportError, ValueError):
pass
cuts = edit_decisions.get("cuts", [])
if not cuts:
return ToolResult(success=False, error="No cuts in edit_decisions")
# Extract subtitle style from edit_decisions if not provided directly
if not inputs.get("subtitle_style"):
ed_subs = edit_decisions.get("subtitles", {})
if ed_subs:
inputs = dict(inputs)
inputs["subtitle_style"] = {
k: v for k, v in ed_subs.items()
if k in ("font", "font_size", "color", "outline_color", "background")
}
if ed_subs.get("source") and not subtitle_path:
subtitle_path = ed_subs["source"]
temp_dir = output_path.parent / ".compose_tmp"
temp_dir.mkdir(parents=True, exist_ok=True)
temp_segments: list[Path] = []
try:
for i, cut in enumerate(cuts):
source = Path(cut["source"])
if not source.exists():
return ToolResult(success=False, error=f"Cut source not found: {source}")
seg_path = temp_dir / f"seg_{i:04d}.mp4"
in_s = cut["in_seconds"]
out_s = cut["out_seconds"]
duration = out_s - in_s
speed = cut.get("speed", 1.0)
if self._is_image(source):
return ToolResult(
success=False,
error=(
f"Still image '{source.name}' in cuts. "
"Use operation='render' (auto-routes to Remotion) "
"or operation='remotion_render' for compositions "
"with images, animations, or component scenes."
),
)
else:
# Video source: trim to segment
cmd = [
"ffmpeg", "-y",
"-i", str(source),
"-ss", str(in_s),
"-to", str(out_s),
]
if speed != 1.0:
vf = f"setpts={1.0/speed}*PTS"
af = self._build_atempo(speed)
cmd.extend(["-filter:v", vf, "-filter:a", af])
cmd.extend(["-c:v", codec, "-crf", str(crf), "-c:a", "aac"])
else:
cmd.extend(["-c", "copy"])
cmd.append(str(seg_path))
self.run_command(cmd)
temp_segments.append(seg_path)
# Step 2: Concat segments
concat_path = temp_dir / "concat_list.txt"
with open(concat_path, "w", encoding="utf-8") as f:
for seg in temp_segments:
safe = str(seg.resolve()).replace("\\", "/")
f.write(f"file '{safe}'\n")
concat_out = temp_dir / "concat.mp4"
cmd = [
"ffmpeg", "-y",
"-f", "concat", "-safe", "0",
"-i", str(concat_path),
"-c", "copy",
str(concat_out),
]
self.run_command(cmd)
# Step 3: Apply subtitles and/or replace audio
final_input = concat_out
vfilters = []
if subtitle_path and Path(subtitle_path).exists():
style = inputs.get("subtitle_style", {})
ass_style = self._build_subtitle_style(style)
sub_escaped = str(Path(subtitle_path).resolve()).replace("\\", "/").replace(":", "\\:")
vfilters.append(f"subtitles='{sub_escaped}':force_style='{ass_style}'")
cmd = ["ffmpeg", "-y", "-i", str(final_input)]
if audio_path and Path(audio_path).exists():
cmd.extend(["-i", audio_path])
if vfilters:
cmd.extend(["-vf", ",".join(vfilters)])
cmd.extend(["-c:v", codec, "-crf", str(crf), "-preset", preset])
else:
cmd.extend(["-c:v", "copy"])
if audio_path and Path(audio_path).exists():
cmd.extend(["-map", "0:v:0", "-map", "1:a:0", "-c:a", "aac", "-shortest"])
else:
cmd.extend(["-c:a", "copy"])
# Apply profile resolution/fps at final output
if profile_name:
try:
from lib.media_profiles import get_profile
p = get_profile(profile_name)
cmd.extend(["-s", f"{p.width}x{p.height}", "-r", str(p.fps)])
except (ImportError, ValueError):
pass
cmd.append(str(output_path))
self.run_command(cmd)
return ToolResult(
success=True,
data={
"operation": "compose",
"cut_count": len(cuts),
"has_subtitles": subtitle_path is not None,
"has_mixed_audio": audio_path is not None,
"profile": profile_name,
"output": str(output_path),
},
artifacts=[str(output_path)],
)
finally:
# Cleanup temp files
for f in temp_segments:
if f.exists():
f.unlink()
for f in [concat_path, concat_out]:
if f.exists():
f.unlink()
if temp_dir.exists():
try:
temp_dir.rmdir()
except OSError:
pass
_REMOTION_SCENE_TYPES = {
"text_card", "stat_card", "callout", "comparison", "progress", "chart",
}
def _needs_remotion(self, cuts: list[dict]) -> bool:
"""Determine if the composition requires Remotion.
Returns True when any cut contains still images, animated scene types,
component types (text_card, stat_card, etc.), or transitions — all of
which benefit from Remotion's React-based rendering over FFmpeg.
"""
for cut in cuts:
source = cut.get("source", "")
if source and Path(source).suffix.lower() in self._IMAGE_EXTENSIONS:
return True
if cut.get("type") in self._REMOTION_SCENE_TYPES:
return True
if cut.get("animation") or cut.get("transition_in") or cut.get("transition_out"):
return True
transform = cut.get("transform", {})
if transform and transform.get("animation"):
return True
return False
def _render(self, inputs: dict[str, Any]) -> ToolResult:
"""High-level render: assemble edit decisions + asset manifest into final video.
This is the primary entry point for the compose-director skill.
It resolves asset IDs, then auto-routes to Remotion (for images,
animations, component scenes) or FFmpeg (for pure video cuts).
The agent should pass edit_decisions, asset_manifest, and optionally
profile, subtitle_path, audio_path, and options.
"""
edit_decisions = inputs.get("edit_decisions")
asset_manifest = inputs.get("asset_manifest")
if not edit_decisions:
return ToolResult(success=False, error="edit_decisions required for render")
if not asset_manifest:
return ToolResult(success=False, error="asset_manifest required for render")
output_path = Path(inputs.get("output_path", "renders/output.mp4"))
output_path.parent.mkdir(parents=True, exist_ok=True)
# Build asset lookup: id -> asset info
asset_lookup = {a["id"]: a for a in asset_manifest.get("assets", [])}
cuts = edit_decisions.get("cuts", [])
if not cuts:
return ToolResult(success=False, error="No cuts in edit_decisions")
# Resolve asset IDs in cuts to file paths
resolved_cuts = []
for cut in cuts:
source_id = cut.get("source", "")
resolved_cut = dict(cut)
if source_id in asset_lookup:
resolved_cut["source"] = asset_lookup[source_id]["path"]
resolved_cuts.append(resolved_cut)
# Also accept profile as "output_profile" (skill convention) or "profile"
profile = inputs.get("profile") or inputs.get("output_profile")
# --- Route: Remotion for rich content, FFmpeg for pure video ---
if self._needs_remotion(resolved_cuts):
remotion_inputs: dict[str, Any] = {
"edit_decisions": dict(edit_decisions, cuts=resolved_cuts),
"output_path": str(output_path),
}
if profile:
remotion_inputs["profile"] = profile
return self._remotion_render(remotion_inputs)
# --- FFmpeg path: pure video cuts (talking-head, etc.) ---
# Handle options
options = inputs.get("options", {})
subtitle_burn = options.get("subtitle_burn", True)
# Resolve subtitle_path from edit_decisions if not provided
subtitle_path = inputs.get("subtitle_path")
if subtitle_burn and not subtitle_path:
ed_subs = edit_decisions.get("subtitles", {})
if ed_subs.get("enabled") and ed_subs.get("source"):
subtitle_path = ed_subs["source"]
# Build compose inputs
compose_inputs = dict(inputs)
compose_inputs["edit_decisions"] = dict(edit_decisions, cuts=resolved_cuts)
compose_inputs["output_path"] = str(output_path)
if subtitle_path:
compose_inputs["subtitle_path"] = subtitle_path
if profile:
compose_inputs["profile"] = profile
return self._compose(compose_inputs)
def _remotion_render(self, inputs: dict[str, Any]) -> ToolResult:
"""Render via Remotion (requires Node.js + npx).
Handles compositions with still images, animated scenes, component
types, and transitions using React-based frame-accurate rendering.
Accepts edit_decisions (with resolved file paths) or raw composition_data.
"""
import shutil
if not shutil.which("npx"):
return ToolResult(
success=False,
error="npx not found. Install Node.js to use Remotion rendering.",
)
composition_data = inputs.get("edit_decisions") or inputs.get("composition_data")
if not composition_data:
return ToolResult(
success=False,
error="edit_decisions or composition_data required for remotion_render",
)
output_path = Path(inputs.get("output_path", "renders/remotion_output.mp4"))
output_path.parent.mkdir(parents=True, exist_ok=True)
# Deep-copy props so we don't mutate the original
props = json.loads(json.dumps(composition_data))
# Convert absolute file paths to file:// URIs for Remotion's
# Img and OffthreadVideo components
for cut in props.get("cuts", []):
source = cut.get("source", "")
if source and not source.startswith(("http://", "https://", "file://")):
resolved = Path(source).resolve()
if resolved.exists():
posix = resolved.as_posix()
cut["source"] = f"file:///{posix}" if not posix.startswith("/") else f"file://{posix}"
# Write props to temp file for Remotion CLI
props_path = output_path.parent / ".remotion_props.json"
with open(props_path, "w", encoding="utf-8") as f:
json.dump(props, f)
# remotion-composer lives at project root
composer_dir = Path(__file__).resolve().parent.parent.parent / "remotion-composer"
if not composer_dir.exists():
return ToolResult(
success=False,
error=f"Remotion composer project not found at {composer_dir}",
)
cmd = [
"npx", "remotion", "render",
str(composer_dir / "src" / "index.tsx"),
"Explainer",
str(output_path),
"--props", str(props_path),
]
# Apply media profile dimensions
profile_name = inputs.get("profile")
if profile_name:
try:
from lib.media_profiles import get_profile
p = get_profile(profile_name)
cmd.extend(["--width", str(p.width), "--height", str(p.height)])
except (ImportError, ValueError):
pass
try:
self.run_command(cmd, timeout=600)
except Exception as e:
return ToolResult(success=False, error=f"Remotion render failed: {e}")
finally:
if props_path.exists():
props_path.unlink()
if not output_path.exists():
return ToolResult(
success=False,
error=f"Remotion render completed but output file missing: {output_path}",
)
return ToolResult(
success=True,
data={
"operation": "remotion_render",
"output": str(output_path),
"profile": profile_name,
},
artifacts=[str(output_path)],
)
def _burn_subtitles(self, inputs: dict[str, Any]) -> ToolResult:
"""Burn subtitle file into video."""
input_path = Path(inputs["input_path"])
subtitle_path = Path(inputs["subtitle_path"])
output_path = Path(inputs.get("output_path", str(input_path.with_stem(f"{input_path.stem}_subtitled"))))
if not input_path.exists():
return ToolResult(success=False, error=f"Input not found: {input_path}")
if not subtitle_path.exists():
return ToolResult(success=False, error=f"Subtitle file not found: {subtitle_path}")
style = inputs.get("subtitle_style", {})
ass_style = self._build_subtitle_style(style)
sub_escaped = str(subtitle_path.resolve()).replace("\\", "/").replace(":", "\\:")
codec = inputs.get("codec", "libx264")
crf = inputs.get("crf", 23)
cmd = [
"ffmpeg", "-y",
"-i", str(input_path),
"-vf", f"subtitles='{sub_escaped}':force_style='{ass_style}'",
"-c:v", codec, "-crf", str(crf),
"-c:a", "copy",
str(output_path),
]
self.run_command(cmd)
return ToolResult(
success=True,
data={
"operation": "burn_subtitles",
"output": str(output_path),
},
artifacts=[str(output_path)],
)
def _overlay(self, inputs: dict[str, Any]) -> ToolResult:
"""Composite overlay images/videos on top of base video."""
input_path = Path(inputs["input_path"])
overlays = inputs.get("overlays", [])
output_path = Path(inputs.get("output_path", str(input_path.with_stem(f"{input_path.stem}_overlay"))))
codec = inputs.get("codec", "libx264")
crf = inputs.get("crf", 23)
if not input_path.exists():
return ToolResult(success=False, error=f"Input not found: {input_path}")
if not overlays:
return ToolResult(success=False, error="No overlays provided")
# Build complex filter for each overlay
input_args = ["-i", str(input_path)]
filter_parts = []
prev_label = "0:v"
for i, ov in enumerate(overlays):
asset_path = Path(ov["asset_path"])
if not asset_path.exists():
return ToolResult(success=False, error=f"Overlay asset not found: {asset_path}")
input_args.extend(["-i", str(asset_path)])
x = int(ov.get("x", 0))
y = int(ov.get("y", 0))
start = ov.get("start_seconds", 0)
end = ov.get("end_seconds")
opacity = ov.get("opacity", 1.0)
overlay_input = f"{i + 1}:v"
# Scale overlay if dimensions specified
if "width" in ov and "height" in ov:
w = int(ov["width"])
h = int(ov["height"])
filter_parts.append(f"[{overlay_input}]scale={w}:{h}[ov_scaled_{i}]")
overlay_input = f"ov_scaled_{i}"
# Build enable expression for timed overlays
enable = f"between(t,{start},{end})" if end else f"gte(t,{start})"
out_label = f"v{i}"
filter_parts.append(
f"[{prev_label}][{overlay_input}]overlay={x}:{y}:enable='{enable}'[{out_label}]"
)
prev_label = out_label
filter_complex = ";".join(filter_parts)
cmd = ["ffmpeg", "-y"]
cmd.extend(input_args)
cmd.extend(["-filter_complex", filter_complex])
cmd.extend(["-map", f"[{prev_label}]", "-map", "0:a?"])
cmd.extend(["-c:v", codec, "-crf", str(crf), "-c:a", "copy"])
cmd.append(str(output_path))
self.run_command(cmd)
return ToolResult(
success=True,
data={
"operation": "overlay",
"overlay_count": len(overlays),
"output": str(output_path),
},
artifacts=[str(output_path)],
)
def _encode(self, inputs: dict[str, Any]) -> ToolResult:
"""Re-encode video with a specific profile/codec settings."""
input_path = Path(inputs["input_path"])
output_path = Path(inputs.get("output_path", str(input_path.with_stem(f"{input_path.stem}_encoded"))))
codec = inputs.get("codec", "libx264")
crf = inputs.get("crf", 23)
preset = inputs.get("preset", "medium")
profile_name = inputs.get("profile")
if not input_path.exists():
return ToolResult(success=False, error=f"Input not found: {input_path}")
cmd = [
"ffmpeg", "-y",
"-i", str(input_path),
"-c:v", codec, "-crf", str(crf), "-preset", preset,
"-c:a", "aac", "-b:a", "192k",
]
# Apply media profile if specified
if profile_name:
try:
from lib.media_profiles import get_profile, ffmpeg_output_args
profile = get_profile(profile_name)
cmd.extend(["-s", f"{profile.width}x{profile.height}"])
cmd.extend(["-r", str(profile.fps)])
except (ImportError, ValueError):
pass # proceed without profile
cmd.append(str(output_path))
self.run_command(cmd)
return ToolResult(
success=True,
data={
"operation": "encode",
"codec": codec,
"crf": crf,
"profile": profile_name,
"output": str(output_path),
},
artifacts=[str(output_path)],
)
@staticmethod
def _build_subtitle_style(style: dict) -> str:
"""Build ASS force_style string from style dict.
Produces modern social-media-style captions by default:
bold, outlined, positioned in the lower portion of the frame.
"""
parts = []
parts.append(f"FontName={style.get('font', 'Arial')}")
parts.append(f"FontSize={style.get('font_size', 16)}")
parts.append(f"Bold={1 if style.get('bold', True) else 0}")
if style.get("primary_color"):
parts.append(f"PrimaryColour={style['primary_color']}")
if style.get("outline_color"):
parts.append(f"OutlineColour={style['outline_color']}")
if style.get("back_color"):
parts.append(f"BackColour={style['back_color']}")
# BorderStyle: 1=outline+shadow (default), 4=opaque box
border_style = style.get("border_style", 1)
parts.append(f"BorderStyle={border_style}")
parts.append(f"Outline={style.get('outline_width', 3)}")
parts.append(f"Shadow={style.get('shadow', 1)}")
parts.append(f"MarginV={style.get('margin_v', 40)}")
parts.append(f"Alignment={style.get('alignment', 2)}")
return ",".join(parts)
@staticmethod
def _build_atempo(factor: float) -> str:
"""Build atempo filter chain for audio speed adjustment."""
filters = []
remaining = factor
while remaining > 100.0:
filters.append("atempo=100.0")
remaining /= 100.0
while remaining < 0.5:
filters.append("atempo=0.5")
remaining /= 0.5
filters.append(f"atempo={remaining:.4f}")
return ",".join(filters)
tools/video/video_selector.py
+136
View File
@@ -0,0 +1,136 @@
"""Capability-level video selector that routes between generation and stock providers.
Provider discovery is automatic — any BaseTool with capability="video_generation"
is picked up from the registry. Adding a new video provider requires only creating
the tool file in tools/video/; no changes to this selector are needed.
"""
from __future__ import annotations
import os
from tools.base_tool import BaseTool, ToolResult, ToolRuntime, ToolStability, ToolStatus, ToolTier
class VideoSelector(BaseTool):
name = "video_selector"
version = "0.3.0"
tier = ToolTier.GENERATE
capability = "video_generation"
provider = "selector"
stability = ToolStability.BETA
runtime = ToolRuntime.HYBRID
agent_skills = ["ai-video-gen", "create-video", "ltx2"]
capabilities = [
"text_to_video", "image_to_video", "stock_video",
"provider_selection", "search_video", "download_video",
]
supports = {
"user_preference_routing": True,
"offline_fallback": True,
"reference_image": True,
"stock_fallback": True,
}
best_for = [
"preflight routing",
"user-facing recommendation flows",
"switching between cloud, local, and stock video tools",
]
input_schema = {
"type": "object",
"required": ["prompt"],
"properties": {
"prompt": {"type": "string"},
"preferred_provider": {
"type": "string",
"description": "Provider name or 'auto'. Valid values are discovered at runtime from the registry.",
"default": "auto",
},
"allowed_providers": {"type": "array", "items": {"type": "string"}},
"operation": {"type": "string", "enum": ["text_to_video", "image_to_video"], "default": "text_to_video"},
"output_path": {"type": "string"},
},
}
def _providers(self) -> list[BaseTool]:
"""Auto-discover video generation providers from the registry."""
from tools.tool_registry import registry
registry.ensure_discovered()
return [t for t in registry.get_by_capability("video_generation")
if t.name != self.name]
@property
def fallback_tools(self) -> list[str]:
"""Dynamically built from discovered providers + image_selector as last resort."""
return [t.name for t in self._providers()] + ["image_selector"]
@property
def provider_matrix(self) -> dict[str, dict[str, str]]:
"""Built at runtime from each provider's best_for field."""
matrix = {}
for tool in self._providers():
strength = ", ".join(tool.best_for) if tool.best_for else tool.name
matrix[tool.provider] = {"tool": tool.name, "strength": strength}
return matrix
def get_status(self) -> ToolStatus:
if any(tool.get_status() == ToolStatus.AVAILABLE for tool in self._providers()):
return ToolStatus.AVAILABLE
return ToolStatus.UNAVAILABLE
def estimate_cost(self, inputs: dict[str, object]) -> float:
tool = self._select_tool(inputs)
return tool.estimate_cost(inputs) if tool else 0.0
def estimate_runtime(self, inputs: dict[str, object]) -> float:
tool = self._select_tool(inputs)
return tool.estimate_runtime(inputs) if tool else 0.0
def execute(self, inputs: dict[str, object]) -> ToolResult:
tool = self._select_tool(inputs)
if tool is None:
return ToolResult(success=False, error="No video generation provider available.")
# Adapt input keys: stock tools use 'query' while generators use 'prompt'
adapted = dict(inputs)
if hasattr(tool, 'input_schema'):
required = tool.input_schema.get("properties", {})
if "query" in required and "query" not in adapted:
adapted["query"] = adapted.get("prompt", "")
result = tool.execute(adapted)
if result.success:
result.data.setdefault("selected_tool", tool.name)
return result
def _select_tool(self, inputs: dict[str, object]) -> BaseTool | None:
preferred = inputs.get("preferred_provider", "auto")
allowed = set(inputs.get("allowed_providers") or [])
candidates = self._providers()
if allowed:
candidates = [tool for tool in candidates if tool.provider in allowed]
env_hint = os.environ.get("VIDEO_GEN_LOCAL_MODEL", "").lower()
env_map = {
"wan2.1-1.3b": "wan",
"wan2.1-14b": "wan",
"hunyuan-1.5": "hunyuan",
"ltx2-local": "ltx",
"cogvideo-5b": "cogvideo",
"cogvideo-2b": "cogvideo",
}
if preferred == "auto" and env_hint in env_map:
preferred = env_map[env_hint]
if preferred != "auto":
ordered = [tool for tool in candidates if tool.provider == preferred]
ordered.extend([tool for tool in candidates if tool.provider != preferred])
else:
ordered = candidates
for tool in ordered:
if tool.get_status() == ToolStatus.AVAILABLE:
return tool
return None
tools/video/video_stitch.py
+962
View File
@@ -0,0 +1,962 @@
"""Video stitch tool wrapping FFmpeg.
Multi-clip assembly with validation, transitions, and spatial layouts.
Supports sequential concatenation (TikTok-style stitch), crossfade/fade
transitions, and spatial compositions (side-by-side, vertical stack,
picture-in-picture) for duet-style content.
"""
from __future__ import annotations
import json
import time
from pathlib import Path
from typing import Any, Optional
from tools.base_tool import (
BaseTool,
Determinism,
ExecutionMode,
ResourceProfile,
RetryPolicy,
ResumeSupport,
ToolResult,
ToolStability,
ToolTier,
)
class VideoStitch(BaseTool):
name = "video_stitch"
version = "0.1.0"
tier = ToolTier.CORE
capability = "video_post"
provider = "ffmpeg"
stability = ToolStability.EXPERIMENTAL
execution_mode = ExecutionMode.SYNC
determinism = Determinism.DETERMINISTIC
dependencies = ["cmd:ffmpeg", "cmd:ffprobe"]
install_instructions = (
"Install FFmpeg: https://ffmpeg.org/download.html\n"
"Windows: winget install FFmpeg\n"
"macOS: brew install ffmpeg\n"
"Linux: sudo apt install ffmpeg"
)
agent_skills = ["ffmpeg", "video_toolkit"]
capabilities = [
"validate_clips",
"stitch",
"crossfade",
"fade_through_black",
"preview_stitch",
"spatial_side_by_side",
"spatial_vertical_stack",
"spatial_picture_in_picture",
]
input_schema = {
"type": "object",
"required": ["operation"],
"properties": {
"operation": {
"type": "string",
"enum": ["validate", "stitch", "preview_stitch", "spatial"],
},
"clips": {
"type": "array",
"items": {"type": "string"},
"description": "List of input video file paths",
},
"output_path": {"type": "string"},
"transition": {
"type": "string",
"enum": ["cut", "crossfade", "fade"],
"default": "cut",
"description": "Transition type: cut (default), crossfade, or fade (fade-through-black)",
},
"transition_duration": {
"type": "number",
"minimum": 0.1,
"maximum": 5.0,
"default": 0.5,
"description": "Transition duration in seconds",
},
"auto_normalize": {
"type": "boolean",
"default": False,
"description": "Re-encode clips to a common format before concat if they differ",
},
"target_resolution": {
"type": "string",
"description": "Target resolution for normalization (e.g. '1920x1080')",
},
"target_fps": {
"type": "integer",
"description": "Target FPS for normalization",
},
"codec": {"type": "string", "default": "libx264"},
"crf": {"type": "integer", "default": 23},
"preset": {"type": "string", "default": "medium"},
"profile": {
"type": "string",
"description": "Media profile name from media_profiles.py",
},
"layout": {
"type": "string",
"enum": ["side_by_side", "vertical_stack", "picture_in_picture"],
"description": "Spatial layout for the spatial operation",
},
"pip_position": {
"type": "string",
"enum": ["top_left", "top_right", "bottom_left", "bottom_right"],
"default": "bottom_right",
"description": "Position of the PiP overlay",
},
"pip_scale": {
"type": "number",
"minimum": 0.1,
"maximum": 0.5,
"default": 0.3,
"description": "Scale of PiP overlay relative to base video",
},
"pip_margin": {
"type": "integer",
"default": 10,
"description": "Margin in pixels for PiP overlay from edges",
},
"dry_run": {
"type": "boolean",
"default": False,
"description": "If true, return what would be done without executing",
},
},
}
resource_profile = ResourceProfile(
cpu_cores=4, ram_mb=2048, vram_mb=0, disk_mb=5000, network_required=False
)
retry_policy = RetryPolicy(max_retries=1, retryable_errors=["Conversion failed"])
resume_support = ResumeSupport.FROM_START
idempotency_key_fields = ["operation", "clips", "transition", "layout"]
side_effects = ["writes video file to output_path"]
user_visible_verification = [
"Play the stitched output and verify clip ordering, transitions, and A/V sync",
]
def execute(self, inputs: dict[str, Any]) -> ToolResult:
operation = inputs["operation"]
start = time.time()
if inputs.get("dry_run"):
return ToolResult(
success=True,
data=self.dry_run(inputs),
)
try:
if operation == "validate":
result = self._validate(inputs)
elif operation == "stitch":
result = self._stitch(inputs)
elif operation == "preview_stitch":
result = self._preview_stitch(inputs)
elif operation == "spatial":
result = self._spatial(inputs)
else:
return ToolResult(success=False, error=f"Unknown operation: {operation}")
except Exception as e:
return ToolResult(success=False, error=str(e))
result.duration_seconds = round(time.time() - start, 2)
return result
def dry_run(self, inputs: dict[str, Any]) -> dict[str, Any]:
"""Preflight check: validate clips and report what would happen."""
clips = inputs.get("clips", [])
operation = inputs.get("operation", "stitch")
info = {
"tool": self.name,
"operation": operation,
"clip_count": len(clips),
"transition": inputs.get("transition", "cut"),
"auto_normalize": inputs.get("auto_normalize", False),
"estimated_cost_usd": self.estimate_cost(inputs),
"estimated_runtime_seconds": self.estimate_runtime(inputs),
"status": self.get_status().value,
"would_execute": True,
}
if clips:
probe_results = []
for clip in clips:
if Path(clip).exists():
probe = self._probe_clip(clip)
if probe:
probe_results.append(probe)
info["clip_info"] = probe_results
return info
# ------------------------------------------------------------------
# Audio-stream detection and silent-audio helpers
# ------------------------------------------------------------------
def _clip_has_audio(self, clip_path: str) -> bool:
"""Return True if *clip_path* contains at least one audio stream."""
cmd = [
"ffprobe", "-v", "quiet",
"-select_streams", "a",
"-show_entries", "stream=codec_type",
"-of", "json",
str(clip_path),
]
try:
proc = self.run_command(cmd)
data = json.loads(proc.stdout)
return len(data.get("streams", [])) > 0
except Exception:
return False
def _ensure_audio_for_clips(
self,
clips: list[str],
temp_dir: Path,
temp_files: list[Path],
) -> list[str]:
"""Return a list of clip paths where every clip is guaranteed to have
an audio stream. Clips that already contain audio are returned as-is.
For clips without audio, a silent stereo AAC track is muxed in and the
path to the new file is returned instead. All generated temp files are
appended to *temp_files* so the caller can clean them up.
"""
result: list[str] = []
for i, clip in enumerate(clips):
if self._clip_has_audio(clip):
result.append(clip)
else:
augmented = temp_dir / f"audio_aug_{i:04d}.mp4"
cmd = [
"ffmpeg", "-y",
"-i", str(clip),
"-f", "lavfi", "-i", "anullsrc=r=44100:cl=stereo",
"-c:v", "copy",
"-c:a", "aac",
"-shortest",
str(augmented),
]
self.run_command(cmd)
temp_files.append(augmented)
result.append(str(augmented))
return result
# ------------------------------------------------------------------
# Probe helper
# ------------------------------------------------------------------
def _probe_clip(self, clip_path: str) -> Optional[dict[str, Any]]:
"""Probe a single clip with ffprobe and return metadata dict."""
cmd = [
"ffprobe", "-v", "quiet",
"-print_format", "json",
"-show_streams",
"-show_format",
str(clip_path),
]
try:
proc = self.run_command(cmd)
data = json.loads(proc.stdout)
except Exception:
return None
info: dict[str, Any] = {"path": str(clip_path)}
# Extract video stream info
for stream in data.get("streams", []):
if stream.get("codec_type") == "video":
info["width"] = stream.get("width")
info["height"] = stream.get("height")
info["video_codec"] = stream.get("codec_name")
info["pixel_format"] = stream.get("pix_fmt")
# Parse fps from r_frame_rate (e.g. "30/1")
rfr = stream.get("r_frame_rate", "0/1")
try:
num, den = rfr.split("/")
info["fps"] = round(int(num) / int(den), 2)
except (ValueError, ZeroDivisionError):
info["fps"] = None
break
# Extract audio stream info
for stream in data.get("streams", []):
if stream.get("codec_type") == "audio":
info["audio_codec"] = stream.get("codec_name")
info["sample_rate"] = stream.get("sample_rate")
info["audio_channels"] = stream.get("channels")
break
# Duration from format
fmt = data.get("format", {})
try:
info["duration"] = float(fmt.get("duration", 0))
except (TypeError, ValueError):
info["duration"] = 0.0
try:
info["file_size_bytes"] = int(fmt.get("size", 0))
except (TypeError, ValueError):
info["file_size_bytes"] = 0
return info
# ------------------------------------------------------------------
# validate
# ------------------------------------------------------------------
def _validate(self, inputs: dict[str, Any]) -> ToolResult:
"""Check clip compatibility: resolution, fps, codec, audio format.
Returns a detailed report of mismatches.
"""
clips = inputs.get("clips", [])
if not clips:
return ToolResult(success=False, error="No clips provided")
# Probe all clips
probes: list[dict[str, Any]] = []
missing: list[str] = []
probe_errors: list[str] = []
for clip in clips:
if not Path(clip).exists():
missing.append(clip)
continue
info = self._probe_clip(clip)
if info is None:
probe_errors.append(clip)
else:
probes.append(info)
if missing:
return ToolResult(
success=False,
error=f"Clips not found: {', '.join(missing)}",
)
if probe_errors:
return ToolResult(
success=False,
error=f"Failed to probe clips: {', '.join(probe_errors)}",
)
# Compare properties across clips
mismatches: list[dict[str, Any]] = []
reference = probes[0]
check_fields = [
("width", "resolution width"),
("height", "resolution height"),
("fps", "frame rate"),
("video_codec", "video codec"),
("pixel_format", "pixel format"),
("audio_codec", "audio codec"),
("sample_rate", "audio sample rate"),
("audio_channels", "audio channels"),
]
for i, probe in enumerate(probes[1:], start=1):
clip_mismatches: list[str] = []
for field_key, label in check_fields:
ref_val = reference.get(field_key)
cur_val = probe.get(field_key)
if ref_val is not None and cur_val is not None and ref_val != cur_val:
clip_mismatches.append(
f"{label}: clip[0]={ref_val} vs clip[{i}]={cur_val}"
)
if clip_mismatches:
mismatches.append({
"clip_index": i,
"clip_path": probe["path"],
"differences": clip_mismatches,
})
compatible = len(mismatches) == 0
total_duration = sum(p.get("duration", 0) for p in probes)
return ToolResult(
success=True,
data={
"operation": "validate",
"clip_count": len(clips),
"compatible": compatible,
"total_duration": round(total_duration, 2),
"reference_clip": {
"path": reference["path"],
"resolution": f"{reference.get('width')}x{reference.get('height')}",
"fps": reference.get("fps"),
"video_codec": reference.get("video_codec"),
"audio_codec": reference.get("audio_codec"),
},
"mismatches": mismatches,
"clips": probes,
},
)
# ------------------------------------------------------------------
# Normalization helper
# ------------------------------------------------------------------
def _resolve_normalization_target(
self, inputs: dict[str, Any], probes: list[dict[str, Any]]
) -> tuple[int, int, int, str, str]:
"""Determine the target resolution, fps, and codecs for normalization.
Returns (width, height, fps, video_codec, audio_codec).
"""
# If a media profile is specified, use it
profile_name = inputs.get("profile")
if profile_name:
try:
from lib.media_profiles import get_profile
profile = get_profile(profile_name)
return (profile.width, profile.height, profile.fps, profile.codec, profile.audio_codec)
except (ImportError, ValueError):
pass
# Explicit target overrides
target_w, target_h = None, None
if inputs.get("target_resolution"):
parts = inputs["target_resolution"].split("x")
if len(parts) == 2:
target_w, target_h = int(parts[0]), int(parts[1])
target_fps = inputs.get("target_fps")
# Fall back to first clip as reference
ref = probes[0] if probes else {}
width = target_w or ref.get("width", 1920)
height = target_h or ref.get("height", 1080)
fps = target_fps or ref.get("fps", 30)
video_codec = inputs.get("codec", "libx264")
audio_codec = "aac"
return (width, height, int(fps), video_codec, audio_codec)
def _normalize_clip(
self,
clip_path: str,
output_path: Path,
width: int,
height: int,
fps: int,
video_codec: str,
audio_codec: str,
crf: int,
preset: str,
) -> None:
"""Re-encode a clip to the target format."""
cmd = [
"ffmpeg", "-y",
"-i", str(clip_path),
"-vf", f"scale={width}:{height}:force_original_aspect_ratio=decrease,pad={width}:{height}:(ow-iw)/2:(oh-ih)/2",
"-r", str(fps),
"-c:v", video_codec, "-crf", str(crf), "-preset", preset,
"-c:a", audio_codec, "-ar", "44100", "-ac", "2",
"-pix_fmt", "yuv420p",
str(output_path),
]
self.run_command(cmd)
def _needs_normalization(self, probes: list[dict[str, Any]]) -> bool:
"""Check whether clips need normalization to be concat-compatible."""
if len(probes) < 2:
return False
ref = probes[0]
for probe in probes[1:]:
for key in ("width", "height", "fps", "video_codec", "audio_codec", "sample_rate"):
if ref.get(key) != probe.get(key) and ref.get(key) is not None:
return True
return False
# ------------------------------------------------------------------
# stitch
# ------------------------------------------------------------------
def _stitch(self, inputs: dict[str, Any]) -> ToolResult:
"""Concatenate clips sequentially with FFmpeg concat demuxer.
Supports transitions: cut (default), crossfade, fade-through-black.
"""
clips = inputs.get("clips", [])
if not clips:
return ToolResult(success=False, error="No clips provided")
if len(clips) < 2:
return ToolResult(success=False, error="At least 2 clips required for stitch")
output_path = Path(inputs.get("output_path", "stitched_output.mp4"))
output_path.parent.mkdir(parents=True, exist_ok=True)
transition = inputs.get("transition", "cut")
transition_dur = inputs.get("transition_duration", 0.5)
auto_normalize = inputs.get("auto_normalize", False)
codec = inputs.get("codec", "libx264")
crf = inputs.get("crf", 23)
preset = inputs.get("preset", "medium")
# Verify all clips exist
for clip in clips:
if not Path(clip).exists():
return ToolResult(success=False, error=f"Clip not found: {clip}")
# Probe clips for compatibility check
probes: list[dict[str, Any]] = []
for clip in clips:
info = self._probe_clip(clip)
if info is None:
return ToolResult(success=False, error=f"Failed to probe clip: {clip}")
probes.append(info)
needs_norm = self._needs_normalization(probes)
# If clips are incompatible and auto_normalize is off, fail with advice
if needs_norm and not auto_normalize and transition == "cut":
return ToolResult(
success=False,
error=(
"Clips have mismatched properties (resolution/fps/codec). "
"Set auto_normalize=true to re-encode to a common format, "
"or use a transition type other than 'cut'."
),
)
temp_dir = output_path.parent / ".stitch_tmp"
temp_dir.mkdir(parents=True, exist_ok=True)
temp_files: list[Path] = []
try:
# Normalize clips if needed
working_clips: list[str] = []
if needs_norm or auto_normalize or transition != "cut":
width, height, fps, vid_codec, aud_codec = self._resolve_normalization_target(inputs, probes)
for i, clip in enumerate(clips):
norm_path = temp_dir / f"norm_{i:04d}.mp4"
self._normalize_clip(clip, norm_path, width, height, fps, vid_codec, aud_codec, crf, preset)
working_clips.append(str(norm_path))
temp_files.append(norm_path)
else:
working_clips = list(clips)
# For crossfade/fade transitions, ensure every clip has an audio
# stream so that the acrossfade filter does not fail. Image-derived
# video clips typically lack audio; we add a silent track for those.
if transition in ("crossfade", "fade"):
working_clips = self._ensure_audio_for_clips(
working_clips, temp_dir, temp_files,
)
if transition == "cut":
result_data = self._stitch_cut(working_clips, output_path, temp_dir, temp_files)
elif transition == "crossfade":
result_data = self._stitch_crossfade(working_clips, output_path, transition_dur, probes)
elif transition == "fade":
result_data = self._stitch_fade_through_black(working_clips, output_path, transition_dur, probes)
else:
return ToolResult(success=False, error=f"Unknown transition type: {transition}")
# Get output file info
file_size = output_path.stat().st_size if output_path.exists() else 0
out_probe = self._probe_clip(str(output_path))
out_duration = out_probe.get("duration", 0) if out_probe else 0
return ToolResult(
success=True,
data={
"operation": "stitch",
"clip_count": len(clips),
"transition": transition,
"transition_duration": transition_dur if transition != "cut" else 0,
"auto_normalized": needs_norm or auto_normalize,
"output": str(output_path),
"duration": round(out_duration, 2),
"file_size_bytes": file_size,
**result_data,
},
artifacts=[str(output_path)],
)
finally:
self._cleanup_temp(temp_dir, temp_files)
def _stitch_cut(
self,
clips: list[str],
output_path: Path,
temp_dir: Path,
temp_files: list[Path],
) -> dict[str, Any]:
"""Simple concat via FFmpeg concat demuxer (no transition)."""
concat_list = temp_dir / "concat_list.txt"
temp_files.append(concat_list)
with open(concat_list, "w", encoding="utf-8") as f:
for clip in clips:
safe_path = str(Path(clip).resolve()).replace("\\", "/")
f.write(f"file '{safe_path}'\n")
cmd = [
"ffmpeg", "-y",
"-f", "concat", "-safe", "0",
"-i", str(concat_list),
"-c", "copy",
str(output_path),
]
self.run_command(cmd)
return {"method": "concat_demuxer"}
def _stitch_crossfade(
self,
clips: list[str],
output_path: Path,
duration: float,
probes: list[dict[str, Any]],
) -> dict[str, Any]:
"""Crossfade between adjacent clips using xfade filter."""
if len(clips) == 2:
# Simple two-clip crossfade
cmd = [
"ffmpeg", "-y",
"-i", clips[0],
"-i", clips[1],
"-filter_complex",
f"[0:v][1:v]xfade=transition=fade:duration={duration}:offset={self._get_xfade_offset(probes, 0, duration)}[v];"
f"[0:a][1:a]acrossfade=d={duration}[a]",
"-map", "[v]", "-map", "[a]",
str(output_path),
]
self.run_command(cmd)
else:
# Chain crossfades for N clips
self._chain_xfade(clips, output_path, duration, probes, transition="fade")
return {"method": "xfade_crossfade"}
def _stitch_fade_through_black(
self,
clips: list[str],
output_path: Path,
duration: float,
probes: list[dict[str, Any]],
) -> dict[str, Any]:
"""Fade-through-black between adjacent clips using xfade fadeblack."""
if len(clips) == 2:
cmd = [
"ffmpeg", "-y",
"-i", clips[0],
"-i", clips[1],
"-filter_complex",
f"[0:v][1:v]xfade=transition=fadeblack:duration={duration}:offset={self._get_xfade_offset(probes, 0, duration)}[v];"
f"[0:a][1:a]acrossfade=d={duration}[a]",
"-map", "[v]", "-map", "[a]",
str(output_path),
]
self.run_command(cmd)
else:
self._chain_xfade(clips, output_path, duration, probes, transition="fadeblack")
return {"method": "xfade_fadeblack"}
def _get_xfade_offset(
self, probes: list[dict[str, Any]], clip_index: int, duration: float
) -> float:
"""Calculate xfade offset for a given clip pair.
The offset is the timestamp in the output where the transition starts,
which equals the duration of the first clip minus the transition duration.
"""
clip_dur = probes[clip_index].get("duration", 0) if clip_index < len(probes) else 0
offset = max(0, clip_dur - duration)
return round(offset, 3)
def _chain_xfade(
self,
clips: list[str],
output_path: Path,
duration: float,
probes: list[dict[str, Any]],
transition: str,
) -> None:
"""Chain xfade filters for N > 2 clips.
Builds a complex filtergraph that progressively applies xfade
between each adjacent pair of clips.
"""
n = len(clips)
input_args: list[str] = []
for clip in clips:
input_args.extend(["-i", clip])
# Calculate cumulative offsets
# Each xfade offset = cumulative duration of all previous segments
# minus cumulative transition overlaps minus current transition duration
video_filters: list[str] = []
audio_filters: list[str] = []
cumulative_offset = 0.0
for i in range(n - 1):
clip_dur = probes[i].get("duration", 0) if i < len(probes) else 0
offset = round(cumulative_offset + clip_dur - duration, 3)
offset = max(0, offset)
if i == 0:
v_in1 = "[0:v]"
a_in1 = "[0:a]"
else:
v_in1 = f"[vfade{i-1}]"
a_in1 = f"[afade{i-1}]"
v_in2 = f"[{i+1}:v]"
a_in2 = f"[{i+1}:a]"
if i < n - 2:
v_out = f"[vfade{i}]"
a_out = f"[afade{i}]"
else:
v_out = "[vout]"
a_out = "[aout]"
video_filters.append(
f"{v_in1}{v_in2}xfade=transition={transition}:duration={duration}:offset={offset}{v_out}"
)
audio_filters.append(
f"{a_in1}{a_in2}acrossfade=d={duration}{a_out}"
)
# Cumulative offset advances by clip duration minus overlap
cumulative_offset = offset
filter_complex = ";".join(video_filters + audio_filters)
cmd = ["ffmpeg", "-y"]
cmd.extend(input_args)
cmd.extend(["-filter_complex", filter_complex])
cmd.extend(["-map", "[vout]", "-map", "[aout]"])
cmd.append(str(output_path))
self.run_command(cmd)
# ------------------------------------------------------------------
# preview_stitch
# ------------------------------------------------------------------
def _preview_stitch(self, inputs: dict[str, Any]) -> ToolResult:
"""Generate a low-resolution preview of the stitched result."""
clips = inputs.get("clips", [])
if not clips:
return ToolResult(success=False, error="No clips provided")
if len(clips) < 2:
return ToolResult(success=False, error="At least 2 clips required for preview")
output_path = Path(inputs.get("output_path", "stitch_preview.mp4"))
output_path.parent.mkdir(parents=True, exist_ok=True)
# Verify all clips exist
for clip in clips:
if not Path(clip).exists():
return ToolResult(success=False, error=f"Clip not found: {clip}")
# Build preview by normalizing to low-res and stitching
preview_inputs = dict(inputs)
preview_inputs["auto_normalize"] = True
preview_inputs["target_resolution"] = "640x360"
preview_inputs["target_fps"] = 24
preview_inputs["crf"] = 30
preview_inputs["preset"] = "ultrafast"
preview_inputs["output_path"] = str(output_path)
# Delegate to _stitch with preview settings
result = self._stitch(preview_inputs)
if result.success:
result.data["operation"] = "preview_stitch"
result.data["preview"] = True
result.data["preview_resolution"] = "640x360"
return result
# ------------------------------------------------------------------
# spatial
# ------------------------------------------------------------------
def _spatial(self, inputs: dict[str, Any]) -> ToolResult:
"""Side-by-side, vertical stack, or picture-in-picture layouts.
Designed for TikTok Stitch/Duet style compositions (D3.5.8).
"""
clips = inputs.get("clips", [])
if not clips or len(clips) < 2:
return ToolResult(
success=False,
error="At least 2 clips required for spatial layout",
)
layout = inputs.get("layout")
if not layout:
return ToolResult(success=False, error="layout is required for spatial operation")
output_path = Path(inputs.get("output_path", "spatial_output.mp4"))
output_path.parent.mkdir(parents=True, exist_ok=True)
codec = inputs.get("codec", "libx264")
crf = inputs.get("crf", 23)
# Verify all clips exist
for clip in clips:
if not Path(clip).exists():
return ToolResult(success=False, error=f"Clip not found: {clip}")
temp_dir = output_path.parent / ".spatial_tmp"
temp_dir.mkdir(parents=True, exist_ok=True)
temp_files: list[Path] = []
try:
# side_by_side and vertical_stack use amix which requires audio
# on both inputs. Ensure silent tracks for audio-less clips.
working_clips = list(clips)
if layout in ("side_by_side", "vertical_stack"):
working_clips = self._ensure_audio_for_clips(
working_clips, temp_dir, temp_files,
)
if layout == "side_by_side":
self._spatial_side_by_side(working_clips, output_path, codec, crf)
elif layout == "vertical_stack":
self._spatial_vertical_stack(working_clips, output_path, codec, crf)
elif layout == "picture_in_picture":
self._spatial_pip(working_clips, output_path, inputs, codec, crf)
else:
return ToolResult(success=False, error=f"Unknown layout: {layout}")
except Exception as e:
return ToolResult(success=False, error=str(e))
finally:
self._cleanup_temp(temp_dir, temp_files)
file_size = output_path.stat().st_size if output_path.exists() else 0
out_probe = self._probe_clip(str(output_path))
out_duration = out_probe.get("duration", 0) if out_probe else 0
return ToolResult(
success=True,
data={
"operation": "spatial",
"layout": layout,
"clip_count": len(clips),
"output": str(output_path),
"duration": round(out_duration, 2),
"file_size_bytes": file_size,
},
artifacts=[str(output_path)],
)
def _spatial_side_by_side(
self, clips: list[str], output_path: Path, codec: str, crf: int
) -> None:
"""Place clips side by side (horizontal split).
Both clips are scaled to the same height and placed left-right.
Uses the first two clips; additional clips are ignored.
"""
input_args = ["-i", clips[0], "-i", clips[1]]
filter_complex = (
"[0:v]scale=-2:480[left];"
"[1:v]scale=-2:480[right];"
"[left][right]hstack=inputs=2[v];"
"[0:a][1:a]amix=inputs=2:duration=shortest[a]"
)
cmd = ["ffmpeg", "-y"]
cmd.extend(input_args)
cmd.extend([
"-filter_complex", filter_complex,
"-map", "[v]", "-map", "[a]",
"-c:v", codec, "-crf", str(crf),
"-c:a", "aac",
"-shortest",
str(output_path),
])
self.run_command(cmd)
def _spatial_vertical_stack(
self, clips: list[str], output_path: Path, codec: str, crf: int
) -> None:
"""Place clips in a vertical stack (top-bottom).
Both clips are scaled to the same width and stacked vertically.
Ideal for portrait/mobile viewing.
"""
input_args = ["-i", clips[0], "-i", clips[1]]
filter_complex = (
"[0:v]scale=540:-2[top];"
"[1:v]scale=540:-2[bottom];"
"[top][bottom]vstack=inputs=2[v];"
"[0:a][1:a]amix=inputs=2:duration=shortest[a]"
)
cmd = ["ffmpeg", "-y"]
cmd.extend(input_args)
cmd.extend([
"-filter_complex", filter_complex,
"-map", "[v]", "-map", "[a]",
"-c:v", codec, "-crf", str(crf),
"-c:a", "aac",
"-shortest",
str(output_path),
])
self.run_command(cmd)
def _spatial_pip(
self,
clips: list[str],
output_path: Path,
inputs: dict[str, Any],
codec: str,
crf: int,
) -> None:
"""Picture-in-picture: overlay second clip on first.
clips[0] is the base (full-screen), clips[1] is the PiP overlay.
"""
pip_position = inputs.get("pip_position", "bottom_right")
pip_scale = inputs.get("pip_scale", 0.3)
pip_margin = inputs.get("pip_margin", 10)
# Build position expression based on corner
position_map = {
"top_left": f"{pip_margin}:{pip_margin}",
"top_right": f"main_w-overlay_w-{pip_margin}:{pip_margin}",
"bottom_left": f"{pip_margin}:main_h-overlay_h-{pip_margin}",
"bottom_right": f"main_w-overlay_w-{pip_margin}:main_h-overlay_h-{pip_margin}",
}
position = position_map.get(pip_position, position_map["bottom_right"])
input_args = ["-i", clips[0], "-i", clips[1]]
filter_complex = (
f"[1:v]scale=iw*{pip_scale}:ih*{pip_scale}[pip];"
f"[0:v][pip]overlay={position}:shortest=1[v]"
)
cmd = ["ffmpeg", "-y"]
cmd.extend(input_args)
cmd.extend([
"-filter_complex", filter_complex,
"-map", "[v]", "-map", "0:a?",
"-c:v", codec, "-crf", str(crf),
"-c:a", "aac",
"-shortest",
str(output_path),
])
self.run_command(cmd)
# ------------------------------------------------------------------
# Cleanup
# ------------------------------------------------------------------
@staticmethod
def _cleanup_temp(temp_dir: Path, temp_files: list[Path]) -> None:
"""Remove temporary files and directory."""
for f in temp_files:
if f.exists():
try:
f.unlink()
except OSError:
pass
if temp_dir.exists():
try:
temp_dir.rmdir()
except OSError:
pass
tools/video/video_trimmer.py
+270
View File
@@ -0,0 +1,270 @@
"""Video trimmer tool wrapping FFmpeg.
Provides cut, trim, speed adjustment, and concatenation of video segments.
All operations are deterministic and produce lossless or near-lossless output
by default.
"""
from __future__ import annotations
import json
import time
from pathlib import Path
from typing import Any
from tools.base_tool import (
BaseTool,
Determinism,
ExecutionMode,
ResourceProfile,
RetryPolicy,
ResumeSupport,
ToolResult,
ToolStability,
ToolTier,
)
class VideoTrimmer(BaseTool):
name = "video_trimmer"
version = "0.1.0"
tier = ToolTier.CORE
capability = "video_post"
provider = "ffmpeg"
stability = ToolStability.EXPERIMENTAL
execution_mode = ExecutionMode.SYNC
determinism = Determinism.DETERMINISTIC
dependencies = ["cmd:ffmpeg"]
install_instructions = (
"Install FFmpeg: https://ffmpeg.org/download.html\n"
"Windows: winget install FFmpeg\n"
"macOS: brew install ffmpeg\n"
"Linux: sudo apt install ffmpeg"
)
agent_skills = ["ffmpeg", "video_toolkit"]
capabilities = ["cut", "trim", "speed_adjust", "concat"]
input_schema = {
"type": "object",
"required": ["operation"],
"properties": {
"operation": {
"type": "string",
"enum": ["cut", "speed", "concat"],
},
"input_path": {"type": "string"},
"output_path": {"type": "string"},
"start_seconds": {"type": "number", "minimum": 0},
"end_seconds": {"type": "number", "minimum": 0},
"speed_factor": {"type": "number", "minimum": 0.1, "maximum": 100.0},
"segments": {
"type": "array",
"items": {
"type": "object",
"properties": {
"input_path": {"type": "string"},
"start_seconds": {"type": "number"},
"end_seconds": {"type": "number"},
},
},
},
"codec": {"type": "string", "default": "copy"},
},
}
resource_profile = ResourceProfile(
cpu_cores=2, ram_mb=1024, vram_mb=0, disk_mb=2000, network_required=False
)
retry_policy = RetryPolicy(max_retries=1, retryable_errors=["FFmpeg error"])
resume_support = ResumeSupport.FROM_START
idempotency_key_fields = ["operation", "input_path", "start_seconds", "end_seconds", "speed_factor"]
side_effects = ["writes video file to output_path"]
user_visible_verification = ["Play trimmed output and verify cut points"]
def execute(self, inputs: dict[str, Any]) -> ToolResult:
operation = inputs["operation"]
start = time.time()
try:
if operation == "cut":
result = self._cut(inputs)
elif operation == "speed":
result = self._speed(inputs)
elif operation == "concat":
result = self._concat(inputs)
else:
return ToolResult(success=False, error=f"Unknown operation: {operation}")
except Exception as e:
return ToolResult(success=False, error=str(e))
result.duration_seconds = round(time.time() - start, 2)
return result
def _cut(self, inputs: dict[str, Any]) -> ToolResult:
input_path = Path(inputs["input_path"])
if not input_path.exists():
return ToolResult(success=False, error=f"Input not found: {input_path}")
start_s = inputs.get("start_seconds", 0)
end_s = inputs.get("end_seconds")
codec = inputs.get("codec", "copy")
output_path = Path(
inputs.get("output_path", str(input_path.with_stem(f"{input_path.stem}_cut")))
)
cmd = [
"ffmpeg", "-y",
"-i", str(input_path),
"-ss", str(start_s),
]
if end_s is not None:
cmd.extend(["-to", str(end_s)])
if codec == "copy":
cmd.extend(["-c", "copy"])
else:
cmd.extend(["-c:v", codec, "-c:a", "aac"])
cmd.append(str(output_path))
self.run_command(cmd)
return ToolResult(
success=True,
data={
"operation": "cut",
"input": str(input_path),
"output": str(output_path),
"start_seconds": start_s,
"end_seconds": end_s,
},
artifacts=[str(output_path)],
)
def _speed(self, inputs: dict[str, Any]) -> ToolResult:
input_path = Path(inputs["input_path"])
if not input_path.exists():
return ToolResult(success=False, error=f"Input not found: {input_path}")
factor = inputs.get("speed_factor", 1.0)
output_path = Path(
inputs.get("output_path", str(input_path.with_stem(f"{input_path.stem}_speed")))
)
# Video: setpts adjusts presentation timestamps (inverse of speed)
# Audio: atempo adjusts audio speed (must chain for >2x)
video_filter = f"setpts={1.0/factor}*PTS"
audio_filters = self._build_atempo_chain(factor)
cmd = [
"ffmpeg", "-y",
"-i", str(input_path),
"-filter:v", video_filter,
"-filter:a", audio_filters,
"-c:v", "libx264", "-preset", "fast",
"-c:a", "aac",
str(output_path),
]
self.run_command(cmd)
return ToolResult(
success=True,
data={
"operation": "speed",
"input": str(input_path),
"output": str(output_path),
"speed_factor": factor,
},
artifacts=[str(output_path)],
)
def _concat(self, inputs: dict[str, Any]) -> ToolResult:
segments = inputs.get("segments", [])
if not segments:
return ToolResult(success=False, error="No segments provided for concat")
output_path = Path(inputs.get("output_path", "concat_output.mp4"))
# First, cut each segment to a temp file if start/end are specified
temp_files: list[Path] = []
temp_dir = output_path.parent / ".concat_tmp"
temp_dir.mkdir(parents=True, exist_ok=True)
try:
for i, seg in enumerate(segments):
seg_input = Path(seg["input_path"])
if not seg_input.exists():
return ToolResult(success=False, error=f"Segment input not found: {seg_input}")
seg_start = seg.get("start_seconds")
seg_end = seg.get("end_seconds")
if seg_start is not None or seg_end is not None:
temp_path = temp_dir / f"seg_{i:04d}{seg_input.suffix}"
cmd = ["ffmpeg", "-y", "-i", str(seg_input)]
if seg_start is not None:
cmd.extend(["-ss", str(seg_start)])
if seg_end is not None:
cmd.extend(["-to", str(seg_end)])
cmd.extend(["-c", "copy", str(temp_path)])
self.run_command(cmd)
temp_files.append(temp_path)
else:
temp_files.append(seg_input)
# Write concat file list
list_path = temp_dir / "concat_list.txt"
with open(list_path, "w", encoding="utf-8") as f:
for tf in temp_files:
# FFmpeg concat demuxer needs forward slashes and escaped quotes
safe_path = str(tf.resolve()).replace("\\", "/")
f.write(f"file '{safe_path}'\n")
cmd = [
"ffmpeg", "-y",
"-f", "concat", "-safe", "0",
"-i", str(list_path),
"-c", "copy",
str(output_path),
]
self.run_command(cmd)
return ToolResult(
success=True,
data={
"operation": "concat",
"segment_count": len(segments),
"output": str(output_path),
},
artifacts=[str(output_path)],
)
finally:
# Clean up temp segment files (but not the originals)
for tf in temp_files:
if tf.parent == temp_dir and tf.exists():
tf.unlink()
if list_path.exists():
list_path.unlink()
if temp_dir.exists():
try:
temp_dir.rmdir()
except OSError:
pass
@staticmethod
def _build_atempo_chain(factor: float) -> str:
"""Build an atempo filter chain. atempo only accepts [0.5, 100.0]."""
if factor <= 0:
factor = 1.0
# Chain multiple atempo filters for extreme values
filters = []
remaining = factor
while remaining > 100.0:
filters.append("atempo=100.0")
remaining /= 100.0
while remaining < 0.5:
filters.append("atempo=0.5")
remaining /= 0.5
filters.append(f"atempo={remaining:.4f}")
return ",".join(filters)
tools/video/wan_video.py
+98
View File
@@ -0,0 +1,98 @@
"""Wan local video generation."""
from __future__ import annotations
import time
from typing import Any
from tools.base_tool import (
BaseTool,
Determinism,
ExecutionMode,
ResourceProfile,
RetryPolicy,
ToolResult,
ToolRuntime,
ToolStability,
ToolStatus,
ToolTier,
)
from tools.video._shared import WAN_VARIANTS, estimate_local_runtime, generate_local_video, local_generation_status, local_install_instructions
class WanVideo(BaseTool):
name = "wan_video"
version = "0.1.0"
tier = ToolTier.GENERATE
capability = "video_generation"
provider = "wan"
stability = ToolStability.EXPERIMENTAL
execution_mode = ExecutionMode.SYNC
determinism = Determinism.STOCHASTIC
runtime = ToolRuntime.LOCAL_GPU
install_instructions = local_install_instructions()
fallback = "hunyuan_video"
fallback_tools = ["hunyuan_video", "ltx_video_local", "cogvideo_video", "image_selector"]
agent_skills = ["ltx2"]
capabilities = ["text_to_video", "image_to_video", "model_selection"]
supports = {
"reference_image": True,
"offline": True,
"native_audio": False,
"local_gpu": True,
}
best_for = [
"best quality-to-VRAM ratio for local generation",
"local pipelines that still want image-to-video support",
]
not_good_for = ["CPU-only machines", "instant iteration on low-end hardware"]
provider_matrix = {key: {"tool": "wan_video", **value, "mode": "local_gpu"} for key, value in WAN_VARIANTS.items()}
input_schema = {
"type": "object",
"required": ["prompt"],
"properties": {
"prompt": {"type": "string"},
"operation": {"type": "string", "enum": ["text_to_video", "image_to_video"], "default": "text_to_video"},
"model_variant": {"type": "string", "enum": sorted(WAN_VARIANTS), "default": "wan2.1-1.3b"},
"reference_image_url": {"type": "string"},
"reference_image_path": {"type": "string"},
"width": {"type": "integer"},
"height": {"type": "integer"},
"num_frames": {"type": "integer"},
"num_inference_steps": {"type": "integer"},
"enable_model_offload": {"type": "boolean", "default": True},
"seed": {"type": "integer"},
"output_path": {"type": "string"},
},
}
resource_profile = ResourceProfile(cpu_cores=2, ram_mb=16000, vram_mb=8000, disk_mb=4000, network_required=False)
retry_policy = RetryPolicy(max_retries=1)
idempotency_key_fields = ["prompt", "model_variant", "operation", "seed"]
side_effects = ["writes video file to output_path", "may download model weights"]
user_visible_verification = ["Watch generated clip for motion coherence and artifacts"]
def get_status(self) -> ToolStatus:
return local_generation_status()
def estimate_cost(self, inputs: dict[str, Any]) -> float:
return 0.0
def estimate_runtime(self, inputs: dict[str, Any]) -> float:
variant = WAN_VARIANTS.get(inputs.get("model_variant", "wan2.1-1.3b"), WAN_VARIANTS["wan2.1-1.3b"])
return estimate_local_runtime(variant["speed"])
def execute(self, inputs: dict[str, Any]) -> ToolResult:
if self.get_status() != ToolStatus.AVAILABLE:
return ToolResult(success=False, error="Wan local video generation is unavailable. " + self.install_instructions)
start = time.time()
try:
result = generate_local_video(tool_name=self.name, variants=WAN_VARIANTS, default_variant="wan2.1-1.3b", inputs=inputs)
except Exception as exc:
return ToolResult(success=False, error=f"Wan video generation failed: {exc}")
result.duration_seconds = round(time.time() - start, 2)
return result