Initial release — OpenMontage: the first open-source agentic video production system

11 production pipelines, 47 tools, 124 agent skills.
Supports cloud APIs (fal.ai, OpenAI, ElevenLabs, Suno, HeyGen, Runway) and
free local providers (diffusers, Piper TTS, WAN 2.1, Hunyuan, CogVideo).

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
calesthio
2026-03-29 08:25:17 -07:00
commit a3e735cc7a
1147 changed files with 240221 additions and 0 deletions
+1
View File
@@ -0,0 +1 @@
"""Audio tools — TTS providers, audio processing, and music generation."""
+186
View File
@@ -0,0 +1,186 @@
"""Audio enhancement tool for noise reduction and cleanup.
Provides noise reduction, normalization, and EQ via FFmpeg audio
filters. Optional pedalboard integration for higher-quality
processing when available.
"""
from __future__ import annotations
import time
from pathlib import Path
from typing import Any
from tools.base_tool import (
BaseTool,
Determinism,
ExecutionMode,
ResourceProfile,
ToolResult,
ToolStability,
ToolTier,
)
PRESETS = {
"clean_speech": {
"description": "Noise gate + highpass + compressor + limiter for clean dialogue",
"af": (
"highpass=f=80,"
"lowpass=f=13000,"
"agate=threshold=0.01:ratio=2:attack=5:release=50,"
"acompressor=threshold=-20dB:ratio=3:attack=5:release=100,"
"loudnorm=I=-16:LRA=11:TP=-1.5"
),
},
"noise_reduce": {
"description": "Aggressive noise reduction for noisy environments",
"af": (
"afftdn=nf=-25:nt=w,"
"highpass=f=100,"
"loudnorm=I=-16:LRA=11:TP=-1.5"
),
},
"normalize_only": {
"description": "Loudness normalization without other processing",
"af": "loudnorm=I=-16:LRA=11:TP=-1.5",
},
"podcast": {
"description": "Podcast-style processing: de-ess, compress, normalize",
"af": (
"highpass=f=80,"
"acompressor=threshold=-18dB:ratio=4:attack=5:release=100:makeup=2,"
"loudnorm=I=-16:LRA=7:TP=-1.5"
),
},
"broadcast": {
"description": "Broadcast-standard processing with tight dynamics",
"af": (
"highpass=f=80,"
"lowpass=f=15000,"
"acompressor=threshold=-24dB:ratio=4:attack=5:release=80:makeup=3,"
"alimiter=limit=0.95:attack=1:release=10,"
"loudnorm=I=-24:LRA=7:TP=-2"
),
},
"voice_clarity": {
"description": "Boost vocal presence with EQ and light compression",
"af": (
"highpass=f=80,"
"equalizer=f=200:t=q:w=1.5:g=-3,"
"equalizer=f=3000:t=q:w=1.0:g=3,"
"equalizer=f=5000:t=q:w=1.5:g=2,"
"acompressor=threshold=-20dB:ratio=2.5:attack=10:release=100,"
"loudnorm=I=-16:LRA=11:TP=-1.5"
),
},
}
class AudioEnhance(BaseTool):
name = "audio_enhance"
version = "0.1.0"
tier = ToolTier.CORE
capability = "audio_processing"
provider = "ffmpeg"
stability = ToolStability.EXPERIMENTAL
execution_mode = ExecutionMode.SYNC
determinism = Determinism.DETERMINISTIC
dependencies = ["cmd:ffmpeg"]
install_instructions = "Install FFmpeg: https://ffmpeg.org/download.html"
agent_skills = ["ffmpeg", "elevenlabs"]
capabilities = [
"noise_reduction",
"normalization",
"compression",
"eq",
"speech_cleanup",
]
input_schema = {
"type": "object",
"required": ["input_path"],
"properties": {
"input_path": {"type": "string"},
"output_path": {"type": "string"},
"preset": {
"type": "string",
"enum": list(PRESETS.keys()),
"default": "clean_speech",
},
"custom_af": {
"type": "string",
"description": "Custom FFmpeg audio filter string",
},
"audio_codec": {"type": "string", "default": "aac"},
"audio_bitrate": {"type": "string", "default": "192k"},
},
}
resource_profile = ResourceProfile(cpu_cores=1, ram_mb=512, vram_mb=0, disk_mb=500)
idempotency_key_fields = ["input_path", "preset", "custom_af"]
side_effects = ["writes enhanced audio/video to output_path"]
user_visible_verification = [
"Listen to enhanced audio and compare with original",
"Verify speech is clear without artifacts or pumping",
]
def execute(self, inputs: dict[str, Any]) -> ToolResult:
input_path = Path(inputs["input_path"])
if not input_path.exists():
return ToolResult(success=False, error=f"Input not found: {input_path}")
output_path = Path(
inputs.get("output_path", str(input_path.with_stem(f"{input_path.stem}_enhanced")))
)
audio_codec = inputs.get("audio_codec", "aac")
audio_bitrate = inputs.get("audio_bitrate", "192k")
af = inputs.get("custom_af")
if not af:
preset_name = inputs.get("preset", "clean_speech")
preset = PRESETS.get(preset_name)
if not preset:
return ToolResult(success=False, error=f"Unknown preset: {preset_name}")
af = preset["af"]
start = time.time()
# Determine if input is video or audio-only
is_video = input_path.suffix.lower() in {".mp4", ".mkv", ".avi", ".mov", ".webm"}
cmd = [
"ffmpeg", "-y",
"-i", str(input_path),
"-af", af,
]
if is_video:
cmd.extend(["-c:v", "copy"])
cmd.extend(["-c:a", audio_codec, "-b:a", audio_bitrate])
cmd.append(str(output_path))
try:
self.run_command(cmd)
except Exception as e:
return ToolResult(success=False, error=f"FFmpeg failed: {e}")
elapsed = time.time() - start
return ToolResult(
success=True,
data={
"input": str(input_path),
"output": str(output_path),
"preset": inputs.get("preset"),
"filter": af,
},
artifacts=[str(output_path)],
duration_seconds=round(elapsed, 2),
)
@staticmethod
def list_presets() -> dict[str, str]:
"""Return available presets and their descriptions."""
return {name: p["description"] for name, p in PRESETS.items()}
+560
View File
@@ -0,0 +1,560 @@
"""Audio mixer tool wrapping FFmpeg and pydub.
Mixes speech, music, and SFX tracks with support for ducking, fades,
and volume normalization. Falls back to FFmpeg-only mode if pydub is
not installed.
"""
from __future__ import annotations
import time
from pathlib import Path
from typing import Any
from tools.base_tool import (
BaseTool,
Determinism,
ExecutionMode,
ResourceProfile,
ToolResult,
ToolStability,
ToolStatus,
ToolTier,
)
class AudioMixer(BaseTool):
name = "audio_mixer"
version = "0.1.0"
tier = ToolTier.CORE
capability = "audio_processing"
provider = "ffmpeg"
stability = ToolStability.EXPERIMENTAL
execution_mode = ExecutionMode.SYNC
determinism = Determinism.DETERMINISTIC
dependencies = ["cmd:ffmpeg"]
install_instructions = (
"FFmpeg is required. pydub is optional for advanced mixing:\n"
"pip install pydub"
)
agent_skills = ["ffmpeg", "video_toolkit"]
capabilities = ["mix", "duck", "fade", "normalize", "extract_audio"]
input_schema = {
"type": "object",
"required": ["operation"],
"properties": {
"operation": {
"type": "string",
"enum": ["mix", "duck", "extract", "full_mix"],
"description": (
"mix: layer multiple tracks with volume/delay/fades. "
"duck: lower music volume when speech is present. "
"extract: extract audio from video file. "
"full_mix: combine narration tracks + music with ducking + normalize "
"in a single call (preferred for compose-director)."
),
},
"tracks": {
"type": "array",
"description": (
"Audio tracks for mix/duck operations (advanced format). "
"For duck, each track needs a 'role' of 'speech' or 'music'. "
"For the simple duck API, use primary_audio/secondary_audio instead."
),
"items": {
"type": "object",
"required": ["path", "role"],
"properties": {
"path": {"type": "string"},
"role": {
"type": "string",
"enum": ["speech", "music", "sfx", "primary", "secondary"],
},
"volume": {
"type": "number",
"minimum": 0,
"maximum": 1.0,
"default": 1.0,
},
"start_seconds": {"type": "number", "minimum": 0},
"fade_in_seconds": {"type": "number", "minimum": 0},
"fade_out_seconds": {"type": "number", "minimum": 0},
},
},
},
"primary_audio": {
"type": "string",
"description": (
"Path to primary/speech audio track (duck operation, simple format). "
"This is the track that stays at full volume (e.g. narration/dialogue). "
"Use with secondary_audio as an alternative to the tracks array."
),
},
"secondary_audio": {
"type": "string",
"description": (
"Path to secondary/music audio track (duck operation, simple format). "
"This track gets ducked (volume lowered) when primary audio is present. "
"Use with primary_audio as an alternative to the tracks array."
),
},
"duck_level": {
"type": "number",
"description": (
"Ducking attenuation in dB for the secondary track (duck operation, "
"simple format). Negative values reduce volume, e.g. -12 means duck "
"by 12dB. Converted to a linear ratio internally. Default: -12."
),
"default": -12,
},
"input_path": {"type": "string", "description": "Input for extract operation"},
"output_path": {"type": "string"},
"ducking": {
"type": "object",
"description": (
"Advanced ducking parameters. Works with both the simple "
"(primary_audio/secondary_audio) and advanced (tracks) formats."
),
"properties": {
"enabled": {"type": "boolean", "default": True},
"music_volume_during_speech": {
"type": "number", "minimum": 0, "maximum": 1.0, "default": 0.15,
},
"attack_ms": {"type": "number", "default": 200},
"release_ms": {"type": "number", "default": 500},
},
},
"normalize": {"type": "boolean", "default": True},
},
}
resource_profile = ResourceProfile(cpu_cores=2, ram_mb=1024, vram_mb=0, disk_mb=500)
idempotency_key_fields = ["operation", "tracks", "ducking"]
side_effects = ["writes mixed audio file to output_path"]
user_visible_verification = [
"Listen to mixed output and verify speech clarity and music ducking",
]
def execute(self, inputs: dict[str, Any]) -> ToolResult:
operation = inputs["operation"]
start = time.time()
try:
if operation == "mix":
result = self._mix(inputs)
elif operation == "duck":
result = self._duck(inputs)
elif operation == "extract":
result = self._extract(inputs)
elif operation == "full_mix":
result = self._full_mix(inputs)
else:
return ToolResult(success=False, error=f"Unknown operation: {operation}")
except Exception as e:
return ToolResult(success=False, error=str(e))
result.duration_seconds = round(time.time() - start, 2)
return result
def _mix(self, inputs: dict[str, Any]) -> ToolResult:
"""Mix multiple audio tracks into one output."""
tracks = inputs.get("tracks", [])
if not tracks:
return ToolResult(success=False, error="No tracks provided")
output_path = Path(inputs.get("output_path", "mixed_audio.wav"))
normalize = inputs.get("normalize", True)
# Validate all inputs exist
for t in tracks:
if not Path(t["path"]).exists():
return ToolResult(success=False, error=f"Track not found: {t['path']}")
# Build FFmpeg complex filter for mixing
filter_parts = []
input_args = []
for i, track in enumerate(tracks):
input_args.extend(["-i", track["path"]])
volume = track.get("volume", 1.0)
delay_ms = int(track.get("start_seconds", 0) * 1000)
fade_in = track.get("fade_in_seconds", 0)
fade_out = track.get("fade_out_seconds", 0)
filters = []
if volume != 1.0:
filters.append(f"volume={volume}")
if delay_ms > 0:
filters.append(f"adelay={delay_ms}|{delay_ms}")
if fade_in > 0:
filters.append(f"afade=t=in:d={fade_in}")
if fade_out > 0:
filters.append(f"afade=t=out:d={fade_out}")
if filters:
filter_chain = ",".join(filters)
filter_parts.append(f"[{i}:a]{filter_chain}[a{i}]")
else:
filter_parts.append(f"[{i}:a]acopy[a{i}]")
# Amix all processed streams
mix_inputs = "".join(f"[a{i}]" for i in range(len(tracks)))
filter_parts.append(
f"{mix_inputs}amix=inputs={len(tracks)}:duration=longest:dropout_transition=2[mixed]"
)
if normalize:
filter_parts.append("[mixed]loudnorm=I=-16:LRA=11:TP=-1.5[out]")
out_label = "[out]"
else:
out_label = "[mixed]"
filter_complex = ";".join(filter_parts)
cmd = ["ffmpeg", "-y"]
cmd.extend(input_args)
cmd.extend(["-filter_complex", filter_complex])
cmd.extend(["-map", out_label, str(output_path)])
self.run_command(cmd)
return ToolResult(
success=True,
data={
"operation": "mix",
"track_count": len(tracks),
"output": str(output_path),
"normalized": normalize,
},
artifacts=[str(output_path)],
)
def _duck(self, inputs: dict[str, Any]) -> ToolResult:
"""Apply ducking: lower music volume when speech is present.
Accepts two input formats:
Simple format (preferred for agents):
{
"operation": "duck",
"primary_audio": "speech.mp3",
"secondary_audio": "music.mp3",
"duck_level": -12,
"output_path": "out.wav"
}
Advanced format (tracks array):
{
"operation": "duck",
"tracks": [
{"path": "speech.mp3", "role": "primary"}, # or "speech"
{"path": "music.mp3", "role": "secondary"} # or "music"
],
"output_path": "out.wav"
}
"""
ducking = inputs.get("ducking", {})
output_path = Path(inputs.get("output_path", "ducked_audio.wav"))
# --- Resolve speech/music paths from either input format ---
speech_path = None
music_path = None
# Simple format: primary_audio / secondary_audio
if "primary_audio" in inputs or "secondary_audio" in inputs:
speech_path = inputs.get("primary_audio")
music_path = inputs.get("secondary_audio")
# If duck_level (dB) is provided, convert to linear ratio for
# music_volume_during_speech. e.g. -12 dB -> 10^(-12/20) ~ 0.25
if "duck_level" in inputs and "ducking" not in inputs:
import math
db = inputs["duck_level"]
ducking = dict(ducking) # copy so we don't mutate caller
ducking.setdefault(
"music_volume_during_speech",
round(math.pow(10, db / 20), 4),
)
# Advanced format: tracks array with role field
tracks = inputs.get("tracks", [])
if tracks and speech_path is None and music_path is None:
# Support both naming conventions: speech/music and primary/secondary
speech_tracks = [
t for t in tracks if t.get("role") in ("speech", "primary")
]
music_tracks = [
t for t in tracks if t.get("role") in ("music", "secondary")
]
if speech_tracks:
speech_path = speech_tracks[0]["path"]
if music_tracks:
music_path = music_tracks[0]["path"]
if not speech_path or not music_path:
return ToolResult(
success=False,
error=(
"Ducking requires a primary (speech) and secondary (music) track. "
"Provide either primary_audio/secondary_audio params, or a tracks "
"array with role='speech'/'primary' and role='music'/'secondary'."
),
)
# Use FFmpeg sidechaincompress for ducking
music_vol = ducking.get("music_volume_during_speech", 0.15)
attack = ducking.get("attack_ms", 200) / 1000
release = ducking.get("release_ms", 500) / 1000
# Sidechain compress: use speech as the key signal to duck music
filter_complex = (
f"[1:a]sidechaincompress="
f"threshold=0.02:ratio=9:attack={attack}:release={release}:"
f"level_sc=1:mix=0.9[ducked];"
f"[ducked]volume={music_vol * 3}[music_out];" # compensate sidechain level
f"[0:a][music_out]amix=inputs=2:duration=longest[out]"
)
cmd = [
"ffmpeg", "-y",
"-i", speech_path,
"-i", music_path,
"-filter_complex", filter_complex,
"-map", "[out]",
str(output_path),
]
self.run_command(cmd)
return ToolResult(
success=True,
data={
"operation": "duck",
"speech_track": speech_path,
"music_track": music_path,
"output": str(output_path),
},
artifacts=[str(output_path)],
)
def _extract(self, inputs: dict[str, Any]) -> ToolResult:
"""Extract audio from a video file."""
input_path = Path(inputs["input_path"])
if not input_path.exists():
return ToolResult(success=False, error=f"Input not found: {input_path}")
output_path = Path(
inputs.get("output_path", str(input_path.with_suffix(".wav")))
)
cmd = [
"ffmpeg", "-y",
"-i", str(input_path),
"-vn",
"-acodec", "pcm_s16le",
"-ar", "16000",
"-ac", "1",
str(output_path),
]
self.run_command(cmd)
return ToolResult(
success=True,
data={
"operation": "extract",
"input": str(input_path),
"output": str(output_path),
},
artifacts=[str(output_path)],
)
def _full_mix(self, inputs: dict[str, Any]) -> ToolResult:
"""One-call mix: layer narration tracks, add music with ducking, normalize.
This is the preferred operation for the compose-director skill.
It combines mix + duck + normalize in a single FFmpeg filter graph.
Input format:
{
"operation": "full_mix",
"tracks": [
{"path": "narration_s1.mp3", "role": "speech", "start_seconds": 0},
{"path": "narration_s2.mp3", "role": "speech", "start_seconds": 10.5},
{"path": "music.mp3", "role": "music", "volume": 0.3}
],
"ducking": {
"enabled": true,
"music_volume_during_speech": 0.15,
"attack_ms": 200,
"release_ms": 500
},
"normalize": true,
"output_path": "mixed_audio.wav"
}
"""
tracks = inputs.get("tracks", [])
if not tracks:
return ToolResult(success=False, error="No tracks provided for full_mix")
output_path = Path(inputs.get("output_path", "full_mix_output.wav"))
output_path.parent.mkdir(parents=True, exist_ok=True)
normalize = inputs.get("normalize", True)
ducking = inputs.get("ducking", {"enabled": True})
speech_tracks = [t for t in tracks if t.get("role") in ("speech", "primary")]
music_tracks = [t for t in tracks if t.get("role") in ("music", "secondary")]
sfx_tracks = [t for t in tracks if t.get("role") == "sfx"]
all_tracks = speech_tracks + music_tracks + sfx_tracks
if not all_tracks:
return ToolResult(success=False, error="No valid tracks (need speech/music/sfx roles)")
# Validate all files exist
for t in all_tracks:
if not Path(t["path"]).exists():
return ToolResult(success=False, error=f"Track not found: {t['path']}")
# Build FFmpeg inputs and filter graph
input_args = []
filter_parts = []
for i, track in enumerate(all_tracks):
input_args.extend(["-i", track["path"]])
volume = track.get("volume", 1.0)
delay_ms = int(track.get("start_seconds", 0) * 1000)
fade_in = track.get("fade_in_seconds", 0)
fade_out = track.get("fade_out_seconds", 0)
filters = []
if volume != 1.0:
filters.append(f"volume={volume}")
if delay_ms > 0:
filters.append(f"adelay={delay_ms}|{delay_ms}")
if fade_in > 0:
filters.append(f"afade=t=in:d={fade_in}")
if fade_out > 0:
filters.append(f"afade=t=out:d={fade_out}")
if filters:
filter_chain = ",".join(filters)
filter_parts.append(f"[{i}:a]{filter_chain}[a{i}]")
else:
filter_parts.append(f"[{i}:a]acopy[a{i}]")
# If ducking is enabled and we have both speech and music, apply sidechain
duck_enabled = ducking.get("enabled", True) if isinstance(ducking, dict) else bool(ducking)
if duck_enabled and speech_tracks and music_tracks:
# Mix speech tracks together first
speech_indices = list(range(len(speech_tracks)))
speech_labels = "".join(f"[a{i}]" for i in speech_indices)
if len(speech_tracks) > 1:
filter_parts.append(
f"{speech_labels}amix=inputs={len(speech_tracks)}:duration=longest[speech_mix]"
)
speech_out = "[speech_mix]"
else:
speech_out = f"[a{speech_indices[0]}]"
# Mix music tracks together
music_start = len(speech_tracks)
music_indices = list(range(music_start, music_start + len(music_tracks)))
music_labels = "".join(f"[a{i}]" for i in music_indices)
if len(music_tracks) > 1:
filter_parts.append(
f"{music_labels}amix=inputs={len(music_tracks)}:duration=longest[music_mix]"
)
music_in = "[music_mix]"
else:
music_in = f"[a{music_indices[0]}]"
# Apply sidechain ducking
duck_params = ducking if isinstance(ducking, dict) else {}
attack = duck_params.get("attack_ms", 200) / 1000
release = duck_params.get("release_ms", 500) / 1000
music_vol = duck_params.get("music_volume_during_speech", 0.15)
filter_parts.append(
f"{music_in}{speech_out}sidechaincompress="
f"threshold=0.02:ratio=9:attack={attack}:release={release}:"
f"level_sc=1:mix=0.9[ducked_music];"
f"[ducked_music]volume={music_vol * 3}[music_out]"
)
# Duplicate speech for final mix (sidechain consumes it as key)
filter_parts.append(
f"{speech_out}acopy[speech_dup]" if speech_out.startswith("[a") else ""
)
# Re-mix speech path: we need speech audio in the output too
# Simpler approach: use amix on original speech and ducked music
# Reset: use a cleaner approach — amerge the speech mix and ducked music
# Actually, let's rebuild. The sidechain approach above uses speech as
# the key signal but doesn't consume it from the output chain.
# FFmpeg sidechaincompress: input 0 = audio to compress, input 1 = key signal
# So music is compressed, speech signal is the key. We need to mix them.
# Remove the last filter_part (the acopy that may be empty)
if filter_parts and filter_parts[-1] == "":
filter_parts.pop()
# Build speech mix for output separately
if len(speech_tracks) > 1:
# speech_mix already exists, make a copy for output
filter_parts.append(f"{speech_labels}amix=inputs={len(speech_tracks)}:duration=longest[speech_out]")
else:
filter_parts.append(f"[a{speech_indices[0]}]acopy[speech_out]")
# Final mix: speech_out + music_out
mix_label = "[speech_out][music_out]amix=inputs=2:duration=longest[premix]"
# Add SFX if present
sfx_start = len(speech_tracks) + len(music_tracks)
if sfx_tracks:
sfx_labels = "".join(f"[a{i}]" for i in range(sfx_start, sfx_start + len(sfx_tracks)))
filter_parts.append(mix_label.replace("[premix]", "[pressfx]"))
filter_parts.append(
f"[pressfx]{sfx_labels}amix=inputs={1 + len(sfx_tracks)}:duration=longest[premix]"
)
else:
filter_parts.append(mix_label)
else:
# No ducking: simple amix of all tracks
all_labels = "".join(f"[a{i}]" for i in range(len(all_tracks)))
filter_parts.append(
f"{all_labels}amix=inputs={len(all_tracks)}:duration=longest:dropout_transition=2[premix]"
)
# Normalize
if normalize:
filter_parts.append("[premix]loudnorm=I=-16:LRA=11:TP=-1.5[out]")
out_label = "[out]"
else:
out_label = "[premix]"
filter_complex = ";".join(p for p in filter_parts if p)
cmd = ["ffmpeg", "-y"]
cmd.extend(input_args)
cmd.extend(["-filter_complex", filter_complex])
cmd.extend(["-map", out_label, str(output_path)])
self.run_command(cmd)
return ToolResult(
success=True,
data={
"operation": "full_mix",
"speech_tracks": len(speech_tracks),
"music_tracks": len(music_tracks),
"sfx_tracks": len(sfx_tracks),
"ducking_enabled": duck_enabled,
"normalized": normalize,
"output": str(output_path),
},
artifacts=[str(output_path)],
)
+187
View File
@@ -0,0 +1,187 @@
"""ElevenLabs text-to-speech provider tool."""
from __future__ import annotations
import os
import time
from pathlib import Path
from typing import Any
from tools.base_tool import (
BaseTool,
Determinism,
ExecutionMode,
ResourceProfile,
RetryPolicy,
ToolResult,
ToolRuntime,
ToolStability,
ToolStatus,
ToolTier,
)
class ElevenLabsTTS(BaseTool):
name = "elevenlabs_tts"
version = "0.1.0"
tier = ToolTier.VOICE
capability = "tts"
provider = "elevenlabs"
stability = ToolStability.EXPERIMENTAL
execution_mode = ExecutionMode.SYNC
determinism = Determinism.STOCHASTIC
runtime = ToolRuntime.API
dependencies = []
install_instructions = (
"Set the ELEVENLABS_API_KEY environment variable:\n"
" export ELEVENLABS_API_KEY=your_key_here\n"
"Get a key at https://elevenlabs.io"
)
fallback = "openai_tts"
fallback_tools = ["openai_tts", "piper_tts"]
agent_skills = ["elevenlabs", "text-to-speech"]
capabilities = [
"text_to_speech",
"voice_selection",
"ssml_support",
"pronunciation_control",
]
supports = {
"voice_cloning": True,
"multilingual": True,
"offline": False,
"native_audio": True,
}
best_for = [
"high-quality narration",
"voice-sensitive spokesperson videos",
"multilingual spoken delivery",
]
not_good_for = [
"fully offline production",
"privacy-constrained local-only workflows",
]
input_schema = {
"type": "object",
"required": ["text"],
"properties": {
"text": {"type": "string", "description": "Text to convert to speech"},
"voice_id": {
"type": "string",
"description": "ElevenLabs voice ID (default: Rachel)",
},
"model_id": {
"type": "string",
"default": "eleven_multilingual_v2",
"description": "TTS model to use",
},
"stability": {
"type": "number",
"default": 0.5,
"minimum": 0,
"maximum": 1,
},
"similarity_boost": {
"type": "number",
"default": 0.75,
"minimum": 0,
"maximum": 1,
},
"style": {
"type": "number",
"default": 0.0,
"minimum": 0,
"maximum": 1,
},
"output_path": {"type": "string"},
"output_format": {
"type": "string",
"default": "mp3_44100_128",
"enum": ["mp3_44100_128", "mp3_44100_192", "pcm_16000", "pcm_24000"],
},
},
}
resource_profile = ResourceProfile(
cpu_cores=1, ram_mb=256, vram_mb=0, disk_mb=50, network_required=True
)
retry_policy = RetryPolicy(max_retries=2, retryable_errors=["rate_limit", "timeout"])
idempotency_key_fields = ["text", "voice_id", "model_id"]
side_effects = ["writes audio file to output_path", "calls ElevenLabs API"]
user_visible_verification = ["Listen to generated audio for natural speech quality"]
DEFAULT_VOICE_ID = "21m00Tcm4TlvDq8ikWAM"
def get_status(self) -> ToolStatus:
if os.environ.get("ELEVENLABS_API_KEY"):
return ToolStatus.AVAILABLE
return ToolStatus.UNAVAILABLE
def estimate_cost(self, inputs: dict[str, Any]) -> float:
return round(len(inputs.get("text", "")) * 0.0003, 4)
def execute(self, inputs: dict[str, Any]) -> ToolResult:
api_key = os.environ.get("ELEVENLABS_API_KEY")
if not api_key:
return ToolResult(success=False, error="No ElevenLabs API key. " + self.install_instructions)
start = time.time()
try:
result = self._generate(inputs, api_key)
except Exception as exc:
return ToolResult(success=False, error=f"TTS generation failed: {exc}")
result.duration_seconds = round(time.time() - start, 2)
result.cost_usd = self.estimate_cost(inputs)
return result
def _generate(self, inputs: dict[str, Any], api_key: str) -> ToolResult:
import requests
text = inputs["text"]
voice_id = inputs.get("voice_id", self.DEFAULT_VOICE_ID)
model_id = inputs.get("model_id", "eleven_multilingual_v2")
output_format = inputs.get("output_format", "mp3_44100_128")
response = requests.post(
f"https://api.elevenlabs.io/v1/text-to-speech/{voice_id}",
headers={
"xi-api-key": api_key,
"Content-Type": "application/json",
"Accept": "audio/mpeg",
},
json={
"text": text,
"model_id": model_id,
"voice_settings": {
"stability": inputs.get("stability", 0.5),
"similarity_boost": inputs.get("similarity_boost", 0.75),
"style": inputs.get("style", 0.0),
},
},
params={"output_format": output_format},
timeout=120,
)
response.raise_for_status()
ext = "mp3" if "mp3" in output_format else "wav"
output_path = Path(inputs.get("output_path", f"tts_output.{ext}"))
output_path.parent.mkdir(parents=True, exist_ok=True)
output_path.write_bytes(response.content)
return ToolResult(
success=True,
data={
"provider": self.provider,
"model": model_id,
"voice_id": voice_id,
"text_length": len(text),
"output": str(output_path),
"format": output_format,
},
artifacts=[str(output_path)],
model=model_id,
)
+149
View File
@@ -0,0 +1,149 @@
"""Music generation tool via ElevenLabs Music API.
Generates background music and sound effects for video production.
Reports unavailable when no API key is configured.
"""
from __future__ import annotations
import os
import time
from pathlib import Path
from typing import Any
from tools.base_tool import (
BaseTool,
Determinism,
ExecutionMode,
ResourceProfile,
RetryPolicy,
ToolResult,
ToolRuntime,
ToolStability,
ToolStatus,
ToolTier,
)
class MusicGen(BaseTool):
name = "music_gen"
version = "0.1.0"
tier = ToolTier.GENERATE
capability = "music_generation"
provider = "elevenlabs"
stability = ToolStability.EXPERIMENTAL
execution_mode = ExecutionMode.SYNC
determinism = Determinism.STOCHASTIC
runtime = ToolRuntime.API
dependencies = [] # checked dynamically via API key
install_instructions = (
"Set the ELEVENLABS_API_KEY environment variable:\n"
" export ELEVENLABS_API_KEY=your_key_here\n"
"Get a key at https://elevenlabs.io"
)
agent_skills = ["music", "sound-effects", "elevenlabs"]
capabilities = [
"generate_background_music",
"generate_sfx",
]
input_schema = {
"type": "object",
"required": ["prompt"],
"properties": {
"prompt": {
"type": "string",
"description": "Music description (mood, genre, instruments, tempo)",
},
"duration_seconds": {
"type": "number",
"default": 60,
"minimum": 3,
"maximum": 600,
"description": "Target duration in seconds (API supports 3-600s)",
},
"output_path": {"type": "string"},
},
}
resource_profile = ResourceProfile(
cpu_cores=1, ram_mb=256, vram_mb=0, disk_mb=50, network_required=True
)
retry_policy = RetryPolicy(max_retries=2, retryable_errors=["rate_limit", "timeout"])
idempotency_key_fields = ["prompt", "duration_seconds"]
side_effects = ["writes audio file to output_path", "calls ElevenLabs API"]
user_visible_verification = [
"Listen to generated music for mood and quality",
]
def get_status(self) -> ToolStatus:
if os.environ.get("ELEVENLABS_API_KEY"):
return ToolStatus.AVAILABLE
return ToolStatus.UNAVAILABLE
def estimate_cost(self, inputs: dict[str, Any]) -> float:
# ElevenLabs music generation pricing is per generation
duration = inputs.get("duration_seconds", 60)
# Approximate: ~$0.05 per 30 seconds
return round(duration / 30 * 0.05, 4)
def execute(self, inputs: dict[str, Any]) -> ToolResult:
api_key = os.environ.get("ELEVENLABS_API_KEY")
if not api_key:
return ToolResult(
success=False,
error="No ElevenLabs API key. " + self.install_instructions,
)
start = time.time()
try:
result = self._generate(inputs, api_key)
except Exception as e:
return ToolResult(success=False, error=f"Music generation failed: {e}")
result.duration_seconds = round(time.time() - start, 2)
result.cost_usd = self.estimate_cost(inputs)
return result
def _generate(self, inputs: dict[str, Any], api_key: str) -> ToolResult:
import requests
prompt = inputs["prompt"]
duration = inputs.get("duration_seconds", 60)
url = "https://api.elevenlabs.io/v1/music"
headers = {
"xi-api-key": api_key,
"Content-Type": "application/json",
}
payload = {
"prompt": prompt,
"music_length_ms": int(duration * 1000),
}
response = requests.post(
url, headers=headers, json=payload, timeout=180
)
response.raise_for_status()
output_path = Path(inputs.get("output_path", "music_output.mp3"))
output_path.parent.mkdir(parents=True, exist_ok=True)
output_path.write_bytes(response.content)
return ToolResult(
success=True,
data={
"provider": "elevenlabs",
"prompt": prompt,
"duration_seconds": duration,
"output": str(output_path),
"format": "mp3",
},
artifacts=[str(output_path)],
)
+154
View File
@@ -0,0 +1,154 @@
"""OpenAI text-to-speech provider tool."""
from __future__ import annotations
import os
import time
from pathlib import Path
from typing import Any
from tools.base_tool import (
BaseTool,
Determinism,
ExecutionMode,
ResourceProfile,
RetryPolicy,
ToolResult,
ToolRuntime,
ToolStability,
ToolStatus,
ToolTier,
)
class OpenAITTS(BaseTool):
name = "openai_tts"
version = "0.1.0"
tier = ToolTier.VOICE
capability = "tts"
provider = "openai"
stability = ToolStability.EXPERIMENTAL
execution_mode = ExecutionMode.SYNC
determinism = Determinism.STOCHASTIC
runtime = ToolRuntime.API
dependencies = []
install_instructions = (
"Set the OPENAI_API_KEY environment variable:\n"
" export OPENAI_API_KEY=your_key_here\n"
"Get a key at https://platform.openai.com/"
)
fallback = "piper_tts"
fallback_tools = ["piper_tts"]
agent_skills = ["openai-docs"]
capabilities = [
"text_to_speech",
"voice_selection",
]
supports = {
"voice_cloning": False,
"multilingual": True,
"offline": False,
"native_audio": True,
}
best_for = [
"general narration fallback",
"API-based production when ElevenLabs is unavailable",
]
not_good_for = [
"voice clone matching",
"fully offline production",
]
input_schema = {
"type": "object",
"required": ["text"],
"properties": {
"text": {"type": "string"},
"voice": {
"type": "string",
"default": "alloy",
"description": "OpenAI voice name",
},
"model": {
"type": "string",
"default": "gpt-4o-mini-tts",
"description": "OpenAI speech model",
},
"format": {
"type": "string",
"default": "mp3",
"enum": ["mp3", "wav", "pcm"],
},
"instructions": {
"type": "string",
"description": "Optional delivery instructions for the voice",
},
"output_path": {"type": "string"},
},
}
resource_profile = ResourceProfile(
cpu_cores=1, ram_mb=256, vram_mb=0, disk_mb=50, network_required=True
)
retry_policy = RetryPolicy(max_retries=2, retryable_errors=["rate_limit", "timeout"])
idempotency_key_fields = ["text", "voice", "model", "format"]
side_effects = ["writes audio file to output_path", "calls OpenAI API"]
user_visible_verification = ["Listen to generated audio for intelligibility and tone"]
def get_status(self) -> ToolStatus:
if os.environ.get("OPENAI_API_KEY"):
return ToolStatus.AVAILABLE
return ToolStatus.UNAVAILABLE
def estimate_cost(self, inputs: dict[str, Any]) -> float:
return round(len(inputs.get("text", "")) * 0.000015, 4)
def execute(self, inputs: dict[str, Any]) -> ToolResult:
if not os.environ.get("OPENAI_API_KEY"):
return ToolResult(success=False, error="No OpenAI API key. " + self.install_instructions)
start = time.time()
try:
result = self._generate(inputs)
except Exception as exc:
return ToolResult(success=False, error=f"OpenAI TTS failed: {exc}")
result.duration_seconds = round(time.time() - start, 2)
result.cost_usd = self.estimate_cost(inputs)
return result
def _generate(self, inputs: dict[str, Any]) -> ToolResult:
from openai import OpenAI
client = OpenAI()
text = inputs["text"]
model = inputs.get("model", "gpt-4o-mini-tts")
voice = inputs.get("voice", "alloy")
fmt = inputs.get("format", "mp3")
output_path = Path(inputs.get("output_path", f"openai_tts.{fmt}"))
output_path.parent.mkdir(parents=True, exist_ok=True)
with client.audio.speech.with_streaming_response.create(
model=model,
voice=voice,
input=text,
response_format=fmt,
instructions=inputs.get("instructions"),
) as response:
response.stream_to_file(output_path)
return ToolResult(
success=True,
data={
"provider": self.provider,
"model": model,
"voice": voice,
"format": fmt,
"text_length": len(text),
"output": str(output_path),
},
artifacts=[str(output_path)],
model=model,
)
+159
View File
@@ -0,0 +1,159 @@
"""Piper local text-to-speech provider tool."""
from __future__ import annotations
import shutil
import subprocess
import time
from pathlib import Path
from typing import Any
from tools.base_tool import (
BaseTool,
Determinism,
ExecutionMode,
ResourceProfile,
RetryPolicy,
ToolResult,
ToolRuntime,
ToolStability,
ToolStatus,
ToolTier,
)
class PiperTTS(BaseTool):
name = "piper_tts"
version = "0.1.0"
tier = ToolTier.VOICE
capability = "tts"
provider = "piper"
stability = ToolStability.EXPERIMENTAL
execution_mode = ExecutionMode.SYNC
determinism = Determinism.DETERMINISTIC
runtime = ToolRuntime.LOCAL
dependencies = ["cmd:piper"]
install_instructions = (
"Install Piper TTS:\n"
" pip install piper-tts\n"
"Or download from https://github.com/rhasspy/piper/releases\n"
"Then download a voice model:\n"
" piper --download-dir ~/.piper/models --model en_US-lessac-medium"
)
agent_skills = ["text-to-speech"]
capabilities = [
"text_to_speech",
"offline_generation",
]
supports = {
"voice_cloning": False,
"multilingual": False,
"offline": True,
"native_audio": True,
}
best_for = [
"offline narration fallback",
"privacy-sensitive local-only workflows",
]
not_good_for = [
"best-in-class expressive voice quality",
"voice clone matching",
]
input_schema = {
"type": "object",
"required": ["text"],
"properties": {
"text": {"type": "string"},
"model": {
"type": "string",
"default": "en_US-lessac-medium",
},
"speaker_id": {
"type": "integer",
"default": 0,
},
"length_scale": {
"type": "number",
"default": 1.0,
},
"sentence_silence": {
"type": "number",
"default": 0.3,
},
"output_path": {"type": "string"},
},
}
resource_profile = ResourceProfile(
cpu_cores=2, ram_mb=512, vram_mb=0, disk_mb=200, network_required=False
)
retry_policy = RetryPolicy(max_retries=1, retryable_errors=[])
idempotency_key_fields = ["text", "model", "speaker_id", "length_scale"]
side_effects = ["writes audio file to output_path"]
user_visible_verification = ["Listen to generated audio for intelligibility"]
def get_status(self) -> ToolStatus:
if shutil.which("piper"):
return ToolStatus.AVAILABLE
try:
import piper # noqa: F401
return ToolStatus.AVAILABLE
except ImportError:
return ToolStatus.UNAVAILABLE
def estimate_cost(self, inputs: dict[str, Any]) -> float:
return 0.0
def execute(self, inputs: dict[str, Any]) -> ToolResult:
if self.get_status() != ToolStatus.AVAILABLE:
return ToolResult(success=False, error="Piper TTS not available. " + self.install_instructions)
start = time.time()
try:
result = self._generate(inputs)
except Exception as exc:
return ToolResult(success=False, error=f"Local TTS generation failed: {exc}")
result.duration_seconds = round(time.time() - start, 2)
return result
def _generate(self, inputs: dict[str, Any]) -> ToolResult:
output_path = Path(inputs.get("output_path", "tts_output.wav"))
output_path.parent.mkdir(parents=True, exist_ok=True)
proc = subprocess.run(
[
"piper",
"--model", inputs.get("model", "en_US-lessac-medium"),
"--speaker", str(inputs.get("speaker_id", 0)),
"--length-scale", str(inputs.get("length_scale", 1.0)),
"--sentence-silence", str(inputs.get("sentence_silence", 0.3)),
"--output_file", str(output_path),
],
input=inputs["text"],
capture_output=True,
text=True,
timeout=300,
)
if proc.returncode != 0:
return ToolResult(success=False, error=f"Piper failed (exit {proc.returncode}): {proc.stderr}")
if not output_path.exists():
return ToolResult(success=False, error=f"Piper output file missing: {output_path}")
return ToolResult(
success=True,
data={
"provider": self.provider,
"model": inputs.get("model", "en_US-lessac-medium"),
"speaker_id": inputs.get("speaker_id", 0),
"text_length": len(inputs["text"]),
"output": str(output_path),
"format": "wav",
},
artifacts=[str(output_path)],
model=inputs.get("model", "en_US-lessac-medium"),
)
+288
View File
@@ -0,0 +1,288 @@
"""Suno AI music generation via sunoapi.org REST API.
Generates full songs, instrumentals, and background music. Async flow:
submit a generation request, poll for completion, download the audio file.
Each request produces 2 tracks; the tool returns the first by default.
"""
from __future__ import annotations
import os
import time
from pathlib import Path
from typing import Any
from tools.base_tool import (
BaseTool,
Determinism,
ExecutionMode,
ResourceProfile,
RetryPolicy,
ToolResult,
ToolRuntime,
ToolStability,
ToolStatus,
ToolTier,
)
class SunoMusic(BaseTool):
name = "suno_music"
version = "0.1.0"
tier = ToolTier.GENERATE
capability = "music_generation"
provider = "suno"
stability = ToolStability.BETA
execution_mode = ExecutionMode.ASYNC
determinism = Determinism.STOCHASTIC
runtime = ToolRuntime.API
dependencies = [] # checked dynamically via env var
install_instructions = (
"Set the SUNO_API_KEY environment variable:\n"
" export SUNO_API_KEY=your_key_here\n"
"Get a key at https://sunoapi.org/api-key"
)
agent_skills = ["music"]
capabilities = [
"generate_background_music",
"generate_song",
"generate_instrumental",
]
supports = {
"instrumental": True,
"vocals": True,
"custom_lyrics": True,
"style_control": True,
"long_form": True,
}
best_for = [
"full song generation with vocals and lyrics",
"high-quality instrumental background music",
"genre-specific music (any genre)",
"longer tracks up to 8 minutes",
]
not_good_for = [
"sound effects (use ElevenLabs SFX instead)",
"sub-10-second stingers (minimum ~30s generation)",
"offline generation",
]
fallback_tools = ["music_gen"]
input_schema = {
"type": "object",
"required": ["prompt"],
"properties": {
"prompt": {
"type": "string",
"description": (
"In simple mode: a description of desired music (max 500 chars). "
"In custom mode: the exact lyrics to sing (max 3000 chars)."
),
},
"style": {
"type": "string",
"description": "Genre/style description, e.g. 'upbeat electronic pop'. Used in custom mode only (max 200 chars).",
},
"title": {
"type": "string",
"description": "Song title. Used in custom mode only (max 80 chars).",
},
"instrumental": {
"type": "boolean",
"default": True,
"description": "True for instrumental only (no vocals), false for vocals.",
},
"custom_mode": {
"type": "boolean",
"default": False,
"description": "False = simple mode (prompt is a description, lyrics auto-generated). True = custom mode (prompt is exact lyrics, style/title required).",
},
"model": {
"type": "string",
"enum": ["V4", "V4_5", "V5"],
"default": "V4",
"description": "Suno model version. V4 = 4min max, V4_5/V5 = 8min max.",
},
"output_path": {"type": "string"},
"track_index": {
"type": "integer",
"default": 0,
"enum": [0, 1],
"description": "Which of the 2 generated tracks to return (0 or 1).",
},
},
}
resource_profile = ResourceProfile(
cpu_cores=1, ram_mb=256, vram_mb=0, disk_mb=100, network_required=True
)
retry_policy = RetryPolicy(max_retries=2, retryable_errors=["rate_limit", "timeout"])
idempotency_key_fields = ["prompt", "style", "instrumental", "model"]
side_effects = ["writes audio file to output_path", "calls Suno API via sunoapi.org"]
user_visible_verification = [
"Listen to generated music for mood, genre accuracy, and quality",
]
_BASE_URL = "https://api.sunoapi.org/api/v1"
_POLL_INTERVAL = 30 # seconds between status checks
_MAX_WAIT = 300 # 5 minutes max wait
def _get_api_key(self) -> str | None:
return os.environ.get("SUNO_API_KEY")
def get_status(self) -> ToolStatus:
if self._get_api_key():
return ToolStatus.AVAILABLE
return ToolStatus.UNAVAILABLE
def estimate_cost(self, inputs: dict[str, Any]) -> float:
# Suno credits cost $0.005 each; a generation is roughly 10 credits
return 0.05
def execute(self, inputs: dict[str, Any]) -> ToolResult:
api_key = self._get_api_key()
if not api_key:
return ToolResult(
success=False,
error="No Suno API key. " + self.install_instructions,
)
start = time.time()
try:
# Step 1: Submit generation request
task_id = self._submit(inputs, api_key)
# Step 2: Poll for completion
result_data = self._poll(task_id, api_key)
# Step 3: Download audio
track_index = inputs.get("track_index", 0)
tracks = result_data.get("data", [])
if not tracks:
return ToolResult(success=False, error="Suno returned no tracks.")
track = tracks[min(track_index, len(tracks) - 1)]
audio_url = track.get("audio_url")
if not audio_url:
return ToolResult(success=False, error="No audio_url in Suno response.")
output_path = self._download(audio_url, inputs, api_key)
except Exception as e:
return ToolResult(success=False, error=f"Suno generation failed: {e}")
duration = round(time.time() - start, 2)
return ToolResult(
success=True,
data={
"provider": "suno",
"model": inputs.get("model", "V4"),
"prompt": inputs["prompt"],
"style": inputs.get("style"),
"title": track.get("title", inputs.get("title")),
"instrumental": inputs.get("instrumental", True),
"duration_seconds": track.get("duration"),
"output": str(output_path),
"format": "mp3",
"track_id": track.get("id"),
"tracks_generated": len(tracks),
},
artifacts=[str(output_path)],
cost_usd=self.estimate_cost(inputs),
duration_seconds=duration,
model=f"suno/{inputs.get('model', 'V4')}",
)
def _submit(self, inputs: dict[str, Any], api_key: str) -> str:
"""Submit a generation request and return the taskId."""
import requests
custom_mode = inputs.get("custom_mode", False)
instrumental = inputs.get("instrumental", True)
model = inputs.get("model", "V4")
payload: dict[str, Any] = {
"model": model,
"customMode": custom_mode,
"instrumental": instrumental,
"callBackUrl": "", # no webhook; we poll
}
if custom_mode:
payload["prompt"] = inputs["prompt"] # exact lyrics
payload["style"] = inputs.get("style", "")
payload["title"] = inputs.get("title", "")
else:
payload["prompt"] = inputs["prompt"][:500] # description, max 500 chars
response = requests.post(
f"{self._BASE_URL}/generate",
headers={
"Authorization": f"Bearer {api_key}",
"Content-Type": "application/json",
},
json=payload,
timeout=30,
)
response.raise_for_status()
data = response.json()
task_id = data.get("data", {}).get("taskId") or data.get("taskId")
if not task_id:
raise RuntimeError(f"No taskId in Suno response: {data}")
return task_id
def _poll(self, task_id: str, api_key: str) -> dict:
"""Poll for task completion and return the result data."""
import requests
elapsed = 0
while elapsed < self._MAX_WAIT:
time.sleep(self._POLL_INTERVAL)
elapsed += self._POLL_INTERVAL
response = requests.get(
f"{self._BASE_URL}/generate/record-info",
params={"taskId": task_id},
headers={"Authorization": f"Bearer {api_key}"},
timeout=30,
)
response.raise_for_status()
result = response.json()
status = result.get("data", {}).get("status") or result.get("status", "")
if status == "SUCCESS":
return result.get("data", result)
elif status in (
"CREATE_TASK_FAILED",
"GENERATE_AUDIO_FAILED",
"SENSITIVE_WORD_ERROR",
):
raise RuntimeError(f"Suno generation failed with status: {status}")
# PENDING, GENERATING, TEXT_SUCCESS, FIRST_SUCCESS — keep polling
raise TimeoutError(
f"Suno generation timed out after {self._MAX_WAIT}s (taskId: {task_id})"
)
def _download(self, audio_url: str, inputs: dict[str, Any], api_key: str) -> Path:
"""Download the audio file to the output path."""
import requests
output_path = Path(inputs.get("output_path", "suno_output.mp3"))
output_path.parent.mkdir(parents=True, exist_ok=True)
response = requests.get(audio_url, timeout=120)
response.raise_for_status()
output_path.write_bytes(response.content)
return output_path
+136
View File
@@ -0,0 +1,136 @@
"""Capability-level text-to-speech selector that chooses among provider tools.
Provider discovery is automatic — any BaseTool with capability="tts"
is picked up from the registry. Adding a new TTS provider requires only creating
the tool file in tools/audio/; no changes to this selector are needed.
"""
from __future__ import annotations
from typing import Any
from tools.base_tool import BaseTool, ToolResult, ToolRuntime, ToolStability, ToolTier, ToolStatus
class TTSSelector(BaseTool):
name = "tts_selector"
version = "0.2.0"
tier = ToolTier.VOICE
capability = "tts"
provider = "selector"
stability = ToolStability.BETA
runtime = ToolRuntime.HYBRID
agent_skills = ["text-to-speech", "elevenlabs", "openai-docs"]
capabilities = [
"text_to_speech",
"provider_selection",
]
supports = {
"user_preference_routing": True,
"offline_fallback": True,
"multilingual": True,
}
best_for = [
"preflight tool selection",
"user-facing recommendation flows",
]
input_schema = {
"type": "object",
"required": ["text"],
"properties": {
"text": {"type": "string"},
"voice_id": {
"type": "string",
"description": "Provider-specific voice ID. Passed through to the selected TTS provider.",
},
"model_id": {
"type": "string",
"description": "TTS model to use (e.g. eleven_multilingual_v2). Passed through to provider.",
},
"stability": {
"type": "number", "minimum": 0, "maximum": 1,
"description": "Voice stability (ElevenLabs). Lower = more expressive.",
},
"similarity_boost": {
"type": "number", "minimum": 0, "maximum": 1,
"description": "Voice similarity boost (ElevenLabs).",
},
"style": {
"type": "number", "minimum": 0, "maximum": 1,
"description": "Style exaggeration (ElevenLabs). Higher = more expressive.",
},
"output_format": {
"type": "string",
"description": "Audio output format (e.g. mp3_44100_128). Passed through to provider.",
},
"preferred_provider": {
"type": "string",
"description": "Provider name or 'auto'. Valid values are discovered at runtime from the registry.",
"default": "auto",
},
"allowed_providers": {
"type": "array",
"items": {"type": "string"},
},
"output_path": {"type": "string"},
},
}
def _providers(self) -> list[BaseTool]:
"""Auto-discover TTS providers from the registry."""
from tools.tool_registry import registry
registry.ensure_discovered()
return [t for t in registry.get_by_capability("tts")
if t.name != self.name]
@property
def fallback_tools(self) -> list[str]:
"""Dynamically built from discovered providers."""
return [t.name for t in self._providers()]
@property
def provider_matrix(self) -> dict[str, dict[str, str]]:
"""Built at runtime from each provider's best_for field."""
matrix = {}
for tool in self._providers():
strength = ", ".join(tool.best_for) if tool.best_for else tool.name
matrix[tool.provider] = {"tool": tool.name, "strength": strength}
return matrix
def get_status(self) -> ToolStatus:
if any(tool.get_status() == ToolStatus.AVAILABLE for tool in self._providers()):
return ToolStatus.AVAILABLE
return ToolStatus.UNAVAILABLE
def estimate_cost(self, inputs: dict[str, Any]) -> float:
tool = self._select_tool(inputs)
return tool.estimate_cost(inputs) if tool else 0.0
def execute(self, inputs: dict[str, Any]) -> ToolResult:
tool = self._select_tool(inputs)
if tool is None:
return ToolResult(success=False, error="No TTS provider available.")
result = tool.execute(inputs)
if result.success:
result.data.setdefault("selected_tool", tool.name)
return result
def _select_tool(self, inputs: dict[str, Any]) -> BaseTool | None:
preferred = inputs.get("preferred_provider", "auto")
allowed = set(inputs.get("allowed_providers") or [])
candidates = self._providers()
if allowed:
candidates = [tool for tool in candidates if tool.provider in allowed]
if preferred != "auto":
ordered = [tool for tool in candidates if tool.provider == preferred]
ordered.extend([tool for tool in candidates if tool.provider != preferred])
else:
ordered = candidates
for tool in ordered:
if tool.get_status() == ToolStatus.AVAILABLE:
return tool
return None