freedak/OpenMontage
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

video-gen: prefer Seedance 2.0 on every gateway that supports it

Browse Source 
Expands the Seedance 2.0 surface from fal.ai-only to every gateway the
OpenMontage toolset can reach, so users on Replicate / Runway /
Higgsfield automatically route cinematic briefs through Seedance 2.0
instead of the gateway's prior default.

- tools/video/seedance_replicate (new): REPLICATE_API_TOKEN adapter for
  bytedance/seedance-2.0 and bytedance/seedance-2.0-fast. Same provider
  family ("seedance"), quality_score=0.95, premium capability flags.
- tools/video/runway_video: add seedance_2.0 / seedance_2.0_fast to the
  model enum as the new default, flag native_audio + multi_shot +
  camera_direction + lip_sync in supports, point agent_skills at
  seedance-2-0 Layer 3 skill. Note Runway Unlimited/Enterprise + non-US
  gating in the model description.
- tools/video/higgsfield_video: same pattern. seedance_2.0 is now the
  default model. Cost estimates calibrated to Higgsfield credit pricing
  (~$0.50-$1.20 per 5s).
- tools/video/_shared: annotate HEYGEN_PROVIDERS seedance_pro/lite as
  Seedance 1.x (2.0 on HeyGen flows through Video Agent / Avatar Shots
  which need a separate wrapper — tracked as follow-up).
- .agents/skills/seedance-2-0: update provider matrix to show which
  gateways are wrapped (fal.ai, Replicate, Runway, Higgsfield) vs. which
  are still roadmap (HeyGen Video Agent, BytePlus, Freepik, aggregators).

The scoring engine already deduplicates by provider="seedance" so
whichever gateway the user has configured wins automatically. Agents
passing preferred_provider="seedance" to video_selector route to the
best available Seedance 2.0 path without naming a specific gateway.
This commit is contained in:
calesthio
2026-04-17 22:04:08 -07:00
parent 16791a3a80
commit 9e17263aa5
5 changed files with 324 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/video/_shared.py
+6 -2
View File
@@ -23,8 +23,12 @@ HEYGEN_PROVIDERS = {
"sora_v2": {"name": "Sora v2", "quality": "high", "speed": "slow"},
"sora_v2_pro": {"name": "Sora v2 Pro", "quality": "highest", "speed": "slow"},
"runway_gen4": {"name": "Runway Gen-4", "quality": "high", "speed": "medium"},
"seedance_lite": {"name": "Seedance Lite", "quality": "medium", "speed": "fast"},
"seedance_pro": {"name": "Seedance Pro", "quality": "high", "speed": "medium"},
# NOTE: HeyGen's `seedance_lite` / `seedance_pro` provider strings map to
# Seedance 1.x. Seedance 2.0 on HeyGen is exposed through Video Agent and
# Avatar Shots endpoints, NOT via the workflow provider parameter. For 2.0
# access today, use `seedance_video` (fal.ai) or `seedance_replicate`.
"seedance_lite": {"name": "Seedance Lite (1.x)", "quality": "medium", "speed": "fast"},
"seedance_pro": {"name": "Seedance Pro (1.x)", "quality": "high", "speed": "medium"},
"ltx_distilled": {"name": "LTX Distilled", "quality": "low", "speed": "fastest"},
}
tools/video/higgsfield_video.py
+27 -11
View File
@@ -42,7 +42,7 @@ class HiggsFieldVideo(BaseTool):
" Get them at https://cloud.higgsfield.ai/api-keys\n"
" Alternatively, set HIGGSFIELD_KEY as a combined key:secret value."
)
agent_skills = ["ai-video-gen"]
agent_skills = ["seedance-2-0", "ai-video-gen"]
capabilities = ["text_to_video", "image_to_video"]
supports = {
@@ -50,14 +50,23 @@ class HiggsFieldVideo(BaseTool):
"image_to_video": True,
"character_consistency": True,
"multi_model_routing": True,
"native_audio": True,
"cinematic_quality": True,
"camera_direction": True,
"lip_sync": True,
"multi_shot": True,
}
best_for = [
"character-consistent video generation (Soul ID)",
"multi-model access through a single API",
"photorealistic and fashion-aware content",
"preferred premium video gen on Higgsfield (Seedance 2.0 is the default model)",
"cinematic trailers, teasers, and high-fidelity clips with native synchronized audio",
"character-consistent video generation (Soul ID + Seedance 2.0 identity consistency)",
"director-level camera control and multi-shot editing in a single generation",
"lip-sync from quoted dialogue in prompts",
"multi-model access through a single API (Seedance 2.0, Kling, Veo, Sora, WAN)",
]
not_good_for = ["offline generation", "fine-grained model control", "budget projects without subscription"]
fallback_tools = ["kling_video", "veo_video", "minimax_video"]
fallback_tools = ["seedance_video", "seedance_replicate", "kling_video", "veo_video", "minimax_video"]
quality_score = 0.9
input_schema = {
"type": "object",
@@ -72,14 +81,16 @@ class HiggsFieldVideo(BaseTool):
"model": {
"type": "string",
"enum": [
"seedance_2.0",
"seedance_2.0_fast",
"kling_3.0",
"veo_3.1",
"sora_2",
"wan_2.5",
"soul_cinema",
],
"default": "kling_3.0",
"description": "Underlying model to use for generation",
"default": "seedance_2.0",
"description": "Underlying model. Defaults to Seedance 2.0 (preferred premium) — see .agents/skills/seedance-2-0/",
},
"duration": {
"type": "string",
@@ -123,10 +134,13 @@ class HiggsFieldVideo(BaseTool):
return ToolStatus.UNAVAILABLE
def estimate_cost(self, inputs: dict[str, Any]) -> float:
model = inputs.get("model", "kling_3.0")
model = inputs.get("model", "seedance_2.0")
duration = int(inputs.get("duration", "5"))
# Approximate per-clip costs based on Higgsfield credit pricing
# Approximate per-clip costs based on Higgsfield credit pricing.
# Seedance 2.0 on Higgsfield runs ~50-80 credits per 5s clip ≈ $0.50-$1.20.
base_costs = {
"seedance_2.0": 0.80,
"seedance_2.0_fast": 0.50,
"kling_3.0": 0.10,
"wan_2.5": 0.10,
"veo_3.1": 0.50,
@@ -137,9 +151,11 @@ class HiggsFieldVideo(BaseTool):
return base * (duration / 5)
def estimate_runtime(self, inputs: dict[str, Any]) -> float:
model = inputs.get("model", "kling_3.0")
if model in ("veo_3.1", "sora_2"):
model = inputs.get("model", "seedance_2.0")
if model in ("veo_3.1", "sora_2", "seedance_2.0"):
return 120.0
if model == "seedance_2.0_fast":
return 60.0
return 60.0
def execute(self, inputs: dict[str, Any]) -> ToolResult:
tools/video/runway_video.py
+27 -7
View File
@@ -34,12 +34,17 @@ _COST_PER_SECOND = {
"gen3a_turbo": 0.05,
"gen4_turbo": 0.05,
"gen4_aleph": 0.15,
# Third-party Seedance 2.0 inside Runway (Enterprise/Unlimited, non-US).
"seedance_2.0": 0.30,
"seedance_2.0_fast": 0.24,
}
_RUNTIME_SECONDS = {
"gen3a_turbo": 25.0,
"gen4_turbo": 30.0,
"gen4_aleph": 60.0,
"seedance_2.0": 120.0,
"seedance_2.0_fast": 60.0,
}
@@ -59,21 +64,29 @@ class RunwayVideo(BaseTool):
"Set RUNWAY_API_KEY to your Runway API secret.\n"
" Get one at https://dev.runwayml.com/"
)
agent_skills = ["ai-video-gen"]
agent_skills = ["seedance-2-0", "ai-video-gen"]
capabilities = ["text_to_video", "image_to_video"]
supports = {
"text_to_video": True,
"image_to_video": True,
"professional_control": True,
"native_audio": True,
"cinematic_quality": True,
"camera_direction": True,
"lip_sync": True,
"multi_shot": True,
}
best_for = [
"highest overall video quality (#1 Elo rating)",
"preferred premium video gen on Runway when Seedance 2.0 model is selected",
"cinematic trailers, teasers, and high-fidelity clips with native synchronized audio (Seedance 2.0 path)",
"director-level camera control and multi-shot editing (Seedance 2.0) or Runway Gen-4 professional control",
"lip-sync from quoted dialogue in prompts (Seedance 2.0)",
"professional video production",
"precise control over generation",
]
not_good_for = ["budget projects", "offline generation", "very long clips"]
fallback_tools = ["kling_video", "veo_video", "minimax_video", "wan_video"]
fallback_tools = ["seedance_video", "seedance_replicate", "kling_video", "veo_video", "minimax_video", "wan_video"]
quality_score = 0.9
input_schema = {
"type": "object",
@@ -87,9 +100,16 @@ class RunwayVideo(BaseTool):
},
"model": {
"type": "string",
"enum": ["gen4_turbo", "gen4_aleph", "gen3a_turbo"],
"default": "gen4_turbo",
"description": "gen4_aleph is highest fidelity, gen4_turbo is balanced, gen3a_turbo is cheapest",
"enum": ["seedance_2.0", "seedance_2.0_fast", "gen4_turbo", "gen4_aleph", "gen3a_turbo"],
"default": "seedance_2.0",
"description": (
"seedance_2.0 = preferred premium default (single-pass synced audio, multi-shot, lip-sync — "
"Runway Unlimited/Enterprise plan, non-US only). "
"seedance_2.0_fast = lower-cost Seedance variant. "
"gen4_aleph = Runway's highest-fidelity native model. "
"gen4_turbo = balanced Runway native. "
"gen3a_turbo = cheapest Runway native."
),
},
"duration": {
"type": "integer",
tools/video/seedance_replicate.py
+253
View File
@@ -0,0 +1,253 @@
"""Seedance 2.0 (ByteDance) video generation via Replicate.
Replicate hosts ByteDance's published Seedance 2.0 models:
- bytedance/seedance-2.0 (standard)
- bytedance/seedance-2.0-fast (fast tier)
Same model family as the fal.ai path (tools/video/seedance_video.py) —
if you have both FAL_KEY and REPLICATE_API_TOKEN the scoring engine
deduplicates by provider=seedance and picks whichever registers first.
"""
from __future__ import annotations
import os
import time
from pathlib import Path
from typing import Any
from tools.base_tool import (
BaseTool,
Determinism,
ExecutionMode,
ResourceProfile,
RetryPolicy,
ToolResult,
ToolRuntime,
ToolStability,
ToolStatus,
ToolTier,
)
class SeedanceReplicate(BaseTool):
name = "seedance_replicate"
version = "0.1.0"
tier = ToolTier.GENERATE
capability = "video_generation"
provider = "seedance"
stability = ToolStability.BETA
execution_mode = ExecutionMode.SYNC
determinism = Determinism.STOCHASTIC
runtime = ToolRuntime.API
dependencies = []
install_instructions = (
"Set REPLICATE_API_TOKEN to your Replicate API token.\n"
" Get one at https://replicate.com/account/api-tokens"
)
agent_skills = ["seedance-2-0", "ai-video-gen"]
capabilities = ["text_to_video", "image_to_video"]
supports = {
"text_to_video": True,
"image_to_video": True,
"reference_image": True,
"native_audio": True,
"cinematic_quality": True,
"camera_direction": True,
"lip_sync": True,
"multi_shot": True,
"aspect_ratio": True,
"seed": True,
}
best_for = [
"preferred premium video gen when REPLICATE_API_TOKEN is available",
"cinematic trailers, teasers, and high-fidelity clips with native synchronized audio",
"director-level camera control and multi-shot editing in a single generation",
"lip-sync from quoted dialogue in prompts",
"consistent character identity across shots",
]
not_good_for = ["offline generation", "budget-constrained projects"]
fallback_tools = ["seedance_video", "veo_video", "kling_video", "minimax_video"]
quality_score = 0.95
input_schema = {
"type": "object",
"required": ["prompt"],
"properties": {
"prompt": {"type": "string"},
"operation": {
"type": "string",
"enum": ["text_to_video", "image_to_video"],
"default": "text_to_video",
},
"model_variant": {
"type": "string",
"enum": ["standard", "fast"],
"default": "standard",
"description": "standard = bytedance/seedance-2.0, fast = bytedance/seedance-2.0-fast",
},
"duration": {
"type": "string",
"enum": ["auto", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15"],
"default": "5",
},
"aspect_ratio": {
"type": "string",
"enum": ["auto", "21:9", "16:9", "4:3", "1:1", "3:4", "9:16"],
"default": "16:9",
},
"resolution": {
"type": "string",
"enum": ["480p", "720p"],
"default": "720p",
},
"generate_audio": {
"type": "boolean",
"default": True,
},
"image_url": {
"type": "string",
"description": "Start frame image URL for image_to_video",
},
"seed": {"type": "integer"},
"output_path": {"type": "string"},
},
}
resource_profile = ResourceProfile(
cpu_cores=1, ram_mb=512, vram_mb=0, disk_mb=500, network_required=True
)
retry_policy = RetryPolicy(max_retries=2, retryable_errors=["rate_limit", "timeout"])
idempotency_key_fields = ["prompt", "model_variant", "operation", "duration", "seed"]
side_effects = ["writes video file to output_path", "calls Replicate API"]
user_visible_verification = [
"Watch generated clip for motion coherence, audio sync, and visual quality"
]
def _get_api_token(self) -> str | None:
return os.environ.get("REPLICATE_API_TOKEN")
def get_status(self) -> ToolStatus:
return ToolStatus.AVAILABLE if self._get_api_token() else ToolStatus.UNAVAILABLE
def estimate_cost(self, inputs: dict[str, Any]) -> float:
variant = inputs.get("model_variant", "standard")
duration = inputs.get("duration", "5")
secs = 5 if duration == "auto" else int(duration)
# Replicate bills per-second at roughly the same rate as fal.ai for this model family.
rate = 0.24 if variant == "fast" else 0.30
return round(rate * secs, 2)
def estimate_runtime(self, inputs: dict[str, Any]) -> float:
return 60.0 if inputs.get("model_variant") == "fast" else 120.0
def execute(self, inputs: dict[str, Any]) -> ToolResult:
token = self._get_api_token()
if not token:
return ToolResult(
success=False,
error="REPLICATE_API_TOKEN not set. " + self.install_instructions,
)
import requests
start = time.time()
variant = inputs.get("model_variant", "standard")
model_slug = (
"bytedance/seedance-2.0-fast" if variant == "fast" else "bytedance/seedance-2.0"
)
payload_input: dict[str, Any] = {"prompt": inputs["prompt"]}
if inputs.get("duration") and inputs["duration"] != "auto":
payload_input["duration"] = int(inputs["duration"])
if inputs.get("aspect_ratio") and inputs["aspect_ratio"] != "auto":
payload_input["aspect_ratio"] = inputs["aspect_ratio"]
if inputs.get("resolution"):
payload_input["resolution"] = inputs["resolution"]
if "generate_audio" in inputs:
payload_input["generate_audio"] = inputs["generate_audio"]
if inputs.get("seed") is not None:
payload_input["seed"] = inputs["seed"]
if inputs.get("operation") == "image_to_video" and inputs.get("image_url"):
payload_input["image"] = inputs["image_url"]
headers = {
"Authorization": f"Bearer {token}",
"Content-Type": "application/json",
"Prefer": "wait=60",
}
try:
submit = requests.post(
f"https://api.replicate.com/v1/models/{model_slug}/predictions",
headers=headers,
json={"input": payload_input},
timeout=90,
)
submit.raise_for_status()
pred = submit.json()
# Poll until completed (Replicate may return the result synchronously
# when Prefer: wait is honored, but fall back to polling).
while pred.get("status") in ("starting", "processing"):
time.sleep(3)
get_url = pred.get("urls", {}).get("get")
if not get_url:
return ToolResult(success=False, error="Replicate response missing poll URL")
poll = requests.get(get_url, headers=headers, timeout=30)
poll.raise_for_status()
pred = poll.json()
status = pred.get("status")
if status != "succeeded":
return ToolResult(
success=False,
error=f"Replicate Seedance 2.0 generation {status}: {pred.get('error')}",
)
output = pred.get("output")
# Replicate returns either a string URL or a list.
video_url = output[0] if isinstance(output, list) else output
if not isinstance(video_url, str):
return ToolResult(success=False, error=f"Unexpected output shape from Replicate: {output!r}")
video_response = requests.get(video_url, timeout=180)
video_response.raise_for_status()
output_path = Path(inputs.get("output_path", "seedance_replicate_output.mp4"))
output_path.parent.mkdir(parents=True, exist_ok=True)
output_path.write_bytes(video_response.content)
except Exception as e:
return ToolResult(
success=False,
error=f"Replicate Seedance 2.0 generation failed: {e}",
)
from tools.video._shared import probe_output
probed = probe_output(output_path)
return ToolResult(
success=True,
data={
"provider": "seedance",
"gateway": "replicate",
"model": model_slug,
"prompt": inputs["prompt"],
"variant": variant,
"aspect_ratio": inputs.get("aspect_ratio", "16:9"),
"resolution": inputs.get("resolution", "720p"),
"generate_audio": inputs.get("generate_audio", True),
"seed": pred.get("input", {}).get("seed"),
"output": str(output_path),
"output_path": str(output_path),
"format": "mp4",
**probed,
},
artifacts=[str(output_path)],
cost_usd=self.estimate_cost(inputs),
duration_seconds=round(time.time() - start, 2),
model=model_slug,
)