Talking-head pipeline: 8 new tools, Remotion TalkingHead composition, and skill rewrites
New tools: face_tracker, visual_qa, eye_enhance, auto_reframe, remotion_caption_burn, showcase_card, silence_cutter. Updated audio_mixer with segmented_music operation and subtitle_gen with ASR corrections. Registered TalkingHead composition in Root.tsx. Rewrote compose/edit/scene director skills for full enhancement chain, Remotion captions, multi-clip assembly, and visual QA. Gitignore cleanup: exclude test demo-props, downloaded music, and generated images.
This commit is contained in:
calesthio
@@ -0,0 +1,537 @@
|
||||
"""Auto-reframe tool for aspect ratio conversion with face tracking.
|
||||
|
||||
Converts video between aspect ratios (e.g. 16:9 → 9:16 for Instagram Reels)
|
||||
while keeping the speaker's face centered in frame. Uses face_tracker data
|
||||
for smooth, content-aware cropping.
|
||||
|
||||
Primary use: converting talking-head footage shot in landscape to vertical
|
||||
format for social media (TikTok, Reels, Shorts).
|
||||
|
||||
Approach: MediaPipe/OpenCV face detection → smoothed bounding box trajectory
|
||||
→ FFmpeg crop filter. No GPU required.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from tools.base_tool import (
|
||||
BaseTool,
|
||||
Determinism,
|
||||
ExecutionMode,
|
||||
ResourceProfile,
|
||||
RetryPolicy,
|
||||
ResumeSupport,
|
||||
ToolResult,
|
||||
ToolStability,
|
||||
ToolTier,
|
||||
)
|
||||
|
||||
|
||||
# Common target aspect ratios
|
||||
ASPECT_PRESETS = {
|
||||
"portrait": (9, 16), # Instagram Reels, TikTok, YouTube Shorts
|
||||
"square": (1, 1), # Instagram Feed
|
||||
"landscape": (16, 9), # YouTube, LinkedIn
|
||||
"cinematic": (21, 9), # Ultra-wide
|
||||
"vertical_4_5": (4, 5), # Instagram portrait post
|
||||
}
|
||||
|
||||
|
||||
class AutoReframe(BaseTool):
|
||||
name = "auto_reframe"
|
||||
version = "0.1.0"
|
||||
tier = ToolTier.CORE
|
||||
capability = "video_post"
|
||||
provider = "ffmpeg"
|
||||
stability = ToolStability.EXPERIMENTAL
|
||||
execution_mode = ExecutionMode.SYNC
|
||||
determinism = Determinism.DETERMINISTIC
|
||||
|
||||
dependencies = ["cmd:ffmpeg"]
|
||||
install_instructions = (
|
||||
"FFmpeg is required. For face-tracked reframing, also install:\n"
|
||||
"pip install mediapipe opencv-python\n\n"
|
||||
"Without MediaPipe/OpenCV, falls back to center-crop."
|
||||
)
|
||||
agent_skills = ["ffmpeg"]
|
||||
|
||||
capabilities = [
|
||||
"aspect_ratio_conversion",
|
||||
"face_tracked_crop",
|
||||
"smart_reframe",
|
||||
"center_crop",
|
||||
]
|
||||
|
||||
input_schema = {
|
||||
"type": "object",
|
||||
"required": ["input_path"],
|
||||
"properties": {
|
||||
"input_path": {"type": "string"},
|
||||
"output_path": {"type": "string"},
|
||||
"target_aspect": {
|
||||
"type": "string",
|
||||
"enum": list(ASPECT_PRESETS.keys()),
|
||||
"default": "portrait",
|
||||
"description": "Target aspect ratio preset",
|
||||
},
|
||||
"target_width": {
|
||||
"type": "integer",
|
||||
"description": "Explicit target width (overrides preset)",
|
||||
},
|
||||
"target_height": {
|
||||
"type": "integer",
|
||||
"description": "Explicit target height (overrides preset)",
|
||||
},
|
||||
"face_tracking_json": {
|
||||
"type": "string",
|
||||
"description": "Path to pre-computed face_tracker JSON. If omitted, runs face detection internally.",
|
||||
},
|
||||
"smoothing_window": {
|
||||
"type": "integer",
|
||||
"default": 15,
|
||||
"minimum": 1,
|
||||
"description": "Number of frames for position smoothing (higher = smoother pan, lower = more responsive)",
|
||||
},
|
||||
"face_padding": {
|
||||
"type": "number",
|
||||
"default": 0.4,
|
||||
"minimum": 0.0,
|
||||
"maximum": 1.0,
|
||||
"description": "Extra space around face as fraction of face size (0.4 = 40% padding)",
|
||||
},
|
||||
"sample_fps": {
|
||||
"type": "number",
|
||||
"default": 5,
|
||||
"description": "Face detection sample rate (only used if no face_tracking_json)",
|
||||
},
|
||||
"codec": {"type": "string", "default": "libx264"},
|
||||
"crf": {"type": "integer", "default": 18},
|
||||
},
|
||||
}
|
||||
|
||||
resource_profile = ResourceProfile(
|
||||
cpu_cores=4, ram_mb=2048, vram_mb=0, disk_mb=4000, network_required=False
|
||||
)
|
||||
retry_policy = RetryPolicy(max_retries=1, retryable_errors=["FFmpeg error"])
|
||||
resume_support = ResumeSupport.FROM_START
|
||||
idempotency_key_fields = [
|
||||
"input_path", "target_aspect", "target_width", "target_height",
|
||||
"smoothing_window", "face_padding",
|
||||
]
|
||||
side_effects = ["writes reframed video to output_path"]
|
||||
user_visible_verification = [
|
||||
"Play reframed output — verify face stays centered and framing is smooth",
|
||||
"Check that no important content is cropped out",
|
||||
]
|
||||
|
||||
def execute(self, inputs: dict[str, Any]) -> ToolResult:
|
||||
input_path = Path(inputs["input_path"])
|
||||
if not input_path.exists():
|
||||
return ToolResult(success=False, error=f"Input not found: {input_path}")
|
||||
|
||||
start = time.time()
|
||||
|
||||
# Get source video dimensions
|
||||
src_w, src_h, src_fps = self._get_video_info(input_path)
|
||||
if src_w == 0 or src_h == 0:
|
||||
return ToolResult(success=False, error="Could not read video dimensions")
|
||||
|
||||
# Determine target crop dimensions (in source pixel space)
|
||||
target_w, target_h = self._compute_crop_size(inputs, src_w, src_h)
|
||||
|
||||
# If source already matches target aspect, no crop needed
|
||||
if target_w == src_w and target_h == src_h:
|
||||
return ToolResult(
|
||||
success=True,
|
||||
data={"message": "Source already matches target aspect ratio", "output": str(input_path)},
|
||||
artifacts=[str(input_path)],
|
||||
)
|
||||
|
||||
# Get face tracking data
|
||||
face_data = self._get_face_data(inputs, input_path, src_fps)
|
||||
|
||||
# Compute per-frame crop positions
|
||||
if face_data and len(face_data) > 0:
|
||||
crop_x, crop_y = self._compute_face_tracked_crop(
|
||||
face_data, src_w, src_h, target_w, target_h,
|
||||
src_fps,
|
||||
inputs.get("smoothing_window", 15),
|
||||
inputs.get("face_padding", 0.4),
|
||||
)
|
||||
method = "face_tracked"
|
||||
else:
|
||||
# Fallback: center crop
|
||||
crop_x = (src_w - target_w) // 2
|
||||
crop_y = (src_h - target_h) // 2
|
||||
method = "center_crop"
|
||||
|
||||
# Determine output resolution
|
||||
out_w, out_h = self._compute_output_resolution(inputs, target_w, target_h, src_w, src_h)
|
||||
|
||||
# Build output path
|
||||
aspect_name = inputs.get("target_aspect", "portrait")
|
||||
output_path = Path(
|
||||
inputs.get("output_path", str(input_path.with_stem(f"{input_path.stem}_{aspect_name}")))
|
||||
)
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Render via FFmpeg
|
||||
codec = inputs.get("codec", "libx264")
|
||||
crf = inputs.get("crf", 18)
|
||||
|
||||
if method == "face_tracked" and isinstance(crop_x, list):
|
||||
# Dynamic crop: write crop coordinates to a file and use sendcmd
|
||||
result = self._render_dynamic_crop(
|
||||
input_path, output_path, crop_x, crop_y,
|
||||
target_w, target_h, out_w, out_h,
|
||||
src_fps, codec, crf,
|
||||
)
|
||||
else:
|
||||
# Static crop
|
||||
result = self._render_static_crop(
|
||||
input_path, output_path,
|
||||
crop_x, crop_y, target_w, target_h,
|
||||
out_w, out_h, codec, crf,
|
||||
)
|
||||
|
||||
if not result.success:
|
||||
return result
|
||||
|
||||
elapsed = time.time() - start
|
||||
|
||||
return ToolResult(
|
||||
success=True,
|
||||
data={
|
||||
"input": str(input_path),
|
||||
"output": str(output_path),
|
||||
"source_resolution": f"{src_w}x{src_h}",
|
||||
"crop_resolution": f"{target_w}x{target_h}",
|
||||
"output_resolution": f"{out_w}x{out_h}",
|
||||
"method": method,
|
||||
"target_aspect": inputs.get("target_aspect", "portrait"),
|
||||
},
|
||||
artifacts=[str(output_path)],
|
||||
duration_seconds=round(elapsed, 2),
|
||||
)
|
||||
|
||||
def _get_video_info(self, path: Path) -> tuple[int, int, float]:
|
||||
"""Get video width, height, fps via ffprobe."""
|
||||
cmd = [
|
||||
"ffprobe", "-v", "quiet",
|
||||
"-select_streams", "v:0",
|
||||
"-show_entries", "stream=width,height,r_frame_rate",
|
||||
"-of", "json", str(path),
|
||||
]
|
||||
try:
|
||||
result = self.run_command(cmd)
|
||||
data = json.loads(result.stdout)
|
||||
stream = data["streams"][0]
|
||||
w = int(stream["width"])
|
||||
h = int(stream["height"])
|
||||
# Parse r_frame_rate (e.g. "30000/1001")
|
||||
fps_parts = stream["r_frame_rate"].split("/")
|
||||
fps = float(fps_parts[0]) / float(fps_parts[1]) if len(fps_parts) == 2 else float(fps_parts[0])
|
||||
return w, h, fps
|
||||
except Exception:
|
||||
return 0, 0, 30.0
|
||||
|
||||
def _compute_crop_size(
|
||||
self, inputs: dict[str, Any], src_w: int, src_h: int
|
||||
) -> tuple[int, int]:
|
||||
"""Compute crop dimensions in source pixel space that match the target aspect ratio."""
|
||||
if "target_width" in inputs and "target_height" in inputs:
|
||||
# Explicit dimensions — compute crop in source space matching this ratio
|
||||
tw, th = inputs["target_width"], inputs["target_height"]
|
||||
else:
|
||||
aspect_name = inputs.get("target_aspect", "portrait")
|
||||
tw, th = ASPECT_PRESETS.get(aspect_name, (9, 16))
|
||||
|
||||
target_ratio = tw / th
|
||||
src_ratio = src_w / src_h
|
||||
|
||||
if target_ratio > src_ratio:
|
||||
# Target is wider — crop height
|
||||
crop_w = src_w
|
||||
crop_h = int(src_w / target_ratio)
|
||||
else:
|
||||
# Target is taller/narrower — crop width
|
||||
crop_h = src_h
|
||||
crop_w = int(src_h * target_ratio)
|
||||
|
||||
# Ensure even dimensions (required by most codecs)
|
||||
crop_w = crop_w - (crop_w % 2)
|
||||
crop_h = crop_h - (crop_h % 2)
|
||||
|
||||
return crop_w, crop_h
|
||||
|
||||
def _compute_output_resolution(
|
||||
self, inputs: dict[str, Any],
|
||||
crop_w: int, crop_h: int,
|
||||
src_w: int, src_h: int,
|
||||
) -> tuple[int, int]:
|
||||
"""Determine final output resolution. Scales to standard sizes."""
|
||||
if "target_width" in inputs and "target_height" in inputs:
|
||||
out_w = inputs["target_width"]
|
||||
out_h = inputs["target_height"]
|
||||
else:
|
||||
aspect_name = inputs.get("target_aspect", "portrait")
|
||||
if aspect_name == "portrait":
|
||||
out_w, out_h = 1080, 1920
|
||||
elif aspect_name == "square":
|
||||
out_w, out_h = 1080, 1080
|
||||
elif aspect_name == "landscape":
|
||||
out_w, out_h = 1920, 1080
|
||||
elif aspect_name == "cinematic":
|
||||
out_w, out_h = 2560, 1080
|
||||
elif aspect_name == "vertical_4_5":
|
||||
out_w, out_h = 1080, 1350
|
||||
else:
|
||||
out_w, out_h = crop_w, crop_h
|
||||
|
||||
# Ensure even
|
||||
out_w = out_w - (out_w % 2)
|
||||
out_h = out_h - (out_h % 2)
|
||||
return out_w, out_h
|
||||
|
||||
def _get_face_data(
|
||||
self, inputs: dict[str, Any], input_path: Path, src_fps: float
|
||||
) -> list[dict]:
|
||||
"""Get face tracking data — from pre-computed JSON or by running detection."""
|
||||
# Check for pre-computed tracking data
|
||||
tracking_json = inputs.get("face_tracking_json")
|
||||
if tracking_json:
|
||||
p = Path(tracking_json)
|
||||
if p.exists():
|
||||
data = json.loads(p.read_text(encoding="utf-8"))
|
||||
return data.get("faces", [])
|
||||
|
||||
# Try to run face_tracker internally
|
||||
try:
|
||||
from tools.analysis.face_tracker import FaceTracker
|
||||
tracker = FaceTracker()
|
||||
if tracker.get_status().name == "UNAVAILABLE":
|
||||
return []
|
||||
sample_fps = inputs.get("sample_fps", 5)
|
||||
result = tracker.execute({
|
||||
"input_path": str(input_path),
|
||||
"sample_fps": sample_fps,
|
||||
})
|
||||
if result.success and result.data:
|
||||
# Read the generated JSON
|
||||
output_file = result.data.get("output")
|
||||
if output_file:
|
||||
data = json.loads(Path(output_file).read_text(encoding="utf-8"))
|
||||
return data.get("faces", [])
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return []
|
||||
|
||||
def _compute_face_tracked_crop(
|
||||
self,
|
||||
faces: list[dict],
|
||||
src_w: int, src_h: int,
|
||||
crop_w: int, crop_h: int,
|
||||
fps: float,
|
||||
smoothing_window: int,
|
||||
face_padding: float,
|
||||
) -> tuple[list[int], list[int]]:
|
||||
"""Compute smoothed crop positions from face tracking data.
|
||||
|
||||
Returns a single (x, y) if face positions are stable enough,
|
||||
or lists of per-frame positions for dynamic cropping.
|
||||
"""
|
||||
if not faces:
|
||||
cx = (src_w - crop_w) // 2
|
||||
cy = (src_h - crop_h) // 2
|
||||
return cx, cy
|
||||
|
||||
# Convert relative bbox centers to pixel positions
|
||||
face_centers_x = []
|
||||
face_centers_y = []
|
||||
face_timestamps = []
|
||||
|
||||
for f in faces:
|
||||
bbox = f["bbox"]
|
||||
# Center of face in pixel space
|
||||
center_x = (bbox["x"] + bbox["width"] / 2) * src_w
|
||||
center_y = (bbox["y"] + bbox["height"] / 2) * src_h
|
||||
face_centers_x.append(center_x)
|
||||
face_centers_y.append(center_y)
|
||||
face_timestamps.append(f["timestamp_seconds"])
|
||||
|
||||
# Check if face position is stable (talking head usually is)
|
||||
x_range = max(face_centers_x) - min(face_centers_x)
|
||||
y_range = max(face_centers_y) - min(face_centers_y)
|
||||
|
||||
# If face barely moves (<10% of frame), use a single static crop
|
||||
if x_range < src_w * 0.10 and y_range < src_h * 0.10:
|
||||
avg_x = sum(face_centers_x) / len(face_centers_x)
|
||||
avg_y = sum(face_centers_y) / len(face_centers_y)
|
||||
|
||||
# Position crop window centered on face, with bias toward upper third
|
||||
crop_x = int(avg_x - crop_w / 2)
|
||||
crop_y = int(avg_y - crop_h * 0.35) # Face in upper 35% of frame
|
||||
|
||||
# Clamp to frame bounds
|
||||
crop_x = max(0, min(crop_x, src_w - crop_w))
|
||||
crop_y = max(0, min(crop_y, src_h - crop_h))
|
||||
|
||||
return crop_x, crop_y
|
||||
|
||||
# Dynamic crop: smooth the trajectory
|
||||
smoothed_x = self._smooth_positions(face_centers_x, smoothing_window)
|
||||
smoothed_y = self._smooth_positions(face_centers_y, smoothing_window)
|
||||
|
||||
# Convert to crop positions (top-left corner), clamped
|
||||
crop_xs = []
|
||||
crop_ys = []
|
||||
for sx, sy in zip(smoothed_x, smoothed_y):
|
||||
cx = int(sx - crop_w / 2)
|
||||
cy = int(sy - crop_h * 0.35)
|
||||
cx = max(0, min(cx, src_w - crop_w))
|
||||
cy = max(0, min(cy, src_h - crop_h))
|
||||
crop_xs.append(cx)
|
||||
crop_ys.append(cy)
|
||||
|
||||
return crop_xs, crop_ys
|
||||
|
||||
def _smooth_positions(self, values: list[float], window: int) -> list[float]:
|
||||
"""Simple moving average smoothing."""
|
||||
smoothed = []
|
||||
for i in range(len(values)):
|
||||
start = max(0, i - window // 2)
|
||||
end = min(len(values), i + window // 2 + 1)
|
||||
smoothed.append(sum(values[start:end]) / (end - start))
|
||||
return smoothed
|
||||
|
||||
def _render_static_crop(
|
||||
self,
|
||||
input_path: Path, output_path: Path,
|
||||
crop_x: int, crop_y: int,
|
||||
crop_w: int, crop_h: int,
|
||||
out_w: int, out_h: int,
|
||||
codec: str, crf: int,
|
||||
) -> ToolResult:
|
||||
"""Render with a static crop position."""
|
||||
vf = f"crop={crop_w}:{crop_h}:{crop_x}:{crop_y},scale={out_w}:{out_h}"
|
||||
|
||||
cmd = [
|
||||
"ffmpeg", "-y",
|
||||
"-i", str(input_path),
|
||||
"-vf", vf,
|
||||
"-c:v", codec, "-crf", str(crf), "-preset", "fast",
|
||||
"-c:a", "aac", "-b:a", "192k",
|
||||
str(output_path),
|
||||
]
|
||||
|
||||
try:
|
||||
self.run_command(cmd, timeout=600)
|
||||
except Exception as e:
|
||||
return ToolResult(success=False, error=f"FFmpeg render failed: {e}")
|
||||
|
||||
return ToolResult(success=True)
|
||||
|
||||
def _render_dynamic_crop(
|
||||
self,
|
||||
input_path: Path, output_path: Path,
|
||||
crop_xs: list[int], crop_ys: list[int],
|
||||
crop_w: int, crop_h: int,
|
||||
out_w: int, out_h: int,
|
||||
fps: float,
|
||||
codec: str, crf: int,
|
||||
) -> ToolResult:
|
||||
"""Render with dynamic crop positions that follow the face.
|
||||
|
||||
Uses FFmpeg's sendcmd filter to update crop position over time.
|
||||
For simplicity and reliability, we interpolate between key positions
|
||||
using FFmpeg expression-based crop.
|
||||
"""
|
||||
# Build a piecewise-linear x(t) and y(t) using FFmpeg expressions
|
||||
# We'll sample at the face tracking rate and interpolate between points
|
||||
if not crop_xs:
|
||||
return ToolResult(success=False, error="No crop positions computed")
|
||||
|
||||
# If very few data points, fall back to static using average
|
||||
if len(crop_xs) < 3:
|
||||
avg_x = int(sum(crop_xs) / len(crop_xs))
|
||||
avg_y = int(sum(crop_ys) / len(crop_ys))
|
||||
return self._render_static_crop(
|
||||
input_path, output_path, avg_x, avg_y,
|
||||
crop_w, crop_h, out_w, out_h, codec, crf,
|
||||
)
|
||||
|
||||
# Build sendcmd script for crop filter position updates
|
||||
# Each command sets the crop x,y at the corresponding timestamp
|
||||
temp_dir = output_path.parent / ".reframe_tmp"
|
||||
temp_dir.mkdir(parents=True, exist_ok=True)
|
||||
sendcmd_path = temp_dir / "crop_commands.txt"
|
||||
|
||||
# Approximate timestamps from the face tracking sample rate
|
||||
# The face data was sampled at sample_fps intervals
|
||||
sample_interval = 1.0 / (fps / max(1, int(fps / 5))) # Approximate
|
||||
|
||||
lines = []
|
||||
for i, (cx, cy) in enumerate(zip(crop_xs, crop_ys)):
|
||||
ts = i * sample_interval
|
||||
lines.append(f"{ts:.3f} [enter] crop x {cx};")
|
||||
lines.append(f"{ts:.3f} [enter] crop y {cy};")
|
||||
|
||||
sendcmd_path.write_text("\n".join(lines), encoding="utf-8")
|
||||
|
||||
# Use crop with sendcmd for dynamic positioning
|
||||
vf = (
|
||||
f"sendcmd=f='{str(sendcmd_path).replace(chr(92), '/')}':flags=enter,"
|
||||
f"crop={crop_w}:{crop_h}:{crop_xs[0]}:{crop_ys[0]},"
|
||||
f"scale={out_w}:{out_h}"
|
||||
)
|
||||
|
||||
cmd = [
|
||||
"ffmpeg", "-y",
|
||||
"-i", str(input_path),
|
||||
"-vf", vf,
|
||||
"-c:v", codec, "-crf", str(crf), "-preset", "fast",
|
||||
"-c:a", "aac", "-b:a", "192k",
|
||||
str(output_path),
|
||||
]
|
||||
|
||||
try:
|
||||
self.run_command(cmd, timeout=600)
|
||||
except Exception:
|
||||
# sendcmd can be finicky — fall back to static crop with average position
|
||||
avg_x = int(sum(crop_xs) / len(crop_xs))
|
||||
avg_y = int(sum(crop_ys) / len(crop_ys))
|
||||
result = self._render_static_crop(
|
||||
input_path, output_path, avg_x, avg_y,
|
||||
crop_w, crop_h, out_w, out_h, codec, crf,
|
||||
)
|
||||
if result.success:
|
||||
result.data = result.data or {}
|
||||
result.data["fallback"] = "sendcmd failed, used static average crop"
|
||||
return result
|
||||
finally:
|
||||
# Clean up
|
||||
if sendcmd_path.exists():
|
||||
sendcmd_path.unlink()
|
||||
try:
|
||||
temp_dir.rmdir()
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
return ToolResult(success=True)
|
||||
|
||||
def estimate_runtime(self, inputs: dict[str, Any]) -> float:
|
||||
"""Estimate runtime in seconds. Roughly 1x realtime for face tracking + render."""
|
||||
return 60.0 # Conservative default
|
||||
|
||||
@staticmethod
|
||||
def list_presets() -> dict[str, str]:
|
||||
"""Return available aspect ratio presets."""
|
||||
return {
|
||||
name: f"{w}:{h}"
|
||||
for name, (w, h) in ASPECT_PRESETS.items()
|
||||
}
|
||||
@@ -0,0 +1,439 @@
|
||||
"""Remotion caption burn tool.
|
||||
|
||||
Renders animated word-by-word captions onto a talking-head video using
|
||||
the Remotion CaptionOverlay component. Falls back to FFmpeg subtitle
|
||||
burning if Remotion is not available.
|
||||
|
||||
The tool:
|
||||
1. Converts word-level transcript segments to Remotion WordCaption JSON
|
||||
2. Writes a props file for the TalkingHead composition
|
||||
3. Renders via ``npx remotion render``
|
||||
4. Returns the captioned video path
|
||||
|
||||
Fallback: if Remotion is unavailable, burns subtitles at the bottom of
|
||||
the frame using FFmpeg's ``subtitles`` filter with bold styling.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import math
|
||||
import re
|
||||
import shutil
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from tools.base_tool import (
|
||||
BaseTool,
|
||||
Determinism,
|
||||
ExecutionMode,
|
||||
ResourceProfile,
|
||||
ToolResult,
|
||||
ToolStability,
|
||||
ToolTier,
|
||||
)
|
||||
|
||||
|
||||
class RemotionCaptionBurn(BaseTool):
|
||||
name = "remotion_caption_burn"
|
||||
version = "0.1.0"
|
||||
tier = ToolTier.CORE
|
||||
capability = "subtitle"
|
||||
provider = "remotion"
|
||||
stability = ToolStability.EXPERIMENTAL
|
||||
execution_mode = ExecutionMode.SYNC
|
||||
determinism = Determinism.DETERMINISTIC
|
||||
|
||||
dependencies = ["cmd:ffmpeg", "cmd:ffprobe"]
|
||||
install_instructions = (
|
||||
"Remotion (optional, preferred): npm install in remotion-composer/\n"
|
||||
"FFmpeg (required for fallback): https://ffmpeg.org/download.html"
|
||||
)
|
||||
agent_skills = ["remotion-best-practices", "ffmpeg"]
|
||||
|
||||
capabilities = [
|
||||
"burn_remotion_captions",
|
||||
"burn_ffmpeg_captions_fallback",
|
||||
]
|
||||
|
||||
input_schema = {
|
||||
"type": "object",
|
||||
"required": ["input_path", "output_path"],
|
||||
"properties": {
|
||||
"input_path": {
|
||||
"type": "string",
|
||||
"description": "Path to the input video (enhanced talking-head footage).",
|
||||
},
|
||||
"output_path": {
|
||||
"type": "string",
|
||||
"description": "Path for the output video with captions burned in.",
|
||||
},
|
||||
"segments": {
|
||||
"type": "array",
|
||||
"description": (
|
||||
"Word-level transcript segments from transcriber tool. "
|
||||
"Each segment has 'words' array with {word, start, end}."
|
||||
),
|
||||
},
|
||||
"srt_path": {
|
||||
"type": "string",
|
||||
"description": (
|
||||
"Path to an SRT file. Used as an alternative to segments. "
|
||||
"If both provided, segments take priority."
|
||||
),
|
||||
},
|
||||
"words_per_page": {
|
||||
"type": "integer",
|
||||
"default": 4,
|
||||
"description": "Words shown at once in the caption overlay.",
|
||||
},
|
||||
"font_size": {
|
||||
"type": "integer",
|
||||
"default": 52,
|
||||
"description": "Font size for captions.",
|
||||
},
|
||||
"highlight_color": {
|
||||
"type": "string",
|
||||
"default": "#22D3EE",
|
||||
"description": "Highlight color for the active word (hex).",
|
||||
},
|
||||
"corrections": {
|
||||
"type": "object",
|
||||
"description": (
|
||||
"Dictionary of word corrections for common misrecognitions. "
|
||||
"Keys are the wrong word (case-insensitive), values are the "
|
||||
"correct replacement. Example: {\"cloud\": \"Claude\"}."
|
||||
),
|
||||
},
|
||||
"force_ffmpeg": {
|
||||
"type": "boolean",
|
||||
"default": False,
|
||||
"description": "Force FFmpeg fallback even if Remotion is available.",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
resource_profile = ResourceProfile(cpu_cores=4, ram_mb=2048, vram_mb=0, disk_mb=500)
|
||||
idempotency_key_fields = ["input_path", "segments", "srt_path"]
|
||||
side_effects = ["writes captioned video to output_path"]
|
||||
user_visible_verification = [
|
||||
"Play the output video and verify captions appear at the bottom of the frame",
|
||||
"Check that the active word is highlighted in the specified color",
|
||||
"Verify face is not occluded by caption text",
|
||||
]
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# Remotion detection
|
||||
# ------------------------------------------------------------------ #
|
||||
|
||||
def _find_remotion_root(self) -> Path | None:
|
||||
"""Find the remotion-composer directory relative to the repo."""
|
||||
candidates = [
|
||||
Path.cwd() / "remotion-composer",
|
||||
Path(__file__).resolve().parent.parent.parent / "remotion-composer",
|
||||
]
|
||||
for p in candidates:
|
||||
if (
|
||||
p.is_dir()
|
||||
and (p / "package.json").exists()
|
||||
and (p / "node_modules").is_dir()
|
||||
):
|
||||
return p
|
||||
return None
|
||||
|
||||
def _remotion_available(self) -> bool:
|
||||
return (
|
||||
shutil.which("npx") is not None
|
||||
and self._find_remotion_root() is not None
|
||||
)
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# Word caption conversion
|
||||
# ------------------------------------------------------------------ #
|
||||
|
||||
def _segments_to_word_captions(
|
||||
self, segments: list[dict], corrections: dict[str, str] | None = None
|
||||
) -> list[dict]:
|
||||
"""Convert transcriber segments to [{word, startMs, endMs}, ...]."""
|
||||
captions: list[dict] = []
|
||||
corr = {k.lower(): v for k, v in (corrections or {}).items()}
|
||||
|
||||
for seg in segments:
|
||||
words = seg.get("words", [])
|
||||
if words:
|
||||
for w in words:
|
||||
raw = w["word"].strip()
|
||||
fixed = corr.get(raw.lower().strip(".,!?;:"), raw)
|
||||
# Preserve trailing punctuation from original
|
||||
trailing = ""
|
||||
if raw and raw[-1] in ".,!?;:":
|
||||
trailing = raw[-1]
|
||||
if fixed != raw and not fixed.endswith(trailing):
|
||||
fixed = fixed + trailing
|
||||
captions.append({
|
||||
"word": fixed,
|
||||
"startMs": int(w["start"] * 1000),
|
||||
"endMs": int(w["end"] * 1000),
|
||||
})
|
||||
elif "text" in seg:
|
||||
text_words = seg["text"].strip().split()
|
||||
dur = seg["end"] - seg["start"]
|
||||
per_word = dur / max(len(text_words), 1)
|
||||
for i, tw in enumerate(text_words):
|
||||
fixed = corr.get(tw.lower().strip(".,!?;:"), tw)
|
||||
captions.append({
|
||||
"word": fixed,
|
||||
"startMs": int((seg["start"] + i * per_word) * 1000),
|
||||
"endMs": int((seg["start"] + (i + 1) * per_word) * 1000),
|
||||
})
|
||||
return captions
|
||||
|
||||
def _srt_to_word_captions(
|
||||
self, srt_path: str, corrections: dict[str, str] | None = None
|
||||
) -> list[dict]:
|
||||
"""Parse SRT file into word captions."""
|
||||
content = Path(srt_path).read_text(encoding="utf-8")
|
||||
blocks = re.split(r"\n\n+", content.strip())
|
||||
corr = {k.lower(): v for k, v in (corrections or {}).items()}
|
||||
captions: list[dict] = []
|
||||
|
||||
for block in blocks:
|
||||
lines = block.strip().split("\n")
|
||||
if len(lines) < 3:
|
||||
continue
|
||||
m = re.match(
|
||||
r"(\d{2}):(\d{2}):(\d{2}),(\d{3})\s*-->\s*"
|
||||
r"(\d{2}):(\d{2}):(\d{2}),(\d{3})",
|
||||
lines[1],
|
||||
)
|
||||
if not m:
|
||||
continue
|
||||
start_ms = (
|
||||
int(m.group(1)) * 3600000
|
||||
+ int(m.group(2)) * 60000
|
||||
+ int(m.group(3)) * 1000
|
||||
+ int(m.group(4))
|
||||
)
|
||||
end_ms = (
|
||||
int(m.group(5)) * 3600000
|
||||
+ int(m.group(6)) * 60000
|
||||
+ int(m.group(7)) * 1000
|
||||
+ int(m.group(8))
|
||||
)
|
||||
text = " ".join(lines[2:]).strip()
|
||||
words = text.split()
|
||||
per_word = (end_ms - start_ms) / max(len(words), 1)
|
||||
for i, w in enumerate(words):
|
||||
fixed = corr.get(w.lower().strip(".,!?;:"), w)
|
||||
captions.append({
|
||||
"word": fixed,
|
||||
"startMs": int(start_ms + i * per_word),
|
||||
"endMs": int(start_ms + (i + 1) * per_word),
|
||||
})
|
||||
return captions
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# Remotion render
|
||||
# ------------------------------------------------------------------ #
|
||||
|
||||
def _render_remotion(
|
||||
self,
|
||||
input_path: str,
|
||||
output_path: str,
|
||||
captions: list[dict],
|
||||
words_per_page: int,
|
||||
font_size: int,
|
||||
highlight_color: str,
|
||||
) -> ToolResult:
|
||||
root = self._find_remotion_root()
|
||||
if root is None:
|
||||
return ToolResult(success=False, error="Remotion root not found")
|
||||
|
||||
# Get video duration in frames
|
||||
dur_cmd = [
|
||||
"ffprobe", "-v", "error",
|
||||
"-show_entries", "format=duration",
|
||||
"-of", "csv=p=0",
|
||||
input_path,
|
||||
]
|
||||
dur_out = self.run_command(dur_cmd, capture=True)
|
||||
duration_s = float(dur_out.strip().split("\n")[0])
|
||||
total_frames = math.ceil(duration_s * 30)
|
||||
|
||||
# Copy video to Remotion public folder
|
||||
pub_dir = root / "public" / "talking-head"
|
||||
pub_dir.mkdir(parents=True, exist_ok=True)
|
||||
video_filename = Path(input_path).name
|
||||
dest_video = pub_dir / video_filename
|
||||
shutil.copy2(input_path, dest_video)
|
||||
|
||||
# Build props JSON
|
||||
props = {
|
||||
"videoSrc": f"public/talking-head/{video_filename}",
|
||||
"captions": captions,
|
||||
"wordsPerPage": words_per_page,
|
||||
"fontSize": font_size,
|
||||
"highlightColor": highlight_color,
|
||||
}
|
||||
props_dir = root / "public" / "demo-props"
|
||||
props_dir.mkdir(parents=True, exist_ok=True)
|
||||
props_file = props_dir / f"caption-burn-{Path(input_path).stem}.json"
|
||||
props_file.write_text(json.dumps(props, indent=2), encoding="utf-8")
|
||||
|
||||
# Render
|
||||
render_cmd = [
|
||||
"npx", "remotion", "render",
|
||||
"src/index.tsx", "TalkingHead",
|
||||
f"--props={props_file.relative_to(root)}",
|
||||
"--width=1080", "--height=1920", "--fps=30",
|
||||
f"--frames=0-{total_frames - 1}",
|
||||
"--codec=h264", "--crf=18",
|
||||
str(Path(output_path).resolve()),
|
||||
]
|
||||
self.run_command(render_cmd, cwd=str(root))
|
||||
|
||||
if not Path(output_path).exists():
|
||||
return ToolResult(success=False, error="Remotion render produced no output")
|
||||
|
||||
return ToolResult(
|
||||
success=True,
|
||||
data={
|
||||
"method": "remotion",
|
||||
"output": output_path,
|
||||
"duration_seconds": round(duration_s, 2),
|
||||
"total_frames": total_frames,
|
||||
"caption_count": len(captions),
|
||||
"words_per_page": words_per_page,
|
||||
},
|
||||
artifacts=[output_path],
|
||||
)
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# FFmpeg fallback
|
||||
# ------------------------------------------------------------------ #
|
||||
|
||||
def _render_ffmpeg(
|
||||
self,
|
||||
input_path: str,
|
||||
output_path: str,
|
||||
captions: list[dict],
|
||||
) -> ToolResult:
|
||||
"""Fall back to FFmpeg subtitle burning at bottom of frame."""
|
||||
# Generate temporary SRT from word captions
|
||||
tmp_srt = Path(output_path).parent / f"_tmp_captions_{int(time.time())}.srt"
|
||||
tmp_srt.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
srt_lines = []
|
||||
idx = 1
|
||||
# Group into pages of ~4 words
|
||||
page_size = 4
|
||||
for i in range(0, len(captions), page_size):
|
||||
page = captions[i : i + page_size]
|
||||
text = " ".join(c["word"] for c in page)
|
||||
start = page[0]["startMs"]
|
||||
end = page[-1]["endMs"]
|
||||
srt_lines.append(str(idx))
|
||||
srt_lines.append(
|
||||
f"{self._ms_to_srt(start)} --> {self._ms_to_srt(end)}"
|
||||
)
|
||||
srt_lines.append(text)
|
||||
srt_lines.append("")
|
||||
idx += 1
|
||||
|
||||
tmp_srt.write_text("\n".join(srt_lines), encoding="utf-8")
|
||||
|
||||
# Escape path for FFmpeg subtitles filter (Windows colon issue)
|
||||
srt_escaped = str(tmp_srt).replace("\\", "/").replace(":", "\\:")
|
||||
|
||||
cmd = [
|
||||
"ffmpeg", "-y",
|
||||
"-i", input_path,
|
||||
"-vf", (
|
||||
f"subtitles='{srt_escaped}'"
|
||||
":force_style='FontName=Segoe UI,FontSize=24,Bold=1,"
|
||||
"PrimaryColour=&H00FFFFFF,OutlineColour=&H00000000,"
|
||||
"Outline=3,Shadow=2,Alignment=2,MarginV=100'"
|
||||
),
|
||||
"-c:v", "libx264", "-preset", "fast", "-crf", "18",
|
||||
"-pix_fmt", "yuv420p",
|
||||
"-c:a", "copy",
|
||||
output_path,
|
||||
]
|
||||
self.run_command(cmd)
|
||||
|
||||
# Clean up temp SRT
|
||||
try:
|
||||
tmp_srt.unlink()
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
if not Path(output_path).exists():
|
||||
return ToolResult(success=False, error="FFmpeg subtitle burn produced no output")
|
||||
|
||||
return ToolResult(
|
||||
success=True,
|
||||
data={
|
||||
"method": "ffmpeg_fallback",
|
||||
"output": output_path,
|
||||
"caption_count": len(captions),
|
||||
"note": "Used FFmpeg fallback. Install Remotion for animated captions.",
|
||||
},
|
||||
artifacts=[output_path],
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _ms_to_srt(ms: int) -> str:
|
||||
h = ms // 3600000
|
||||
m = (ms % 3600000) // 60000
|
||||
s = (ms % 60000) // 1000
|
||||
rem = ms % 1000
|
||||
return f"{h:02d}:{m:02d}:{s:02d},{rem:03d}"
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# Main execute
|
||||
# ------------------------------------------------------------------ #
|
||||
|
||||
def execute(self, inputs: dict[str, Any]) -> ToolResult:
|
||||
input_path = inputs["input_path"]
|
||||
output_path = inputs["output_path"]
|
||||
corrections = inputs.get("corrections")
|
||||
force_ffmpeg = inputs.get("force_ffmpeg", False)
|
||||
words_per_page = inputs.get("words_per_page", 4)
|
||||
font_size = inputs.get("font_size", 52)
|
||||
highlight_color = inputs.get("highlight_color", "#22D3EE")
|
||||
|
||||
if not Path(input_path).exists():
|
||||
return ToolResult(success=False, error=f"Input video not found: {input_path}")
|
||||
|
||||
Path(output_path).parent.mkdir(parents=True, exist_ok=True)
|
||||
start = time.time()
|
||||
|
||||
# Build word captions from segments or SRT
|
||||
segments = inputs.get("segments")
|
||||
srt_path = inputs.get("srt_path")
|
||||
|
||||
if segments:
|
||||
captions = self._segments_to_word_captions(segments, corrections)
|
||||
elif srt_path:
|
||||
captions = self._srt_to_word_captions(srt_path, corrections)
|
||||
else:
|
||||
return ToolResult(
|
||||
success=False,
|
||||
error="Provide either 'segments' (from transcriber) or 'srt_path'.",
|
||||
)
|
||||
|
||||
if not captions:
|
||||
return ToolResult(success=False, error="No caption words extracted.")
|
||||
|
||||
# Choose render method
|
||||
if not force_ffmpeg and self._remotion_available():
|
||||
result = self._render_remotion(
|
||||
input_path, output_path, captions,
|
||||
words_per_page, font_size, highlight_color,
|
||||
)
|
||||
else:
|
||||
result = self._render_ffmpeg(input_path, output_path, captions)
|
||||
|
||||
result.duration_seconds = round(time.time() - start, 2)
|
||||
return result
|
||||
@@ -0,0 +1,226 @@
|
||||
"""Showcase card tool wrapping FFmpeg.
|
||||
|
||||
Creates a presentation-ready 9:16 card from a source video: letterboxes
|
||||
the content, adds a bold title at the top, a subtitle description at the
|
||||
bottom, and a dark background. Designed for Instagram Reels / TikTok
|
||||
showcase segments.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from tools.base_tool import (
|
||||
BaseTool,
|
||||
Determinism,
|
||||
ExecutionMode,
|
||||
ResourceProfile,
|
||||
ToolResult,
|
||||
ToolStability,
|
||||
ToolTier,
|
||||
)
|
||||
|
||||
|
||||
class ShowcaseCard(BaseTool):
|
||||
name = "showcase_card"
|
||||
version = "0.1.0"
|
||||
tier = ToolTier.CORE
|
||||
capability = "video_post"
|
||||
provider = "ffmpeg"
|
||||
stability = ToolStability.EXPERIMENTAL
|
||||
execution_mode = ExecutionMode.SYNC
|
||||
determinism = Determinism.DETERMINISTIC
|
||||
|
||||
dependencies = ["cmd:ffmpeg", "cmd:ffprobe"]
|
||||
install_instructions = "Install FFmpeg: https://ffmpeg.org/download.html"
|
||||
agent_skills = ["ffmpeg", "video_toolkit"]
|
||||
|
||||
capabilities = ["create_showcase_card"]
|
||||
|
||||
input_schema = {
|
||||
"type": "object",
|
||||
"required": ["input_path", "output_path", "title"],
|
||||
"properties": {
|
||||
"input_path": {
|
||||
"type": "string",
|
||||
"description": "Path to the source video.",
|
||||
},
|
||||
"output_path": {
|
||||
"type": "string",
|
||||
"description": "Path for the output showcase card video.",
|
||||
},
|
||||
"title": {
|
||||
"type": "string",
|
||||
"description": "Bold title text displayed at the top of the card.",
|
||||
},
|
||||
"subtitle": {
|
||||
"type": "string",
|
||||
"default": "",
|
||||
"description": "Subtitle text displayed at the bottom of the card.",
|
||||
},
|
||||
"output_width": {
|
||||
"type": "integer",
|
||||
"default": 1080,
|
||||
"description": "Output width in pixels.",
|
||||
},
|
||||
"output_height": {
|
||||
"type": "integer",
|
||||
"default": 1920,
|
||||
"description": "Output height in pixels.",
|
||||
},
|
||||
"background_color": {
|
||||
"type": "string",
|
||||
"default": "0x0A0F1A",
|
||||
"description": "Background color in hex (FFmpeg format, e.g. 0x0A0F1A).",
|
||||
},
|
||||
"title_font": {
|
||||
"type": "string",
|
||||
"default": "segoeuib.ttf",
|
||||
"description": "Font file for the title. Uses system font lookup.",
|
||||
},
|
||||
"title_font_size": {
|
||||
"type": "integer",
|
||||
"default": 52,
|
||||
"description": "Font size for the title.",
|
||||
},
|
||||
"subtitle_font_size": {
|
||||
"type": "integer",
|
||||
"default": 28,
|
||||
"description": "Font size for the subtitle.",
|
||||
},
|
||||
"title_color": {
|
||||
"type": "string",
|
||||
"default": "white",
|
||||
"description": "Title text color.",
|
||||
},
|
||||
"watermark": {
|
||||
"type": "string",
|
||||
"default": "",
|
||||
"description": "Optional watermark text overlaid on the video (e.g. brand name).",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
resource_profile = ResourceProfile(cpu_cores=2, ram_mb=1024, vram_mb=0, disk_mb=500)
|
||||
idempotency_key_fields = ["input_path", "title", "subtitle"]
|
||||
side_effects = ["writes showcase card video to output_path"]
|
||||
user_visible_verification = [
|
||||
"Play output and verify title, subtitle, and video are positioned correctly",
|
||||
"Verify the video content is fully visible (not cropped)",
|
||||
]
|
||||
|
||||
def execute(self, inputs: dict[str, Any]) -> ToolResult:
|
||||
input_path = inputs["input_path"]
|
||||
output_path = inputs["output_path"]
|
||||
title = inputs["title"]
|
||||
subtitle = inputs.get("subtitle", "")
|
||||
out_w = inputs.get("output_width", 1080)
|
||||
out_h = inputs.get("output_height", 1920)
|
||||
bg_color = inputs.get("background_color", "0x0A0F1A")
|
||||
title_font = inputs.get("title_font", "segoeuib.ttf")
|
||||
title_font_size = inputs.get("title_font_size", 52)
|
||||
subtitle_font_size = inputs.get("subtitle_font_size", 28)
|
||||
title_color = inputs.get("title_color", "white")
|
||||
watermark = inputs.get("watermark", "")
|
||||
|
||||
if not Path(input_path).exists():
|
||||
return ToolResult(success=False, error=f"Input not found: {input_path}")
|
||||
|
||||
Path(output_path).parent.mkdir(parents=True, exist_ok=True)
|
||||
start = time.time()
|
||||
|
||||
# Get source dimensions
|
||||
probe_cmd = [
|
||||
"ffprobe", "-v", "error",
|
||||
"-select_streams", "v:0",
|
||||
"-show_entries", "stream=width,height",
|
||||
"-of", "csv=p=0",
|
||||
input_path,
|
||||
]
|
||||
probe_out = self.run_command(probe_cmd, capture=True).strip()
|
||||
src_w, src_h = [int(x.strip()) for x in probe_out.split(",")[:2]]
|
||||
|
||||
# Calculate letterbox dimensions — fit source into output width,
|
||||
# center vertically in the frame.
|
||||
scale_factor = out_w / src_w
|
||||
scaled_h = int(src_h * scale_factor)
|
||||
# Ensure even dimensions
|
||||
scaled_h = scaled_h if scaled_h % 2 == 0 else scaled_h + 1
|
||||
pad_y = (out_h - scaled_h) // 2
|
||||
|
||||
# Build filter chain
|
||||
filters = [
|
||||
f"scale={out_w}:{scaled_h}",
|
||||
f"pad={out_w}:{out_h}:0:{pad_y}:color={bg_color}",
|
||||
]
|
||||
|
||||
# Title text at top
|
||||
title_escaped = title.replace("'", "\\'").replace(":", "\\:")
|
||||
filters.append(
|
||||
f"drawtext=text='{title_escaped}'"
|
||||
f":fontfile='{title_font}'"
|
||||
f":fontsize={title_font_size}"
|
||||
f":fontcolor={title_color}"
|
||||
f":borderw=3:bordercolor=black"
|
||||
f":x=(w-text_w)/2:y=60"
|
||||
)
|
||||
|
||||
# Subtitle text at bottom
|
||||
if subtitle:
|
||||
sub_escaped = subtitle.replace("'", "\\'").replace(":", "\\:")
|
||||
filters.append(
|
||||
f"drawtext=text='{sub_escaped}'"
|
||||
f":fontfile='segoeui.ttf'"
|
||||
f":fontsize={subtitle_font_size}"
|
||||
f":fontcolor=white@0.85"
|
||||
f":x=(w-text_w)/2:y=h-100"
|
||||
)
|
||||
|
||||
# Watermark centered on video
|
||||
if watermark:
|
||||
wm_escaped = watermark.replace("'", "\\'").replace(":", "\\:")
|
||||
filters.append(
|
||||
f"drawtext=text='{wm_escaped}'"
|
||||
f":fontfile='segoeui.ttf'"
|
||||
f":fontsize=36"
|
||||
f":fontcolor=white@0.3"
|
||||
f":x=(w-text_w)/2:y=(h-text_h)/2"
|
||||
)
|
||||
|
||||
vf = ",".join(filters)
|
||||
|
||||
cmd = [
|
||||
"ffmpeg", "-y",
|
||||
"-i", input_path,
|
||||
"-vf", vf,
|
||||
"-c:v", "libx264", "-preset", "fast", "-crf", "18",
|
||||
"-pix_fmt", "yuv420p",
|
||||
"-c:a", "aac", "-b:a", "192k",
|
||||
output_path,
|
||||
]
|
||||
|
||||
try:
|
||||
self.run_command(cmd)
|
||||
except Exception as e:
|
||||
return ToolResult(success=False, error=f"FFmpeg failed: {e}")
|
||||
|
||||
if not Path(output_path).exists():
|
||||
return ToolResult(success=False, error="No output produced")
|
||||
|
||||
elapsed = round(time.time() - start, 2)
|
||||
|
||||
return ToolResult(
|
||||
success=True,
|
||||
data={
|
||||
"output": output_path,
|
||||
"source_resolution": f"{src_w}x{src_h}",
|
||||
"output_resolution": f"{out_w}x{out_h}",
|
||||
"title": title,
|
||||
"subtitle": subtitle,
|
||||
"letterbox_y_offset": pad_y,
|
||||
},
|
||||
artifacts=[output_path],
|
||||
duration_seconds=elapsed,
|
||||
)
|
||||
@@ -0,0 +1,481 @@
|
||||
"""Silence cutter tool for automatic jump cuts.
|
||||
|
||||
Detects silent segments in talking-head footage and removes them,
|
||||
creating tight jump cuts. Uses FFmpeg's silencedetect filter — no
|
||||
external dependencies beyond FFmpeg.
|
||||
|
||||
Modes:
|
||||
- remove: Cut out silent segments entirely (jump cut)
|
||||
- speed_up: Speed up silent segments instead of cutting (less jarring)
|
||||
- mark: Don't cut — just output silence timestamps for manual review
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from tools.base_tool import (
|
||||
BaseTool,
|
||||
Determinism,
|
||||
ExecutionMode,
|
||||
ResourceProfile,
|
||||
RetryPolicy,
|
||||
ResumeSupport,
|
||||
ToolResult,
|
||||
ToolStability,
|
||||
ToolTier,
|
||||
)
|
||||
|
||||
|
||||
class SilenceCutter(BaseTool):
|
||||
name = "silence_cutter"
|
||||
version = "0.1.0"
|
||||
tier = ToolTier.CORE
|
||||
capability = "video_post"
|
||||
provider = "ffmpeg"
|
||||
stability = ToolStability.EXPERIMENTAL
|
||||
execution_mode = ExecutionMode.SYNC
|
||||
determinism = Determinism.DETERMINISTIC
|
||||
|
||||
dependencies = ["cmd:ffmpeg"]
|
||||
install_instructions = "Install FFmpeg: https://ffmpeg.org/download.html"
|
||||
agent_skills = ["ffmpeg"]
|
||||
|
||||
capabilities = [
|
||||
"silence_detection",
|
||||
"jump_cut",
|
||||
"silence_removal",
|
||||
"silence_speedup",
|
||||
]
|
||||
|
||||
input_schema = {
|
||||
"type": "object",
|
||||
"required": ["input_path"],
|
||||
"properties": {
|
||||
"input_path": {"type": "string"},
|
||||
"output_path": {"type": "string"},
|
||||
"mode": {
|
||||
"type": "string",
|
||||
"enum": ["remove", "speed_up", "mark"],
|
||||
"default": "remove",
|
||||
"description": "remove=jump cut, speed_up=fast-forward silence, mark=detect only",
|
||||
},
|
||||
"silence_threshold_db": {
|
||||
"type": "number",
|
||||
"default": -35,
|
||||
"description": "Audio level below this (in dB) is considered silence. Lower = more sensitive.",
|
||||
},
|
||||
"min_silence_duration": {
|
||||
"type": "number",
|
||||
"default": 0.5,
|
||||
"minimum": 0.1,
|
||||
"description": "Minimum silence duration in seconds to trigger a cut",
|
||||
},
|
||||
"padding_seconds": {
|
||||
"type": "number",
|
||||
"default": 0.08,
|
||||
"minimum": 0.0,
|
||||
"description": "Seconds of silence to keep on each side of speech (prevents clipped words)",
|
||||
},
|
||||
"silence_speed_factor": {
|
||||
"type": "number",
|
||||
"default": 6.0,
|
||||
"minimum": 1.5,
|
||||
"maximum": 100.0,
|
||||
"description": "Speed multiplier for silent segments (only used in speed_up mode)",
|
||||
},
|
||||
"codec": {"type": "string", "default": "libx264"},
|
||||
"crf": {"type": "integer", "default": 18},
|
||||
},
|
||||
}
|
||||
|
||||
resource_profile = ResourceProfile(
|
||||
cpu_cores=4, ram_mb=2048, vram_mb=0, disk_mb=4000, network_required=False
|
||||
)
|
||||
retry_policy = RetryPolicy(max_retries=1, retryable_errors=["FFmpeg error"])
|
||||
resume_support = ResumeSupport.FROM_START
|
||||
idempotency_key_fields = [
|
||||
"input_path", "mode", "silence_threshold_db",
|
||||
"min_silence_duration", "padding_seconds",
|
||||
]
|
||||
side_effects = ["writes cut video to output_path"]
|
||||
user_visible_verification = [
|
||||
"Watch output for unnaturally clipped words at cut points",
|
||||
"Compare duration: output should be noticeably shorter than input",
|
||||
]
|
||||
|
||||
def execute(self, inputs: dict[str, Any]) -> ToolResult:
|
||||
input_path = Path(inputs["input_path"])
|
||||
if not input_path.exists():
|
||||
return ToolResult(success=False, error=f"Input not found: {input_path}")
|
||||
|
||||
mode = inputs.get("mode", "remove")
|
||||
start = time.time()
|
||||
|
||||
# Step 1: Detect silence segments
|
||||
threshold_db = inputs.get("silence_threshold_db", -35)
|
||||
min_dur = inputs.get("min_silence_duration", 0.5)
|
||||
padding = inputs.get("padding_seconds", 0.08)
|
||||
|
||||
silences = self._detect_silence(input_path, threshold_db, min_dur)
|
||||
|
||||
if not silences:
|
||||
elapsed = time.time() - start
|
||||
return ToolResult(
|
||||
success=True,
|
||||
data={
|
||||
"message": "No silence detected — video unchanged",
|
||||
"silence_segments": 0,
|
||||
"input": str(input_path),
|
||||
"output": str(input_path),
|
||||
},
|
||||
artifacts=[str(input_path)],
|
||||
duration_seconds=round(elapsed, 2),
|
||||
)
|
||||
|
||||
# Get total duration
|
||||
total_duration = self._get_duration(input_path)
|
||||
|
||||
# Step 2: Compute speech segments (inverse of silence)
|
||||
speech_segments = self._compute_speech_segments(
|
||||
silences, total_duration, padding
|
||||
)
|
||||
|
||||
# Step 3: Handle based on mode
|
||||
if mode == "mark":
|
||||
elapsed = time.time() - start
|
||||
output_json = Path(
|
||||
inputs.get("output_path", str(input_path.with_suffix(".silence.json")))
|
||||
)
|
||||
result_data = {
|
||||
"silences": silences,
|
||||
"speech_segments": speech_segments,
|
||||
"total_duration": total_duration,
|
||||
"silence_duration": sum(s["duration"] for s in silences),
|
||||
"speech_duration": sum(s["end"] - s["start"] for s in speech_segments),
|
||||
}
|
||||
output_json.parent.mkdir(parents=True, exist_ok=True)
|
||||
output_json.write_text(json.dumps(result_data, indent=2), encoding="utf-8")
|
||||
return ToolResult(
|
||||
success=True,
|
||||
data={
|
||||
"mode": "mark",
|
||||
"silence_segments": len(silences),
|
||||
"speech_segments": len(speech_segments),
|
||||
"silence_duration_seconds": round(result_data["silence_duration"], 2),
|
||||
"speech_duration_seconds": round(result_data["speech_duration"], 2),
|
||||
"output": str(output_json),
|
||||
},
|
||||
artifacts=[str(output_json)],
|
||||
duration_seconds=round(elapsed, 2),
|
||||
)
|
||||
|
||||
output_path = Path(
|
||||
inputs.get("output_path", str(input_path.with_stem(f"{input_path.stem}_cut")))
|
||||
)
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
codec = inputs.get("codec", "libx264")
|
||||
crf = inputs.get("crf", 18)
|
||||
|
||||
if mode == "speed_up":
|
||||
speed_factor = inputs.get("silence_speed_factor", 6.0)
|
||||
result = self._render_speed_up(
|
||||
input_path, output_path, silences, speech_segments,
|
||||
total_duration, speed_factor, codec, crf,
|
||||
)
|
||||
else:
|
||||
result = self._render_jump_cut(
|
||||
input_path, output_path, speech_segments, codec, crf,
|
||||
)
|
||||
|
||||
if not result.success:
|
||||
return result
|
||||
|
||||
elapsed = time.time() - start
|
||||
|
||||
silence_dur = sum(s["duration"] for s in silences)
|
||||
speech_dur = sum(s["end"] - s["start"] for s in speech_segments)
|
||||
|
||||
return ToolResult(
|
||||
success=True,
|
||||
data={
|
||||
"mode": mode,
|
||||
"input": str(input_path),
|
||||
"output": str(output_path),
|
||||
"input_duration": round(total_duration, 2),
|
||||
"output_duration": round(speech_dur, 2) if mode == "remove" else None,
|
||||
"silence_removed_seconds": round(silence_dur, 2),
|
||||
"silence_segments": len(silences),
|
||||
"speech_segments": len(speech_segments),
|
||||
"time_saved_percent": round(silence_dur / total_duration * 100, 1) if total_duration > 0 else 0,
|
||||
},
|
||||
artifacts=[str(output_path)],
|
||||
duration_seconds=round(elapsed, 2),
|
||||
)
|
||||
|
||||
def _detect_silence(
|
||||
self, input_path: Path, threshold_db: float, min_duration: float
|
||||
) -> list[dict]:
|
||||
"""Detect silent segments using FFmpeg silencedetect filter."""
|
||||
cmd = [
|
||||
"ffmpeg",
|
||||
"-i", str(input_path),
|
||||
"-af", f"silencedetect=noise={threshold_db}dB:d={min_duration}",
|
||||
"-f", "null", "-",
|
||||
]
|
||||
|
||||
try:
|
||||
result = self.run_command(cmd, timeout=300)
|
||||
output = result.stderr
|
||||
except Exception as e:
|
||||
# FFmpeg writes to stderr even on success for filters
|
||||
output = str(e)
|
||||
|
||||
# Parse silencedetect output
|
||||
# Format: [silencedetect @ ...] silence_start: 1.234
|
||||
# [silencedetect @ ...] silence_end: 2.567 | silence_duration: 1.333
|
||||
starts = re.findall(r"silence_start:\s*([\d.]+)", output)
|
||||
ends = re.findall(r"silence_end:\s*([\d.]+)", output)
|
||||
durations = re.findall(r"silence_duration:\s*([\d.]+)", output)
|
||||
|
||||
silences = []
|
||||
for i in range(min(len(starts), len(ends))):
|
||||
silences.append({
|
||||
"start": float(starts[i]),
|
||||
"end": float(ends[i]),
|
||||
"duration": float(durations[i]) if i < len(durations) else float(ends[i]) - float(starts[i]),
|
||||
})
|
||||
|
||||
return silences
|
||||
|
||||
def _get_duration(self, input_path: Path) -> float:
|
||||
"""Get video duration via ffprobe."""
|
||||
cmd = [
|
||||
"ffprobe", "-v", "quiet",
|
||||
"-show_entries", "format=duration",
|
||||
"-of", "json", str(input_path),
|
||||
]
|
||||
try:
|
||||
result = self.run_command(cmd)
|
||||
data = json.loads(result.stdout)
|
||||
return float(data["format"]["duration"])
|
||||
except Exception:
|
||||
return 0.0
|
||||
|
||||
def _compute_speech_segments(
|
||||
self, silences: list[dict], total_duration: float, padding: float
|
||||
) -> list[dict]:
|
||||
"""Compute speech segments as the inverse of silence segments, with padding."""
|
||||
segments = []
|
||||
cursor = 0.0
|
||||
|
||||
for silence in silences:
|
||||
speech_end = silence["start"] + padding
|
||||
if speech_end > cursor:
|
||||
segments.append({"start": cursor, "end": min(speech_end, total_duration)})
|
||||
cursor = max(cursor, silence["end"] - padding)
|
||||
|
||||
# Final segment after last silence
|
||||
if cursor < total_duration:
|
||||
segments.append({"start": cursor, "end": total_duration})
|
||||
|
||||
# Merge very short gaps (segments < 0.05s apart)
|
||||
merged = []
|
||||
for seg in segments:
|
||||
if seg["end"] - seg["start"] < 0.01:
|
||||
continue # Skip tiny segments
|
||||
if merged and seg["start"] - merged[-1]["end"] < 0.05:
|
||||
merged[-1]["end"] = seg["end"]
|
||||
else:
|
||||
merged.append(seg)
|
||||
|
||||
return merged
|
||||
|
||||
def _render_jump_cut(
|
||||
self,
|
||||
input_path: Path, output_path: Path,
|
||||
speech_segments: list[dict],
|
||||
codec: str, crf: int,
|
||||
) -> ToolResult:
|
||||
"""Remove silence by concatenating speech segments."""
|
||||
if not speech_segments:
|
||||
return ToolResult(success=False, error="No speech segments found")
|
||||
|
||||
temp_dir = output_path.parent / ".silence_cut_tmp"
|
||||
temp_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
try:
|
||||
# Cut each speech segment
|
||||
seg_files = []
|
||||
for i, seg in enumerate(speech_segments):
|
||||
seg_path = temp_dir / f"seg_{i:04d}.mp4"
|
||||
cmd = [
|
||||
"ffmpeg", "-y",
|
||||
"-i", str(input_path),
|
||||
"-ss", f"{seg['start']:.3f}",
|
||||
"-to", f"{seg['end']:.3f}",
|
||||
"-c:v", codec, "-crf", str(crf), "-preset", "fast",
|
||||
"-c:a", "aac", "-b:a", "192k",
|
||||
# Force keyframe at start for clean cuts
|
||||
"-force_key_frames", f"{seg['start']:.3f}",
|
||||
str(seg_path),
|
||||
]
|
||||
self.run_command(cmd, timeout=120)
|
||||
if seg_path.exists() and seg_path.stat().st_size > 0:
|
||||
seg_files.append(seg_path)
|
||||
|
||||
if not seg_files:
|
||||
return ToolResult(success=False, error="No segments were successfully cut")
|
||||
|
||||
# Concat all segments
|
||||
list_path = temp_dir / "concat_list.txt"
|
||||
with open(list_path, "w", encoding="utf-8") as f:
|
||||
for sf in seg_files:
|
||||
safe_path = str(sf.resolve()).replace("\\", "/")
|
||||
f.write(f"file '{safe_path}'\n")
|
||||
|
||||
cmd = [
|
||||
"ffmpeg", "-y",
|
||||
"-f", "concat", "-safe", "0",
|
||||
"-i", str(list_path),
|
||||
"-c", "copy",
|
||||
str(output_path),
|
||||
]
|
||||
self.run_command(cmd, timeout=120)
|
||||
|
||||
return ToolResult(success=True)
|
||||
except Exception as e:
|
||||
return ToolResult(success=False, error=f"Jump cut render failed: {e}")
|
||||
finally:
|
||||
# Clean up temp files
|
||||
for f in temp_dir.glob("*"):
|
||||
try:
|
||||
f.unlink()
|
||||
except OSError:
|
||||
pass
|
||||
try:
|
||||
temp_dir.rmdir()
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
def _render_speed_up(
|
||||
self,
|
||||
input_path: Path, output_path: Path,
|
||||
silences: list[dict], speech_segments: list[dict],
|
||||
total_duration: float,
|
||||
speed_factor: float,
|
||||
codec: str, crf: int,
|
||||
) -> ToolResult:
|
||||
"""Speed up silent segments instead of removing them.
|
||||
|
||||
This is less jarring than jump cuts — the viewer sees a brief
|
||||
fast-forward during pauses.
|
||||
"""
|
||||
temp_dir = output_path.parent / ".silence_speed_tmp"
|
||||
temp_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
try:
|
||||
# Build a timeline of segments: speech at 1x, silence at Nx
|
||||
all_segments = []
|
||||
|
||||
for seg in speech_segments:
|
||||
all_segments.append({"start": seg["start"], "end": seg["end"], "speed": 1.0})
|
||||
|
||||
for sil in silences:
|
||||
all_segments.append({"start": sil["start"], "end": sil["end"], "speed": speed_factor})
|
||||
|
||||
# Sort by start time and merge overlaps
|
||||
all_segments.sort(key=lambda s: s["start"])
|
||||
|
||||
# Process each segment
|
||||
seg_files = []
|
||||
for i, seg in enumerate(all_segments):
|
||||
seg_path = temp_dir / f"seg_{i:04d}.mp4"
|
||||
duration = seg["end"] - seg["start"]
|
||||
if duration < 0.05:
|
||||
continue
|
||||
|
||||
if seg["speed"] == 1.0:
|
||||
# Normal speed
|
||||
cmd = [
|
||||
"ffmpeg", "-y",
|
||||
"-i", str(input_path),
|
||||
"-ss", f"{seg['start']:.3f}",
|
||||
"-to", f"{seg['end']:.3f}",
|
||||
"-c:v", codec, "-crf", str(crf), "-preset", "fast",
|
||||
"-c:a", "aac", "-b:a", "192k",
|
||||
str(seg_path),
|
||||
]
|
||||
else:
|
||||
# Speed up
|
||||
pts = 1.0 / seg["speed"]
|
||||
atempo_chain = self._build_atempo_chain(seg["speed"])
|
||||
cmd = [
|
||||
"ffmpeg", "-y",
|
||||
"-i", str(input_path),
|
||||
"-ss", f"{seg['start']:.3f}",
|
||||
"-to", f"{seg['end']:.3f}",
|
||||
"-filter:v", f"setpts={pts:.4f}*PTS",
|
||||
"-filter:a", atempo_chain,
|
||||
"-c:v", codec, "-crf", str(crf), "-preset", "fast",
|
||||
"-c:a", "aac", "-b:a", "192k",
|
||||
str(seg_path),
|
||||
]
|
||||
|
||||
self.run_command(cmd, timeout=120)
|
||||
if seg_path.exists() and seg_path.stat().st_size > 0:
|
||||
seg_files.append(seg_path)
|
||||
|
||||
if not seg_files:
|
||||
return ToolResult(success=False, error="No segments rendered")
|
||||
|
||||
# Concat
|
||||
list_path = temp_dir / "concat_list.txt"
|
||||
with open(list_path, "w", encoding="utf-8") as f:
|
||||
for sf in seg_files:
|
||||
safe_path = str(sf.resolve()).replace("\\", "/")
|
||||
f.write(f"file '{safe_path}'\n")
|
||||
|
||||
cmd = [
|
||||
"ffmpeg", "-y",
|
||||
"-f", "concat", "-safe", "0",
|
||||
"-i", str(list_path),
|
||||
"-c", "copy",
|
||||
str(output_path),
|
||||
]
|
||||
self.run_command(cmd, timeout=120)
|
||||
|
||||
return ToolResult(success=True)
|
||||
except Exception as e:
|
||||
return ToolResult(success=False, error=f"Speed-up render failed: {e}")
|
||||
finally:
|
||||
for f in temp_dir.glob("*"):
|
||||
try:
|
||||
f.unlink()
|
||||
except OSError:
|
||||
pass
|
||||
try:
|
||||
temp_dir.rmdir()
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
@staticmethod
|
||||
def _build_atempo_chain(factor: float) -> str:
|
||||
"""Build atempo filter chain. atempo accepts [0.5, 100.0]."""
|
||||
filters = []
|
||||
remaining = factor
|
||||
while remaining > 100.0:
|
||||
filters.append("atempo=100.0")
|
||||
remaining /= 100.0
|
||||
while remaining < 0.5:
|
||||
filters.append("atempo=0.5")
|
||||
remaining /= 0.5
|
||||
filters.append(f"atempo={remaining:.4f}")
|
||||
return ",".join(filters)
|
||||
|
||||
def estimate_runtime(self, inputs: dict[str, Any]) -> float:
|
||||
return 45.0
|
||||
Reference in New Issue
Block a user