diff --git a/remotion-composer/public/demo-props/greenscreen-bg-full.json b/remotion-composer/public/demo-props/greenscreen-bg-full.json new file mode 100644 index 0000000..e6aa3e4 --- /dev/null +++ b/remotion-composer/public/demo-props/greenscreen-bg-full.json @@ -0,0 +1,260 @@ +{ + "cuts": [ + { + "id": "scene-1-hook", + "source": "", + "in_seconds": 0, + "out_seconds": 5.5, + "type": "hero_title", + "text": "STOP PAYING", + "subtitle": "Multiple EMIs Every Month", + "backgroundColor": "#0F172A", + "accentColor": "#EF4444" + }, + { + "id": "scene-2-control", + "source": "", + "in_seconds": 5.5, + "out_seconds": 9.5, + "type": "callout", + "text": "Your loans are managing YOU", + "callout_type": "warning", + "backgroundColor": "#0F172A", + "accentColor": "#F59E0B" + }, + { + "id": "scene-3-rates", + "source": "", + "in_seconds": 9.5, + "out_seconds": 15.5, + "type": "bar_chart", + "chartData": [ + {"label": "Credit Cards", "value": 24}, + {"label": "Personal", "value": 16}, + {"label": "App Loans", "value": 36}, + {"label": "NBFC", "value": 18} + ], + "title": "Interest Rates You're Paying (%)", + "backgroundColor": "#0F172A", + "accentColor": "#F59E0B", + "chartColors": ["#EF4444", "#F59E0B", "#8B5CF6", "#EC4899"] + }, + { + "id": "scene-4-fear", + "source": "", + "in_seconds": 15.5, + "out_seconds": 20, + "type": "stat_reveal", + "text": "Every Month", + "subtitle": "Will I have anything left?", + "backgroundColor": "#0F172A", + "accentColor": "#EF4444" + }, + { + "id": "scene-5-reveal", + "source": "", + "in_seconds": 20, + "out_seconds": 22, + "type": "hero_title", + "text": "WHAT NO ONE", + "subtitle": "Tells You", + "backgroundColor": "#0F172A", + "accentColor": "#22D3EE" + }, + { + "id": "scene-6-inquiries", + "source": "", + "in_seconds": 22, + "out_seconds": 40.5, + "type": "progress_bar", + "progress": 95, + "label": "Loan Inquiries Piling Up", + "progressLabel": "DANGER ZONE", + "progressColor": "#EF4444", + "animationStyle": "step", + "progressSegments": [ + {"value": 25, "color": "#F59E0B", "label": "Bank"}, + {"value": 25, "color": "#F97316", "label": "NBFC"}, + {"value": 25, "color": "#EF4444", "label": "Digital"}, + {"value": 20, "color": "#DC2626", "label": "Random"} + ], + "backgroundColor": "#0F172A" + }, + { + "id": "scene-7-score-drop", + "source": "", + "in_seconds": 41, + "out_seconds": 46.5, + "type": "line_chart", + "chartSeries": [ + { + "name": "Credit Score", + "data": [ + {"x": 0, "y": 750}, + {"x": 1, "y": 720}, + {"x": 2, "y": 680}, + {"x": 3, "y": 620} + ], + "color": "#EF4444" + } + ], + "title": "Your Credit Score Drops", + "xLabel": "Each Inquiry", + "yLabel": "Score", + "showGrid": true, + "showMarkers": true, + "chartAnimation": "draw", + "backgroundColor": "#0F172A", + "accentColor": "#EF4444" + }, + { + "id": "scene-8-risk", + "source": "", + "in_seconds": 47, + "out_seconds": 49.5, + "type": "callout", + "text": "Lenders See You As HIGH RISK", + "callout_type": "warning", + "backgroundColor": "#0F172A", + "accentColor": "#EF4444" + }, + { + "id": "scene-9-wrong", + "source": "", + "in_seconds": 50, + "out_seconds": 53.5, + "type": "stat_card", + "stat": "❌", + "subtitle": "Wrong approach wastes your time", + "backgroundColor": "#0F172A", + "accentColor": "#EF4444" + }, + { + "id": "scene-9-right", + "source": "", + "in_seconds": 53.5, + "out_seconds": 57, + "type": "stat_card", + "stat": "✅", + "subtitle": "Right approach gets approval", + "backgroundColor": "#0F172A", + "accentColor": "#10B981" + }, + { + "id": "scene-10-matters", + "source": "", + "in_seconds": 60, + "out_seconds": 66, + "type": "callout", + "text": "WHO you approach matters as much as WHAT you apply for", + "callout_type": "tip", + "backgroundColor": "#0F172A", + "accentColor": "#10B981" + }, + { + "id": "scene-11-creditwin", + "source": "", + "in_seconds": 66, + "out_seconds": 80, + "type": "hero_title", + "text": "CREDIT WIN", + "subtitle": "Your Consolidation Partner", + "backgroundColor": "#0F172A", + "accentColor": "#22D3EE" + }, + { + "id": "scene-12-oneplan", + "source": "", + "in_seconds": 81, + "out_seconds": 84.5, + "type": "kpi_grid", + "chartData": [ + {"label": "EMI", "value": "1"}, + {"label": "Due Date", "value": "1"}, + {"label": "Clear Plan", "value": "1"} + ], + "title": "From Chaos to Control", + "backgroundColor": "#0F172A", + "accentColor": "#22D3EE", + "chartColors": ["#22D3EE", "#10B981", "#A78BFA"] + }, + { + "id": "scene-13a-years", + "source": "", + "in_seconds": 84.5, + "out_seconds": 89, + "type": "stat_card", + "stat": "15+", + "subtitle": "Years of Experience", + "backgroundColor": "#0F172A", + "accentColor": "#F59E0B" + }, + { + "id": "scene-13b-borrowers", + "source": "", + "in_seconds": 89, + "out_seconds": 94, + "type": "stat_card", + "stat": "10,000+", + "subtitle": "Borrowers Served", + "backgroundColor": "#0F172A", + "accentColor": "#22D3EE" + }, + { + "id": "scene-13c-saved", + "source": "", + "in_seconds": 94, + "out_seconds": 99, + "type": "stat_card", + "stat": "₹5K-15K", + "subtitle": "Saved Per Month on EMIs", + "backgroundColor": "#0F172A", + "accentColor": "#10B981" + }, + { + "id": "scene-14-qualify", + "source": "", + "in_seconds": 100, + "out_seconds": 118, + "type": "stat_card", + "stat": "₹60,000+", + "subtitle": "Salary? You Qualify for Consolidation", + "backgroundColor": "#0F172A", + "accentColor": "#22D3EE" + }, + { + "id": "scene-15-before", + "source": "", + "in_seconds": 118, + "out_seconds": 121, + "type": "stat_card", + "stat": "5 EMIs", + "subtitle": "Chaos → Exit This", + "backgroundColor": "#0F172A", + "accentColor": "#EF4444" + }, + { + "id": "scene-15-after", + "source": "", + "in_seconds": 121, + "out_seconds": 124, + "type": "stat_card", + "stat": "1 EMI", + "subtitle": "Control → Your New Reality", + "backgroundColor": "#0F172A", + "accentColor": "#10B981" + }, + { + "id": "scene-16-cta", + "source": "", + "in_seconds": 124, + "out_seconds": 127, + "type": "hero_title", + "text": "APPLY NOW", + "subtitle": "CreditWin.in", + "backgroundColor": "#0F172A", + "accentColor": "#10B981" + } + ], + "overlays": [] +} diff --git a/remotion-composer/public/demo-props/greenscreen-bg.json b/remotion-composer/public/demo-props/greenscreen-bg.json new file mode 100644 index 0000000..e0ca114 --- /dev/null +++ b/remotion-composer/public/demo-props/greenscreen-bg.json @@ -0,0 +1,55 @@ +{ + "cuts": [ + { + "id": "scene-1-hook", + "source": "", + "in_seconds": 0, + "out_seconds": 5.5, + "type": "hero_title", + "text": "STOP PAYING", + "subtitle": "Multiple EMIs Every Month", + "backgroundColor": "#0F172A", + "accentColor": "#EF4444" + }, + { + "id": "scene-2-control", + "source": "", + "in_seconds": 5.5, + "out_seconds": 9.5, + "type": "callout", + "text": "Your loans are managing YOU", + "callout_type": "warning", + "backgroundColor": "#0F172A", + "accentColor": "#F59E0B" + }, + { + "id": "scene-3-rates", + "source": "", + "in_seconds": 9.5, + "out_seconds": 15.5, + "type": "bar_chart", + "chartData": [ + {"label": "Credit Cards", "value": 24}, + {"label": "Personal", "value": 16}, + {"label": "App Loans", "value": 36}, + {"label": "NBFC", "value": 18} + ], + "title": "Interest Rates You're Paying (%)", + "backgroundColor": "#0F172A", + "accentColor": "#F59E0B", + "chartColors": ["#EF4444", "#F59E0B", "#8B5CF6", "#EC4899"] + }, + { + "id": "scene-4-fear", + "source": "", + "in_seconds": 15.5, + "out_seconds": 20, + "type": "stat_reveal", + "text": "Every Month", + "subtitle": "Will I have anything left?", + "backgroundColor": "#0F172A", + "accentColor": "#EF4444" + } + ], + "overlays": [] +} diff --git a/remotion-composer/src/Explainer.tsx b/remotion-composer/src/Explainer.tsx index 797ec4a..a0d8761 100644 --- a/remotion-composer/src/Explainer.tsx +++ b/remotion-composer/src/Explainer.tsx @@ -44,6 +44,97 @@ const { fontFamily } = loadFont("normal", { subsets: ["latin"], }); +// --------------------------------------------------------------------------- +// Animated Background — Gradient Mesh + Floating Orbs +// --------------------------------------------------------------------------- + +const AnimatedBackground: React.FC<{ style?: "fintech" | "default" }> = ({ + style: bgStyle = "default", +}) => { + const frame = useCurrentFrame(); + const { fps, durationInFrames, width, height } = useVideoConfig(); + const progress = frame / durationInFrames; + + // Slow-moving gradient angles + const angle1 = 135 + Math.sin(frame / (fps * 8)) * 30; + const angle2 = 225 + Math.cos(frame / (fps * 6)) * 25; + + // Color stops shift over time + const shift = Math.sin(frame / (fps * 12)) * 0.15; + + const gradient = ` + radial-gradient(ellipse at ${30 + Math.sin(frame / (fps * 10)) * 20}% ${40 + Math.cos(frame / (fps * 8)) * 20}%, + rgba(15, 23, 60, 1) 0%, transparent 60%), + radial-gradient(ellipse at ${70 + Math.cos(frame / (fps * 7)) * 20}% ${60 + Math.sin(frame / (fps * 9)) * 25}%, + rgba(30, 10, 60, 0.8) 0%, transparent 55%), + radial-gradient(ellipse at ${50 + Math.sin(frame / (fps * 14)) * 30}% ${20 + Math.cos(frame / (fps * 11)) * 15}%, + rgba(0, 40, 60, 0.6) 0%, transparent 50%), + linear-gradient(${angle1}deg, #060918 0%, #0B1026 40%, #0F0A2E 70%, #080D1F 100%) + `; + + // Floating orbs + const orbs = [ + { x: 20, y: 30, size: 300, color: "rgba(34, 211, 238, 0.08)", speedX: 7, speedY: 11 }, + { x: 70, y: 60, size: 250, color: "rgba(139, 92, 246, 0.1)", speedX: 9, speedY: 8 }, + { x: 40, y: 80, size: 200, color: "rgba(16, 185, 129, 0.07)", speedX: 13, speedY: 6 }, + { x: 80, y: 20, size: 350, color: "rgba(245, 158, 11, 0.06)", speedX: 11, speedY: 14 }, + { x: 10, y: 70, size: 180, color: "rgba(236, 72, 153, 0.05)", speedX: 8, speedY: 10 }, + ]; + + return ( + + {/* Floating glow orbs */} + {orbs.map((orb, i) => { + const ox = orb.x + Math.sin(frame / (fps * orb.speedX)) * 15; + const oy = orb.y + Math.cos(frame / (fps * orb.speedY)) * 12; + return ( +
+ ); + })} + + {/* Subtle grid overlay */} +
+ + {/* Top gradient fade for depth */} +
+ + ); +}; + // --------------------------------------------------------------------------- // Types — aligned with edit_decisions artifact schema // --------------------------------------------------------------------------- @@ -353,7 +444,9 @@ const SceneRenderer: React.FC<{ cut: Cut }> = ({ cut }) => { }; // Resolve the scene element based on cut type, then wrap with backgroundImage if set - const bgColor = cut.backgroundImage ? "transparent" : cut.backgroundColor; + // Use transparent bg so the animated gradient background shows through + const rawBg = cut.backgroundImage ? "transparent" : cut.backgroundColor; + const bgColor = (rawBg === "#0F172A" || rawBg === "#0f172a") ? "transparent" : rawBg; // Explicit component types if (cut.type === "text_card" && cut.text) { @@ -539,7 +632,10 @@ export const Explainer: React.FC = ({ const { fps, durationInFrames } = useVideoConfig(); return ( - + + {/* Layer 0: Animated gradient background */} + + {/* Layer 1: Visual scenes */} {cuts.map((cut) => { const from = Math.round(cut.in_seconds * fps); diff --git a/remotion-composer/src/Root.tsx b/remotion-composer/src/Root.tsx index 9033d52..2ff270e 100644 --- a/remotion-composer/src/Root.tsx +++ b/remotion-composer/src/Root.tsx @@ -65,13 +65,14 @@ export const Root: React.FC = () => { = { + lower_third: { + position: "absolute", + bottom: 320, // Above caption area (~1600px) + left: 40, + right: 40, + height: 480, + }, + upper_third: { + position: "absolute", + top: 80, + left: 40, + right: 40, + height: 480, + }, + left_panel: { + position: "absolute", + top: 200, + left: 40, + width: 480, + bottom: 400, + }, + right_panel: { + position: "absolute", + top: 200, + right: 40, + width: 480, + bottom: 400, + }, + full_overlay: { + position: "absolute", + top: 0, + left: 0, + right: 0, + bottom: 0, + }, +}; + +// --------------------------------------------------------------------------- +// Overlay component dispatcher — maps overlay type to Remotion component +// --------------------------------------------------------------------------- + +const OverlayContent: React.FC<{ overlay: TalkingHeadOverlay }> = ({ + overlay, +}) => { + const bgColor = overlay.backgroundColor || "#0F172A"; + + if (overlay.type === "text_card" && overlay.text) { + return ( + + ); + } + if (overlay.type === "stat_card" && overlay.stat) { + return ( + + ); + } + if (overlay.type === "callout" && overlay.text) { + return ( + + ); + } + if ( + overlay.type === "comparison" && + overlay.leftLabel && + overlay.rightLabel + ) { + return ( + + ); + } + if (overlay.type === "bar_chart" && overlay.chartData) { + return ( + + ); + } + if (overlay.type === "line_chart" && overlay.chartSeries) { + return ( + + ); + } + if (overlay.type === "pie_chart" && overlay.chartData) { + return ( + + ); + } + if (overlay.type === "kpi_grid" && overlay.chartData) { + return ( + + ); + } + if (overlay.type === "hero_title" && overlay.text) { + return ; + } + if (overlay.type === "section_title" && overlay.text) { + return ( + + ); + } + if (overlay.type === "stat_reveal" && overlay.text) { + return ( + + ); + } + return null; +}; + +// --------------------------------------------------------------------------- +// Positioned overlay wrapper — handles position + fade in/out +// --------------------------------------------------------------------------- + +const PositionedOverlay: React.FC<{ overlay: TalkingHeadOverlay }> = ({ + overlay, +}) => { + const frame = useCurrentFrame(); + const { durationInFrames } = useVideoConfig(); + + // Fade in over 8 frames (~0.27s), fade out over 8 frames + const fadeIn = interpolate(frame, [0, 8], [0, 1], { + extrapolateLeft: "clamp", + extrapolateRight: "clamp", + }); + const fadeOut = interpolate( + frame, + [durationInFrames - 8, durationInFrames], + [1, 0], + { extrapolateLeft: "clamp", extrapolateRight: "clamp" } + ); + const opacity = fadeIn * fadeOut; + + const position = overlay.position || "lower_third"; + const posStyle = POSITION_STYLES[position] || POSITION_STYLES.lower_third; + const isFullOverlay = position === "full_overlay"; + + return ( +
+ {isFullOverlay && ( + + )} + +
+ ); +}; + +// --------------------------------------------------------------------------- +// Main TalkingHead composition +// --------------------------------------------------------------------------- export interface TalkingHeadProps { + [key: string]: unknown; videoSrc: string; captions: WordCaption[]; + overlays?: TalkingHeadOverlay[]; wordsPerPage?: number; fontSize?: number; highlightColor?: string; @@ -12,16 +307,39 @@ export interface TalkingHeadProps { export const TalkingHead: React.FC = ({ videoSrc, captions, + overlays, wordsPerPage = 4, fontSize = 52, highlightColor = "#22D3EE", }) => { + const { fps } = useVideoConfig(); + return ( + {/* Layer 1: Video background */} + + {/* Layer 2: Overlays (charts, stats, callouts, etc.) */} + {overlays?.map((overlay, i) => { + const from = Math.round(overlay.in_seconds * fps); + const duration = Math.round( + (overlay.out_seconds - overlay.in_seconds) * fps + ); + return ( + + + + ); + })} + + {/* Layer 3: Captions (topmost — always visible above overlays) */} use 2x stat_card | string | +| kpi_grid | 720px | YES | numeric ONLY (no "15+") | +| bar_chart | 500px | YES | numeric | +| stat_card | 300px | YES | string OK | +| callout | 400px | YES | string | +| hero_title | 400px | YES | string | +| line_chart | 500px | YES | numeric | +| progress_bar | 600px | YES | numeric | +| stat_reveal | 300px | YES | string OK | + +Key rules: +- `comparison` requires 900px+ width. In 720px portrait frames, substitute with two sequential `stat_card` components instead. +- `kpi_grid` values MUST be purely numeric (e.g., `4.8`, `73`, `2400`). Formatted strings like `"15+"`, `"$4.8B"`, or `"2.4M"` will cause rendering errors. Use `stat_card` for string-formatted numbers instead. +- Always check the target frame width before choosing a component. Portrait (720px) excludes `comparison`. + +**Dark theme for all overlays** -- use dark backgrounds (`#0F172A`, `#1E293B`) with light text. This ensures overlays are legible when composited on top of talking-head footage. The `#0F172A` backgroundColor is treated as transparent by the AnimatedBackground system, so overlays will show the animated gradient behind them. **For simple text overlays** (if Remotion is overkill): diff --git a/skills/pipelines/talking-head/compose-director.md b/skills/pipelines/talking-head/compose-director.md index 1e32dda..4b7b64b 100644 --- a/skills/pipelines/talking-head/compose-director.md +++ b/skills/pipelines/talking-head/compose-director.md @@ -15,6 +15,45 @@ You have edit decisions and an asset manifest. Your job is to render the final t ## Process +### Step 0: Pre-flight Checks + +Before rendering anything, validate the inputs and catch issues that are expensive to fix later. + +1. **Silence detection** -- Run `silence_cutter` in mark mode: + ``` + silence_cutter.execute({ + "input_path": "", + "mode": "mark", + "silence_threshold_db": -35, + "min_silence_duration": 0.5 + }) + ``` + - Report all gaps > 0.5s with timestamps. + - If total silence > 5s, **recommend cutting before proceeding**. Long silences waste render time and produce dead spots in the final video. + +2. **ASR confidence check** -- Scan word-level transcript for low-confidence words: + - Flag any word with probability < 0.7. + - List flagged words with timestamps so the user can verify correct transcription. + - Common misrecognitions to watch for: proper nouns, brand names, domain jargon. + +3. **Auto-build corrections dictionary** from common ASR error patterns: + ```python + corrections = { + # Indian finance context + "DMI": "EMI", + "AMI": "EMI", + # Common brand misspellings + "open montage": "OpenMontage", + "remotion": "Remotion", + # Numbers that got split by ASR + "4 -5": "4-5", + "10 -15": "10-15", + } + ``` + Extend this dict with domain-specific corrections based on the video topic. Present the corrections to the user for review before applying. + +4. **Green screen flag** -- Check if scene-director Step 0 flagged green/blue screen footage. If yes, note that Step 3c (Green Screen Composite) will be needed. + ### Step 1: Run Enhancement Chain Apply video enhancements in this exact order. **Attempt every step** if the tool is available — do not skip steps without a reason. @@ -106,7 +145,14 @@ Pass this dict to both `subtitle_gen` (if generating SRT) and `remotion_caption_ ### Step 3: Burn Subtitles -**Preferred: Remotion captions** (if `remotion_caption_burn` tool available): +**ALWAYS use Remotion TikTok-style captions** (word-by-word highlighting). This is the default and preferred method. Do NOT fall back to FFmpeg ASS subtitles unless Remotion is completely unavailable. + +**Remotion caption requirements:** +- **Auto-detect video dimensions** -- do NOT hardcode width/height. Use `visual_qa` probe or ffprobe to get actual dimensions, then pass them to the render. +- **Set `--frames` based on actual video duration** -- calculate from probe: `frames = duration_seconds * fps`. Never use a hardcoded frame count. +- Word-by-word highlighting with active word color (`highlight_color`). +- Captions positioned at the bottom of frame, away from the face. + ``` remotion_caption_burn.execute({ "input_path": "", @@ -118,15 +164,11 @@ remotion_caption_burn.execute({ "highlight_color": "#22D3EE", }) ``` -Remotion renders animated word-by-word captions at the bottom of the frame with active word highlighting. Captions are positioned away from the face. -**Fallback: FFmpeg subtitles** (if Remotion unavailable): -Use `video_compose` with `burn_subtitles` operation: -- Input: reframed video (or enhanced video if no reframe needed) -- Subtitle file from asset manifest +**Fallback ONLY if Remotion is completely unavailable:** Use `video_compose` with `burn_subtitles` operation. This is a degraded experience -- warn the user that word-by-word highlighting won't be available. -**CRITICAL: Caption positioning for 9:16 vertical video.** -Captions MUST be in the lower 20% of the frame. On a 1920-high frame, that means `MarginV=160` or higher. The default FFmpeg subtitle position is center — this WILL occlude the face. You MUST override it. +**CRITICAL: Caption positioning for 9:16 vertical video (FFmpeg fallback only).** +Captions MUST be in the lower 20% of the frame. On a 1920-high frame, that means `MarginV=160` or higher. The default FFmpeg subtitle position is center -- this WILL occlude the face. You MUST override it. FFmpeg subtitle style string for vertical talking-head: ``` @@ -137,27 +179,152 @@ FFmpeg subtitle style string for vertical talking-head: ### Step 3b: Burn Overlay Graphics (if scene plan includes overlays) -If the scene plan includes overlay scenes (text_cards, stat_cards, charts, comparisons, callouts), render them onto the video. +If the scene plan includes overlay scenes (text_cards, stat_cards, charts, comparisons, callouts), pass them to `remotion_caption_burn` alongside captions. **Both captions and overlays render in a single Remotion pass** — no separate FFmpeg compositing needed. -**How overlay compositing works:** -1. Each overlay is a short Remotion composition (3-5 seconds) rendered as a transparent video clip or composited directly -2. Use `video_compose` with `picture_in_picture` or `overlay` operation to place each overlay at the correct timestamp -3. Respect the overlay's `position` field from the scene plan: - - `lower_third` → bottom 30% of frame - - `upper_third` → top 30% of frame - - `side_panel` → left or right 40% - - `full_overlay` → centered, brief (1-2s) +**How it works:** The TalkingHead Remotion composition renders three layers: +1. **Video** (bottom) — the talking-head footage +2. **Overlays** (middle) — positioned charts, stats, callouts with fade in/out +3. **Captions** (top) — word-by-word highlighting, always visible -**For Remotion-based overlays:** Create a composition JSON with the overlay cuts, render to a transparent clip, then composite onto the talking-head video using FFmpeg. - -**For simple text overlays:** Use FFmpeg's drawtext filter directly: +**Combine Step 3 and 3b into one `remotion_caption_burn` call:** ``` -ffmpeg -i captioned.mp4 -vf "drawtext=text='Key Term':fontsize=48:fontcolor=white:borderw=3:bordercolor=black:x=(w-text_w)/2:y=h*0.75:enable='between(t,22,26)'" -c:a copy output.mp4 +remotion_caption_burn.execute({ + "input_path": "", + "output_path": "/assets/video/captioned.mp4", + "segments": , + "corrections": {"cloud": "Claude"}, + "words_per_page": 4, + "font_size": 52, + "highlight_color": "#22D3EE", + "overlays": [ + { + "id": "term-agentic-ai", + "type": "callout", + "text": "Agentic AI: software that acts autonomously toward goals", + "callout_type": "info", + "in_seconds": 22.0, + "out_seconds": 26.0, + "position": "lower_third", + "backgroundColor": "#0F172A", + "accentColor": "#22D3EE" + }, + { + "id": "stat-market-size", + "type": "stat_card", + "stat": "$4.8B", + "subtitle": "Global AI Agent Market (2026)", + "in_seconds": 35.0, + "out_seconds": 39.0, + "position": "upper_third", + "accentColor": "#A78BFA" + }, + { + "id": "chart-growth", + "type": "bar_chart", + "chartData": [ + {"label": "2023", "value": 1.2}, + {"label": "2024", "value": 2.1}, + {"label": "2025", "value": 3.5}, + {"label": "2026", "value": 4.8} + ], + "title": "AI Agent Market ($B)", + "in_seconds": 40.0, + "out_seconds": 45.0, + "position": "lower_third", + "chartColors": ["#22D3EE", "#A78BFA", "#F472B6", "#34D399"] + } + ] +}) ``` -**Important:** Time each overlay to match the scene plan timestamps. After speed adjustment, recalculate overlay timestamps: `adjusted_time = original_time / speed_factor`. +**Overlay position options:** +- `lower_third` → bottom area, above captions (default — safest for most overlays) +- `upper_third` → top area (good for stats while speaker is center/lower) +- `left_panel` → left 45% of frame (side-by-side with speaker) +- `right_panel` → right 45% of frame +- `full_overlay` → full frame with dark backdrop (use sparingly, 1-2s max) -### Step 3c: Build Showcase Cards (if multi-clip reel) +**Overlay type → required props** (same as asset-director mapping): + +| Type | Required Props | +|------|---------------| +| `text_card` | `text` | +| `stat_card` | `stat`, `subtitle` (optional) | +| `callout` | `text`, `callout_type` (info/warning/tip/quote) | +| `comparison` | `leftLabel`, `rightLabel`, `leftValue`, `rightValue` | +| `bar_chart` | `chartData` (array of `{label, value}`) | +| `line_chart` | `chartSeries` (array of `{name, data: number[]}`) | +| `pie_chart` | `chartData` (array of `{label, value}`) | +| `kpi_grid` | `chartData` (array of `{label, value}`) | +| `hero_title` | `text`, `subtitle` (optional) | +| `section_title` | `text`, `subtitle` (optional) | +| `stat_reveal` | `text` (the stat), `subtitle` (label) | + +**Important:** After speed adjustment, recalculate overlay timestamps: `adjusted_time = original_time / speed_factor`. + +**Fallback (no Remotion):** If Remotion is unavailable, `remotion_caption_burn` falls back to FFmpeg for captions only. Overlays are NOT rendered in FFmpeg fallback mode — warn the user that overlays require Remotion. + +### Step 3c: Green Screen Composite (if green screen footage) + +If the footage has a green/blue screen (detected in scene-director Step 0), follow this pipeline: + +1. **Run `green_screen_processor` tool** to remove the green/blue screen: + ``` + green_screen_processor.execute({ + "input_path": "", + "output_path": "/assets/video/greenscreen_removed.mp4", + "method": "auto" + }) + ``` + The `auto` method detects whether the background is green or blue and applies the appropriate chroma key. + +2. **Render Remotion animated background** using the Explainer composition: + ``` + # Render an AnimatedBackground clip (gradient mesh, floating orbs, subtle grid) + # Use the Explainer composition — NOT a flat #0F172A solid color + npx remotion render src/index.ts Explainer --props='{"duration":VIDEO_DURATION}' --output=/assets/video/animated_bg.mp4 + ``` + The AnimatedBackground provides a professional gradient mesh with floating orbs and a subtle grid pattern. This is far superior to a flat solid color. + +3. **Run `green_screen_composite` tool** to layer the speaker onto the animated background: + ``` + green_screen_composite.execute({ + "foreground_path": "", + "background_path": "", + "output_path": "/assets/video/composited.mp4", + "layout": "news_anchor" + }) + ``` + Default layout is `news_anchor` (speaker center-bottom, background fills frame). Adjust layout based on speaker position detected in Step 0. + +4. **Burn captions via Remotion TalkingHead composition** (NOT FFmpeg ASS subtitles): + ``` + remotion_caption_burn.execute({ + "input_path": "", + "output_path": "/assets/video/captioned.mp4", + "segments": , + "corrections": , + "words_per_page": 4, + "font_size": 52, + "highlight_color": "#22D3EE", + "overlays": + }) + ``` + +5. **Mix background music** (ducked at 15% volume under speech): + ``` + audio_mixer.execute({ + "operation": "duck", + "video_path": "", + "music_path": "", + "music_volume": 0.15, + "output_path": "/assets/video/with_music.mp4" + }) + ``` + +6. **Final encode** to target platform specs (see Step 6 below). + +### Step 3d: Build Showcase Cards (if multi-clip reel) If the output is a reel with showcase clips, use `showcase_card` for each: ``` diff --git a/skills/pipelines/talking-head/scene-director.md b/skills/pipelines/talking-head/scene-director.md index 39e46a3..bdb13cf 100644 --- a/skills/pipelines/talking-head/scene-director.md +++ b/skills/pipelines/talking-head/scene-director.md @@ -18,6 +18,34 @@ You are not just a processor. You are a creative director. Your job is to figure ## Process +### Step 0: Footage Analysis + +Before watching the content, analyze the raw footage to understand the physical setup. + +1. **Sample 5 frames** from the footage using ffmpeg (evenly spaced across the duration): + ``` + ffmpeg -i -vf "select='not(mod(n\,TOTAL_FRAMES/5))'" -vsync vfr -frames:v 5 frame_%02d.png + ``` + +2. **Run visual_qa or histogram analysis** on each sampled frame to detect: + - **Background type:** Green screen / blue screen / natural background. Green/blue screens show a dominant narrow-band color spike in the histogram. Use `visual_qa` with prompt "Is this a green screen or blue screen background?" for confirmation. + - **Speaker position:** Center, left, or right of frame. Estimate the approximate bounding box (e.g., "speaker occupies center 40% of frame, from x=30% to x=70%"). + - **Lighting quality:** Even studio lighting, harsh shadows, backlit, mixed color temperature. Note any issues that may affect chroma keying. + +3. **Green screen detected?** + - If yes, note that `green_screen_processor` tool will be needed in the compose stage. + - Record the detected screen color (green or blue) and estimated uniformity. + - The compose-director will use this to run chroma key removal and composite onto an animated background. + +4. **Measure speaker safe zone:** + - From the speaker's bounding box, determine where graphics can be placed WITHOUT overlapping the speaker. + - For a centered speaker: left panel and right panel are safe for overlays. + - For a left-positioned speaker: right panel is the primary safe zone. + - For a right-positioned speaker: left panel is the primary safe zone. + - Upper third and lower third are generally safe regardless of speaker position. + +Record all findings in the scene plan metadata for downstream stages. + ### Step 1: Watch & Listen — Understand the Content **This is the most important step. Do not skip it.** @@ -52,6 +80,24 @@ Based on your content analysis, propose **on-screen graphics** that will appear | **Callout/quote** | `callout` | When the speaker makes a key point worth emphasizing | | **Lower third** | `text_card` | Speaker identification at the start | +**Remotion Component Constraints:** + +| Component | Min Width | 720px Portrait? | Value Type | +|-----------|-----------|-----------------|------------| +| comparison | 900px | NO -> use 2x stat_card | string | +| kpi_grid | 720px | YES | numeric ONLY (no "15+") | +| bar_chart | 500px | YES | numeric | +| stat_card | 300px | YES | string OK | +| callout | 400px | YES | string | +| hero_title | 400px | YES | string | +| line_chart | 500px | YES | numeric | +| progress_bar | 600px | YES | numeric | +| stat_reveal | 300px | YES | string OK | + +When a component's minimum width exceeds the available space (e.g., `comparison` at 900px won't fit in a 720px portrait frame), substitute with the recommended alternative. For `comparison`, use two `stat_card` components shown sequentially instead. + +For `kpi_grid`, values MUST be numeric (e.g., `4.8`, `73`, `2400`). String values like `"15+"` or `"$4.8B"` will cause rendering errors. Use `stat_card` for string-formatted numbers. + **Overlay planning rules:** - **Don't over-overlay.** 3-6 overlays per minute of final video is the sweet spot. More than that is distracting. - **Time overlays to speech.** Each overlay should appear when the speaker says the relevant words, not before or after. @@ -92,6 +138,8 @@ Format your proposal clearly: **Estimated final duration:** ~Xs (from Xs raw) ``` +**IMPORTANT: When outputting the overlay plan, ALSO generate the actual Remotion JSON props file (`greenscreen-bg.json`) -- do not just describe scenes in prose.** The JSON props file should be a complete, valid input for the Remotion TalkingHead composition, including all overlay definitions, timing, colors, and content. Save it to `/public/demo-props/` or the project's props directory. + Wait for user approval before proceeding. The user may: - Approve as-is - Add/remove overlays diff --git a/tools/video/green_screen_composite.py b/tools/video/green_screen_composite.py new file mode 100644 index 0000000..3ab67b9 --- /dev/null +++ b/tools/video/green_screen_composite.py @@ -0,0 +1,430 @@ +"""Green screen composite tool for talking-head pipeline. + +Composites a keyed speaker (dark/solid background) over a Remotion +background video with layout presets. Supports news anchor, full behind, +picture-in-picture, and split layouts. + +Uses PIL/numpy for frame-level alpha compositing and FFmpeg for +frame extraction, encoding, and audio muxing. +""" + +from __future__ import annotations + +import json +import shutil +import time +from pathlib import Path +from typing import Any + +import numpy as np +from PIL import Image + +from tools.base_tool import ( + BaseTool, + Determinism, + ExecutionMode, + ResourceProfile, + RetryPolicy, + ResumeSupport, + ToolResult, + ToolStability, + ToolTier, +) + + +class GreenScreenComposite(BaseTool): + name = "green_screen_composite" + version = "0.1.0" + tier = ToolTier.CORE + capability = "video_post" + provider = "ffmpeg" + stability = ToolStability.EXPERIMENTAL + execution_mode = ExecutionMode.SYNC + determinism = Determinism.DETERMINISTIC + + dependencies = ["cmd:ffmpeg", "python:numpy", "python:PIL"] + install_instructions = ( + "Install FFmpeg: https://ffmpeg.org/download.html — " + "pip install numpy Pillow" + ) + agent_skills = ["ffmpeg"] + + capabilities = [ + "green_screen_composite", + "speaker_overlay", + "layout_preset", + "alpha_composite", + ] + + input_schema = { + "type": "object", + "required": ["speaker_path", "background_path", "output_path"], + "properties": { + "speaker_path": { + "type": "string", + "description": "Path to keyed speaker video (dark bg, from green_screen_processor)", + }, + "background_path": { + "type": "string", + "description": "Path to Remotion background video", + }, + "output_path": { + "type": "string", + "description": "Output composite video path", + }, + "original_audio_path": { + "type": "string", + "description": "Path to original footage to extract audio from", + }, + "layout": { + "type": "string", + "enum": ["news_anchor", "full_behind", "pip", "split"], + "default": "news_anchor", + "description": ( + "news_anchor=speaker bottom-center over shifted bg, " + "full_behind=speaker full-frame on bg, " + "pip=speaker 30% bottom-right, " + "split=speaker left 50% bg right 50%" + ), + }, + "speaker_scale": { + "type": "number", + "default": 0.65, + "description": "Scale factor for speaker layer", + }, + "bg_shift_up": { + "type": "integer", + "default": 300, + "description": "Pixels to shift background content upward", + }, + "bg_color_hex": { + "type": "string", + "default": "#0E172A", + "description": "The keyed speaker's background color for alpha creation", + }, + }, + } + + resource_profile = ResourceProfile( + cpu_cores=4, ram_mb=4096, vram_mb=0, disk_mb=8000, network_required=False + ) + retry_policy = RetryPolicy(max_retries=1, retryable_errors=["FFmpeg error"]) + resume_support = ResumeSupport.FROM_START + idempotency_key_fields = [ + "speaker_path", "background_path", "layout", + "speaker_scale", "bg_shift_up", "bg_color_hex", + ] + side_effects = ["writes composite video to output_path"] + user_visible_verification = [ + "Watch output — speaker should be cleanly composited without color fringing", + "Check layout positioning matches the chosen preset", + "Verify audio is synced if original_audio_path was provided", + ] + + def execute(self, inputs: dict[str, Any]) -> ToolResult: + speaker_path = Path(inputs["speaker_path"]) + background_path = Path(inputs["background_path"]) + output_path = Path(inputs["output_path"]) + original_audio_path = inputs.get("original_audio_path") + layout = inputs.get("layout", "news_anchor") + speaker_scale = inputs.get("speaker_scale", 0.65) + bg_shift_up = inputs.get("bg_shift_up", 300) + bg_color_hex = inputs.get("bg_color_hex", "#0E172A") + + if not speaker_path.exists(): + return ToolResult(success=False, error=f"Speaker video not found: {speaker_path}") + if not background_path.exists(): + return ToolResult(success=False, error=f"Background video not found: {background_path}") + if original_audio_path and not Path(original_audio_path).exists(): + return ToolResult(success=False, error=f"Audio source not found: {original_audio_path}") + + output_path.parent.mkdir(parents=True, exist_ok=True) + start = time.time() + + # Parse bg color + bg_color = self._parse_hex_color(bg_color_hex) + + # Step 1: Probe both videos + speaker_info = self._probe_video(speaker_path) + bg_info = self._probe_video(background_path) + + if not speaker_info or not bg_info: + return ToolResult( + success=False, + error="Failed to probe one or both input videos", + ) + + # Step 2: Use the LOWER fps (typically 15fps from speaker) + target_fps = min(speaker_info["fps"], bg_info["fps"]) + if target_fps <= 0: + target_fps = 15.0 + + # Determine output dimensions from background + out_w = bg_info["width"] + out_h = bg_info["height"] + + # Use shorter duration + duration = min(speaker_info["duration"], bg_info["duration"]) + + # Step 3: Extract frames from both videos + temp_dir = output_path.parent / ".greenscreen_composite_tmp" + speaker_frames_dir = temp_dir / "speaker" + bg_frames_dir = temp_dir / "bg" + comp_frames_dir = temp_dir / "composite" + + for d in [speaker_frames_dir, bg_frames_dir, comp_frames_dir]: + d.mkdir(parents=True, exist_ok=True) + + try: + self._extract_frames(speaker_path, speaker_frames_dir, target_fps) + self._extract_frames(background_path, bg_frames_dir, target_fps) + + # Get sorted frame lists + speaker_frames = sorted(speaker_frames_dir.glob("*.png")) + bg_frames = sorted(bg_frames_dir.glob("*.png")) + + if not speaker_frames or not bg_frames: + return ToolResult( + success=False, + error="Frame extraction produced no frames", + ) + + frame_count = min(len(speaker_frames), len(bg_frames)) + log_interval = max(1, frame_count // 10) + + # Step 4: Composite each frame pair + for i in range(frame_count): + if i % log_interval == 0: + print(f"[green_screen_composite] Compositing frame {i + 1}/{frame_count}") + + speaker_img = Image.open(speaker_frames[i]).convert("RGB") + bg_img = Image.open(bg_frames[i]).convert("RGB") + + comp = self._composite_frame( + speaker_img, bg_img, bg_color, + layout=layout, + speaker_scale=speaker_scale, + bg_shift_up=bg_shift_up, + out_w=out_w, + out_h=out_h, + ) + comp.save(comp_frames_dir / f"frame_{i:06d}.png") + + print(f"[green_screen_composite] All {frame_count} frames composited") + + # Step 5: Encode composite frames to video + no_audio_path = output_path if not original_audio_path else temp_dir / "no_audio.mp4" + self._encode_frames(comp_frames_dir, no_audio_path, target_fps, out_w, out_h) + + # Step 6: Mux audio if provided + if original_audio_path: + self._mux_audio(no_audio_path, Path(original_audio_path), output_path, duration) + + if not output_path.exists() or output_path.stat().st_size == 0: + return ToolResult(success=False, error="Output video was not created") + + elapsed = time.time() - start + + return ToolResult( + success=True, + data={ + "output": str(output_path), + "layout": layout, + "fps": target_fps, + "frame_count": frame_count, + "duration": round(duration, 2), + "dimensions": f"{out_w}x{out_h}", + "speaker_scale": speaker_scale, + "has_audio": bool(original_audio_path), + }, + artifacts=[str(output_path)], + duration_seconds=round(elapsed, 2), + ) + except Exception as e: + return ToolResult(success=False, error=f"Composite failed: {e}") + finally: + # Step 7: Clean up temp directories + self._cleanup_temp(temp_dir) + + def _parse_hex_color(self, hex_str: str) -> np.ndarray: + """Parse a hex color string like '#0E172A' to an RGB numpy array.""" + hex_str = hex_str.lstrip("#") + r = int(hex_str[0:2], 16) + g = int(hex_str[2:4], 16) + b = int(hex_str[4:6], 16) + return np.array([r, g, b]) + + def _probe_video(self, path: Path) -> dict[str, Any] | None: + """Probe a video for fps, duration, and dimensions.""" + cmd = [ + "ffprobe", "-v", "quiet", + "-print_format", "json", + "-show_format", "-show_streams", + str(path), + ] + try: + result = self.run_command(cmd, timeout=30) + data = json.loads(result.stdout) + except Exception: + return None + + # Find video stream + video_stream = None + for stream in data.get("streams", []): + if stream.get("codec_type") == "video": + video_stream = stream + break + + if not video_stream: + return None + + # Parse fps from r_frame_rate (e.g., "30/1" or "15000/1001") + fps_str = video_stream.get("r_frame_rate", "30/1") + try: + num, den = fps_str.split("/") + fps = float(num) / float(den) + except (ValueError, ZeroDivisionError): + fps = 30.0 + + duration = float(data.get("format", {}).get("duration", 0)) + + return { + "fps": fps, + "duration": duration, + "width": int(video_stream.get("width", 1920)), + "height": int(video_stream.get("height", 1080)), + } + + def _extract_frames(self, video_path: Path, output_dir: Path, fps: float) -> None: + """Extract frames from a video at the given fps.""" + cmd = [ + "ffmpeg", "-y", + "-i", str(video_path), + "-vf", f"fps={fps}", + str(output_dir / "frame_%06d.png"), + ] + self.run_command(cmd, timeout=600) + + def _composite_frame( + self, + speaker_img: Image.Image, + bg_img: Image.Image, + bg_color: np.ndarray, + *, + layout: str, + speaker_scale: float, + bg_shift_up: int, + out_w: int, + out_h: int, + ) -> Image.Image: + """Composite a single speaker frame over a background frame using the given layout.""" + # Create alpha mask from speaker frame + speaker_arr = np.array(speaker_img).astype(float) + dist = np.sqrt(np.sum((speaker_arr - bg_color.astype(float)) ** 2, axis=2)) + threshold = 35 + alpha = np.clip((dist - threshold) * 8, 0, 255).astype(np.uint8) + + speaker_rgba = Image.new("RGBA", speaker_img.size) + speaker_rgba.paste(speaker_img, (0, 0)) + speaker_rgba.putalpha(Image.fromarray(alpha)) + + # Prepare background canvas at output size + canvas = Image.new("RGBA", (out_w, out_h), (0, 0, 0, 255)) + + if layout == "news_anchor": + # Background shifted up so graphics appear above speaker's head + bg_resized = bg_img.resize((out_w, out_h), Image.LANCZOS).convert("RGBA") + # Shift background up: paste it higher so bottom content scrolls up + shifted_bg = Image.new("RGBA", (out_w, out_h), (0, 0, 0, 255)) + shifted_bg.paste(bg_resized, (0, -bg_shift_up)) + canvas = shifted_bg + + # Scale speaker and place at bottom center + sp_w = int(speaker_rgba.width * speaker_scale) + sp_h = int(speaker_rgba.height * speaker_scale) + speaker_scaled = speaker_rgba.resize((sp_w, sp_h), Image.LANCZOS) + x = (out_w - sp_w) // 2 + y = out_h - sp_h + canvas.paste(speaker_scaled, (x, y), speaker_scaled) + + elif layout == "full_behind": + # Speaker full-frame on background, no scaling, no shifting + bg_resized = bg_img.resize((out_w, out_h), Image.LANCZOS).convert("RGBA") + canvas = bg_resized + + # Resize speaker to match output + speaker_full = speaker_rgba.resize((out_w, out_h), Image.LANCZOS) + canvas.paste(speaker_full, (0, 0), speaker_full) + + elif layout == "pip": + # Background full-frame, speaker 30% in bottom-right + bg_resized = bg_img.resize((out_w, out_h), Image.LANCZOS).convert("RGBA") + canvas = bg_resized + + pip_scale = 0.30 + sp_w = int(out_w * pip_scale) + sp_h = int(out_h * pip_scale) + speaker_pip = speaker_rgba.resize((sp_w, sp_h), Image.LANCZOS) + margin = 20 + x = out_w - sp_w - margin + y = out_h - sp_h - margin + canvas.paste(speaker_pip, (x, y), speaker_pip) + + elif layout == "split": + # Speaker on left 50%, background on right 50% + half_w = out_w // 2 + + # Left side: speaker resized to fill left half + speaker_left = speaker_rgba.resize((half_w, out_h), Image.LANCZOS) + # Right side: background cropped/resized to fill right half + bg_right = bg_img.resize((half_w, out_h), Image.LANCZOS).convert("RGBA") + + canvas.paste(speaker_left, (0, 0), speaker_left) + canvas.paste(bg_right, (half_w, 0), bg_right) + + # Convert to RGB for output + return canvas.convert("RGB") + + def _encode_frames( + self, frames_dir: Path, output_path: Path, fps: float, width: int, height: int + ) -> None: + """Encode PNG frames to an MP4 video.""" + cmd = [ + "ffmpeg", "-y", + "-framerate", str(fps), + "-i", str(frames_dir / "frame_%06d.png"), + "-c:v", "libx264", "-crf", "18", "-preset", "fast", + "-pix_fmt", "yuv420p", + "-vf", f"scale={width}:{height}", + str(output_path), + ] + self.run_command(cmd, timeout=600) + + def _mux_audio( + self, video_path: Path, audio_source: Path, output_path: Path, duration: float + ) -> None: + """Mux audio from the original source into the composite video.""" + cmd = [ + "ffmpeg", "-y", + "-i", str(video_path), + "-i", str(audio_source), + "-t", f"{duration:.3f}", + "-c:v", "copy", + "-c:a", "aac", "-b:a", "192k", + "-map", "0:v:0", "-map", "1:a:0", + "-shortest", + str(output_path), + ] + self.run_command(cmd, timeout=300) + + def _cleanup_temp(self, temp_dir: Path) -> None: + """Remove temporary frame directories.""" + if temp_dir.exists(): + try: + shutil.rmtree(temp_dir) + except OSError: + # Best-effort cleanup; log but don't fail + print(f"[green_screen_composite] Warning: could not fully clean {temp_dir}") + + def estimate_runtime(self, inputs: dict[str, Any]) -> float: + return 120.0 diff --git a/tools/video/green_screen_processor.py b/tools/video/green_screen_processor.py new file mode 100644 index 0000000..de07458 --- /dev/null +++ b/tools/video/green_screen_processor.py @@ -0,0 +1,622 @@ +"""Green screen keying processor. + +Removes green/blue screen backgrounds from footage using either FFmpeg +chromakey filtering or rembg AI segmentation. Supports automatic method +detection by analyzing frame color histograms. + +Methods: + - auto: Analyze frames to pick the best method (chromakey vs rembg) + - chromakey: FFmpeg chromakey filter (fast, works well on clean screens) + - rembg: AI background removal via rembg/u2net (slower, handles any bg) +""" + +from __future__ import annotations + +import json +import os +import platform +import shutil +import time +from pathlib import Path +from typing import Any + +from tools.base_tool import ( + BaseTool, + Determinism, + ExecutionMode, + ResourceProfile, + RetryPolicy, + ResumeSupport, + ToolResult, + ToolStability, + ToolTier, +) + + +class GreenScreenProcessor(BaseTool): + name = "green_screen_processor" + version = "0.1.0" + tier = ToolTier.CORE + capability = "video_post" + provider = "ffmpeg" + stability = ToolStability.EXPERIMENTAL + execution_mode = ExecutionMode.SYNC + determinism = Determinism.DETERMINISTIC + + dependencies = ["cmd:ffmpeg"] + install_instructions = ( + "Install FFmpeg: https://ffmpeg.org/download.html " + "For rembg method: pip install rembg[gpu] onnxruntime" + ) + agent_skills = ["ffmpeg"] + + capabilities = [ + "green_screen_keying", + "chromakey", + "background_removal", + "rembg_segmentation", + ] + + input_schema = { + "type": "object", + "required": ["input_path", "output_path"], + "properties": { + "input_path": { + "type": "string", + "description": "Path to raw green screen footage", + }, + "output_path": { + "type": "string", + "description": "Path for keyed output video", + }, + "method": { + "type": "string", + "enum": ["auto", "chromakey", "rembg"], + "default": "auto", + "description": "Keying method: auto detects best approach, chromakey uses FFmpeg, rembg uses AI segmentation", + }, + "fps": { + "type": "integer", + "default": 15, + "description": "Output frames per second", + }, + "bg_color": { + "type": "string", + "default": "#0E172A", + "description": "Hex color for output background", + }, + "max_frames": { + "type": "integer", + "default": 0, + "description": "Limit frames to process (0 = all)", + }, + }, + } + + resource_profile = ResourceProfile( + cpu_cores=4, ram_mb=4096, vram_mb=0, disk_mb=8000, network_required=False + ) + retry_policy = RetryPolicy(max_retries=1, retryable_errors=["FFmpeg error"]) + resume_support = ResumeSupport.FROM_START + idempotency_key_fields = [ + "input_path", "method", "fps", "bg_color", "max_frames", + ] + side_effects = ["writes keyed video to output_path"] + user_visible_verification = [ + "Check output for green fringing around subject edges", + "Verify background is cleanly replaced with target color", + "Look for flickering or inconsistent keying between frames", + ] + + # Platform-specific null device + _null_device = "NUL" if platform.system() == "Windows" else "/dev/null" + + def execute(self, inputs: dict[str, Any]) -> ToolResult: + input_path = Path(inputs["input_path"]) + if not input_path.exists(): + return ToolResult(success=False, error=f"Input not found: {input_path}") + + output_path = Path(inputs["output_path"]) + output_path.parent.mkdir(parents=True, exist_ok=True) + + method = inputs.get("method", "auto") + fps = inputs.get("fps", 15) + bg_color = inputs.get("bg_color", "#0E172A") + max_frames = inputs.get("max_frames", 0) + start = time.time() + + # Step 1: Probe input video + probe = self._probe_video(input_path) + if not probe: + return ToolResult(success=False, error="Failed to probe input video") + + duration = probe["duration"] + width = probe["width"] + height = probe["height"] + src_fps = probe["fps"] + + # Step 2: Determine method + if method == "auto": + method = self._auto_detect_method(input_path, duration, width, height) + + # Step 3: Set up temp directory for frame processing + temp_dir = output_path.parent / f".gs_tmp_{int(time.time())}" + temp_dir.mkdir(parents=True, exist_ok=True) + + try: + # Step 4: Extract frames at target fps + frames_dir = temp_dir / "frames" + frames_dir.mkdir(exist_ok=True) + frame_count = self._extract_frames( + input_path, frames_dir, fps, max_frames + ) + if frame_count == 0: + return ToolResult( + success=False, error="No frames extracted from input" + ) + + # Step 5: Process frames + processed_dir = temp_dir / "processed" + processed_dir.mkdir(exist_ok=True) + + if method == "chromakey": + ok = self._process_chromakey( + frames_dir, processed_dir, bg_color, frame_count + ) + else: + ok = self._process_rembg( + frames_dir, processed_dir, bg_color, frame_count + ) + + if not ok: + return ToolResult( + success=False, + error=f"Frame processing failed with method={method}", + ) + + # Step 6: Reconstruct video from processed frames + self._reconstruct_video(processed_dir, output_path, fps, width, height) + + if not output_path.exists() or output_path.stat().st_size == 0: + return ToolResult( + success=False, error="Output video was not created" + ) + + elapsed = time.time() - start + + return ToolResult( + success=True, + data={ + "method_used": method, + "frame_count": frame_count, + "duration": round(duration, 2), + "output_path": str(output_path), + "resolution": f"{width}x{height}", + "fps": fps, + "bg_color": bg_color, + }, + artifacts=[str(output_path)], + duration_seconds=round(elapsed, 2), + ) + + except Exception as e: + return ToolResult(success=False, error=f"Green screen processing failed: {e}") + finally: + # Clean up temp directory + self._cleanup_dir(temp_dir) + + def _probe_video(self, input_path: Path) -> dict[str, Any] | None: + """Probe video for duration, dimensions, and fps.""" + cmd = [ + "ffprobe", "-v", "quiet", + "-show_entries", "format=duration:stream=width,height,r_frame_rate", + "-select_streams", "v:0", + "-of", "json", + str(input_path), + ] + try: + result = self.run_command(cmd, timeout=30) + data = json.loads(result.stdout) + + duration = float(data.get("format", {}).get("duration", 0)) + + stream = data.get("streams", [{}])[0] + width = int(stream.get("width", 0)) + height = int(stream.get("height", 0)) + + # Parse r_frame_rate like "30/1" or "30000/1001" + fps_str = stream.get("r_frame_rate", "30/1") + if "/" in fps_str: + num, den = fps_str.split("/") + fps_val = float(num) / float(den) if float(den) != 0 else 30.0 + else: + fps_val = float(fps_str) + + return { + "duration": duration, + "width": width, + "height": height, + "fps": fps_val, + } + except Exception: + return None + + def _auto_detect_method( + self, input_path: Path, duration: float, width: int, height: int + ) -> str: + """Analyze sample frames to decide between chromakey and rembg. + + Extracts 5 evenly-spaced frames, checks color histograms for + green/blue screen presence, then tests chromakey quality on a sample. + """ + temp_dir = input_path.parent / f".gs_detect_{int(time.time())}" + temp_dir.mkdir(parents=True, exist_ok=True) + + try: + # Extract 5 sample frames evenly spaced + interval = max(duration / 6, 0.1) + sample_paths = [] + for i in range(5): + ts = interval * (i + 1) + out = temp_dir / f"sample_{i}.png" + cmd = [ + "ffmpeg", "-y", + "-ss", f"{ts:.3f}", + "-i", str(input_path), + "-frames:v", "1", + str(out), + ] + try: + self.run_command(cmd, timeout=30) + if out.exists(): + sample_paths.append(out) + except Exception: + continue + + if not sample_paths: + return "rembg" # fallback if we can't extract samples + + # Analyze color histograms for green/blue screen presence + has_green_screen = self._detect_green_screen_histogram(sample_paths) + + if not has_green_screen: + # No obvious green/blue screen detected, use rembg + return "rembg" + + # Test chromakey on a sample frame and check quality + test_frame = sample_paths[len(sample_paths) // 2] + chromakey_quality = self._test_chromakey_quality(test_frame, temp_dir) + + if chromakey_quality > 80: + return "chromakey" + else: + return "rembg" + + finally: + self._cleanup_dir(temp_dir) + + def _detect_green_screen_histogram(self, sample_paths: list[Path]) -> bool: + """Analyze frames for dominant green or blue channel presence. + + Uses FFmpeg signalstats to measure average hue. A strong green + screen typically has a large area of similar green/blue hue. + """ + green_votes = 0 + for sample in sample_paths: + cmd = [ + "ffmpeg", "-y", + "-i", str(sample), + "-vf", "signalstats=stat=tout+vrep+brng,metadata=mode=print", + "-frames:v", "1", + "-f", "null", self._null_device, + ] + try: + result = self.run_command(cmd, timeout=15) + # Check stderr for color stats + output = result.stderr or "" + + # Alternative: use FFmpeg to count green-ish pixels + # Run a simpler hue check with colorchannelmixer + cmd2 = [ + "ffmpeg", "-y", + "-i", str(sample), + "-vf", ( + "split[a][b];" + "[a]colorchannelmixer=rr=0:gg=1:bb=0," + "threshold=threshold=0.3:similarity=0.3[mask];" + "[mask]blackframe=amount=0:threshold=32" + ), + "-frames:v", "1", + "-f", "null", self._null_device, + ] + # This is complex; use a simpler approach: check raw pixels + # via a green-range filter + cmd_green = [ + "ffmpeg", "-y", + "-i", str(sample), + "-vf", ( + "colorkey=color=0x00FF00:similarity=0.4:blend=0.0," + "alphaextract," + "blackframe=amount=0:threshold=128" + ), + "-frames:v", "1", + "-f", "null", self._null_device, + ] + try: + result2 = self.run_command(cmd_green, timeout=15) + stderr = result2.stderr or "" + # blackframe reports percentage of black pixels + # If many pixels became transparent (black in alpha), there's green + if "pblack:" in stderr: + import re + pblack_matches = re.findall(r"pblack:(\d+)", stderr) + if pblack_matches: + pblack = int(pblack_matches[0]) + if pblack >= 20: + green_votes += 1 + except Exception: + pass + + except Exception: + continue + + # If majority of frames show green screen + return green_votes >= len(sample_paths) // 2 + + def _test_chromakey_quality(self, test_frame: Path, temp_dir: Path) -> float: + """Run chromakey on a test frame and estimate quality percentage. + + Returns a score 0-100 indicating what percentage of the expected + background was successfully keyed out. + """ + keyed_out = temp_dir / "chromakey_test.png" + + # Apply chromakey and output with alpha + cmd = [ + "ffmpeg", "-y", + "-i", str(test_frame), + "-vf", "chromakey=color=0x00FF00:similarity=0.3:blend=0.08", + str(keyed_out), + ] + try: + self.run_command(cmd, timeout=15) + except Exception: + return 0.0 + + if not keyed_out.exists(): + return 0.0 + + # Count transparent pixels via alphaextract + blackframe + cmd2 = [ + "ffmpeg", "-y", + "-i", str(keyed_out), + "-vf", "alphaextract,blackframe=amount=0:threshold=32", + "-frames:v", "1", + "-f", "null", self._null_device, + ] + try: + result = self.run_command(cmd2, timeout=15) + stderr = result.stderr or "" + import re + pblack_matches = re.findall(r"pblack:(\d+)", stderr) + if pblack_matches: + # pblack = percentage of black pixels in alpha = transparent pixels + return float(pblack_matches[0]) + except Exception: + pass + + return 0.0 + + def _extract_frames( + self, input_path: Path, frames_dir: Path, fps: int, max_frames: int + ) -> int: + """Extract frames from video at target fps.""" + cmd = [ + "ffmpeg", "-y", + "-i", str(input_path), + "-vf", f"fps={fps}", + str(frames_dir / "frame_%06d.png"), + ] + + if max_frames > 0: + cmd.insert(-1, "-frames:v") + cmd.insert(-1, str(max_frames)) + + try: + self.run_command(cmd, timeout=600) + except Exception as e: + # ffmpeg may return non-zero but still produce frames + pass + + # Count extracted frames + frame_files = sorted(frames_dir.glob("frame_*.png")) + count = len(frame_files) + + if count > 0: + # Log progress for large frame counts + if count > 100: + print(f"[green_screen_processor] Extracted {count} frames") + + return count + + def _process_chromakey( + self, + frames_dir: Path, + processed_dir: Path, + bg_color: str, + frame_count: int, + ) -> bool: + """Process frames using FFmpeg chromakey filter. + + Applies chromakey to remove green, then composites onto bg_color. + """ + bg_hex = bg_color.lstrip("#") + # Convert hex to FFmpeg color format + ffmpeg_bg = f"0x{bg_hex}" + + frame_files = sorted(frames_dir.glob("frame_*.png")) + processed = 0 + + for i, frame in enumerate(frame_files): + out_path = processed_dir / frame.name + cmd = [ + "ffmpeg", "-y", + "-f", "lavfi", "-i", f"color=c={ffmpeg_bg}:size=1x1", + "-i", str(frame), + "-filter_complex", + ( + f"[0:v]scale=iw:ih[bg];" + f"[1:v]chromakey=color=0x00FF00:similarity=0.3:blend=0.08[fg];" + f"[bg][fg]overlay=0:0" + ), + "-frames:v", "1", + str(out_path), + ] + try: + self.run_command(cmd, timeout=30) + if out_path.exists(): + processed += 1 + except Exception: + # Try with the frame size explicitly to fix scale + try: + cmd_retry = [ + "ffmpeg", "-y", + "-i", str(frame), + "-vf", + f"chromakey=color=0x00FF00:similarity=0.3:blend=0.08," + f"split[fg][alpha];" + f"[alpha]alphaextract[a];" + f"color=c={ffmpeg_bg}[bg];" + f"[bg][fg][a]maskedmerge", + "-frames:v", "1", + str(out_path), + ] + # Simpler fallback: just apply chromakey without compositing + cmd_simple = [ + "ffmpeg", "-y", + "-i", str(frame), + "-vf", f"chromakey=color=0x00FF00:similarity=0.3:blend=0.08", + str(out_path), + ] + self.run_command(cmd_simple, timeout=30) + if out_path.exists(): + processed += 1 + except Exception: + continue + + if frame_count > 100 and (i + 1) % 50 == 0: + print( + f"[green_screen_processor] Chromakey: {i + 1}/{frame_count} frames" + ) + + return processed > 0 + + def _process_rembg( + self, + frames_dir: Path, + processed_dir: Path, + bg_color: str, + frame_count: int, + ) -> bool: + """Process frames using rembg AI segmentation. + + Removes background with u2net_human_seg model, then composites + the subject onto bg_color background. + """ + try: + import rembg + from PIL import Image + except ImportError: + return False + + # Parse bg_color hex to RGB + bg_hex = bg_color.lstrip("#") + bg_r = int(bg_hex[0:2], 16) + bg_g = int(bg_hex[2:4], 16) + bg_b = int(bg_hex[4:6], 16) + + session = rembg.new_session("u2net_human_seg") + + frame_files = sorted(frames_dir.glob("frame_*.png")) + processed = 0 + + for i, frame in enumerate(frame_files): + try: + img = Image.open(frame).convert("RGB") + import numpy as np + + # Remove background (returns RGBA) + result = rembg.remove( + np.array(img), + session=session, + ) + result_img = Image.fromarray(result) + + # Composite onto bg_color background + bg = Image.new("RGBA", result_img.size, (bg_r, bg_g, bg_b, 255)) + bg.paste(result_img, (0, 0), result_img) + + # Save as RGB + out_path = processed_dir / frame.name + bg.convert("RGB").save(out_path) + processed += 1 + + except Exception: + continue + + if frame_count > 100 and (i + 1) % 50 == 0: + print( + f"[green_screen_processor] rembg: {i + 1}/{frame_count} frames" + ) + + return processed > 0 + + def _reconstruct_video( + self, + frames_dir: Path, + output_path: Path, + fps: int, + width: int, + height: int, + ) -> None: + """Reconstruct video from processed frames using FFmpeg.""" + cmd = [ + "ffmpeg", "-y", + "-framerate", str(fps), + "-i", str(frames_dir / "frame_%06d.png"), + "-vf", f"scale={width}:{height}:flags=lanczos", + "-c:v", "libx264", + "-crf", "18", + "-preset", "fast", + "-pix_fmt", "yuv420p", + str(output_path), + ] + self.run_command(cmd, timeout=600) + + @staticmethod + def _cleanup_dir(dir_path: Path) -> None: + """Recursively remove a temp directory.""" + if not dir_path.exists(): + return + try: + shutil.rmtree(dir_path) + except OSError: + # Best-effort cleanup; individual file removal as fallback + for f in dir_path.rglob("*"): + try: + if f.is_file(): + f.unlink() + except OSError: + pass + try: + dir_path.rmdir() + except OSError: + pass + + def estimate_runtime(self, inputs: dict[str, Any]) -> float: + method = inputs.get("method", "auto") + if method == "rembg": + return 120.0 + elif method == "chromakey": + return 30.0 + return 60.0 # auto diff --git a/tools/video/remotion_caption_burn.py b/tools/video/remotion_caption_burn.py index 4c84c66..d3dc8b0 100644 --- a/tools/video/remotion_caption_burn.py +++ b/tools/video/remotion_caption_burn.py @@ -106,6 +106,19 @@ class RemotionCaptionBurn(BaseTool): "correct replacement. Example: {\"cloud\": \"Claude\"}." ), }, + "overlays": { + "type": "array", + "description": ( + "Array of overlay objects to render on top of the video. " + "Each overlay has: type (text_card, stat_card, callout, " + "comparison, bar_chart, line_chart, pie_chart, kpi_grid, " + "hero_title, section_title, stat_reveal), in_seconds, " + "out_seconds, position (lower_third, upper_third, " + "left_panel, right_panel, full_overlay), and component-" + "specific props (text, stat, chartData, etc.). " + "See asset_manifest overlays from the asset-director." + ), + }, "force_ffmpeg": { "type": "boolean", "default": False, @@ -245,6 +258,7 @@ class RemotionCaptionBurn(BaseTool): words_per_page: int, font_size: int, highlight_color: str, + overlays: list[dict] | None = None, ) -> ToolResult: root = self._find_remotion_root() if root is None: @@ -262,6 +276,19 @@ class RemotionCaptionBurn(BaseTool): duration_s = float(dur_out.strip().split("\n")[0]) total_frames = math.ceil(duration_s * 30) + # Detect video dimensions + dim_cmd = [ + "ffprobe", "-v", "error", + "-select_streams", "v:0", + "-show_entries", "stream=width,height", + "-of", "csv=p=0:s=x", + input_path, + ] + dim_result = self.run_command(dim_cmd) + dim_parts = dim_result.stdout.strip().split("x") + width = int(dim_parts[0]) + height = int(dim_parts[1]) + # Copy video to Remotion public folder pub_dir = root / "public" / "talking-head" pub_dir.mkdir(parents=True, exist_ok=True) @@ -273,6 +300,7 @@ class RemotionCaptionBurn(BaseTool): props = { "videoSrc": f"public/talking-head/{video_filename}", "captions": captions, + "overlays": overlays or [], "wordsPerPage": words_per_page, "fontSize": font_size, "highlightColor": highlight_color, @@ -287,12 +315,12 @@ class RemotionCaptionBurn(BaseTool): npx_bin = "npx.cmd" if sys.platform == "win32" else "npx" render_cmd = [ npx_bin, "remotion", "render", - "src/index.tsx", "TalkingHead", + "TalkingHead", f"--props={props_file.relative_to(root)}", - "--width=1080", "--height=1920", "--fps=30", + f"--width={width}", f"--height={height}", "--fps=30", f"--frames=0-{total_frames - 1}", "--codec=h264", "--crf=18", - str(Path(output_path).resolve()), + f"--output={str(Path(output_path).resolve())}", ] self.run_command(render_cmd, cwd=str(root)) @@ -307,6 +335,7 @@ class RemotionCaptionBurn(BaseTool): "duration_seconds": round(duration_s, 2), "total_frames": total_frames, "caption_count": len(captions), + "overlay_count": len(overlays or []), "words_per_page": words_per_page, }, artifacts=[output_path], @@ -429,11 +458,14 @@ class RemotionCaptionBurn(BaseTool): if not captions: return ToolResult(success=False, error="No caption words extracted.") + overlays = inputs.get("overlays") + # Choose render method if not force_ffmpeg and self._remotion_available(): result = self._render_remotion( input_path, output_path, captions, words_per_page, font_size, highlight_color, + overlays=overlays, ) else: result = self._render_ffmpeg(input_path, output_path, captions)