From 7b40edb82c3d21e2f31c9a85ec49b6c70c4b03bd Mon Sep 17 00:00:00 2001 From: calesthio Date: Sat, 4 Apr 2026 14:23:46 -0700 Subject: [PATCH] Fix gap #22 and add CinematicRenderer captions + music support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - AGENT_GUIDE.md: add "never read source code" rule — skills are the interface, not .py files - animation.yaml: add Layer 2 skill-first guardrail in assets stage - video-reference-analyst.md: Step 4b now mandates Layer 2 before Layer 3, explicitly forbids reading implementation code - CinematicRenderer: add TikTok-style CaptionOverlay and separate music track support (narration + music as independent audio layers) - cinematic/types.ts: add CinematicCaptionConfig and music prop types --- AGENT_GUIDE.md | 6 ++-- pipeline_defs/animation.yaml | 1 + remotion-composer/src/CinematicRenderer.tsx | 33 +++++++++++++++++++-- remotion-composer/src/cinematic/types.ts | 17 +++++++++++ skills/meta/video-reference-analyst.md | 11 +++++-- 5 files changed, 60 insertions(+), 8 deletions(-) diff --git a/AGENT_GUIDE.md b/AGENT_GUIDE.md index 496c4a9..74dd4c7 100644 --- a/AGENT_GUIDE.md +++ b/AGENT_GUIDE.md @@ -580,8 +580,10 @@ OpenMontage has three instruction layers: Reading order: 1. registry / tool contract — discover what's available -2. relevant pipeline or creative skill — know HOW to use it in this context -3. underlying vendor skill — **mandatory before calling any generation tool** +2. relevant pipeline or creative skill (Layer 2) — know HOW to use it in this context +3. underlying vendor skill (Layer 3) — **mandatory before calling any generation tool** + +**NEVER read tool source code (`tools/*.py`) to understand how to use a tool.** Skills exist precisely so you don't need implementation details. Layer 2 tells you *what* and *when*. Layer 3 tells you *how*. If you're reading `.py` files to figure out input schemas or provider options, you're doing it wrong — that information belongs in the skill layer. **Layer 3 is not optional.** Every generation tool (video, image, TTS, music) has an `agent_skills` field listing its Layer 3 skills. These skills contain provider-specific prompt engineering, parameter tuning, and quality techniques. Read them before writing prompts. The difference between a generic prompt and a skill-informed prompt is the difference between "usable" and "cinematic." diff --git a/pipeline_defs/animation.yaml b/pipeline_defs/animation.yaml index d0bbc37..2aab5f9 100644 --- a/pipeline_defs/animation.yaml +++ b/pipeline_defs/animation.yaml @@ -198,6 +198,7 @@ stages: - "Layer 3 skills read for EVERY generation tool before writing prompts (check agent_skills field)" - "Clip duration maximized (prefer 10s over 5s) to reduce API calls and cost" - "Use selector tools (video_selector, tts_selector, image_selector) — never call provider tools directly" + - "Read Layer 2 skills for each tool BEFORE reading source code — skills contain usage guidance, source code is implementation detail" success_criteria: - Schema-valid asset_manifest artifact - All referenced asset files exist on disk diff --git a/remotion-composer/src/CinematicRenderer.tsx b/remotion-composer/src/CinematicRenderer.tsx index ac4d5c8..e021994 100644 --- a/remotion-composer/src/CinematicRenderer.tsx +++ b/remotion-composer/src/CinematicRenderer.tsx @@ -24,6 +24,7 @@ function resolveAsset(src: string): string { return staticFile(clean); } import { CinematicRendererProps, CinematicTone, CinematicVideoScene } from "./cinematic/types"; +import { CaptionOverlay } from "./components/CaptionOverlay"; const FPS = 30; @@ -339,19 +340,34 @@ export const CinematicRenderer: React.FC = ({ titleWidth = 1320, signalLineCount = 18, soundtrack, + music, + captions, }) => { return ( + {/* Layer 1: Narration audio */} {soundtrack ? ( ) : null} + {/* Layer 2: Music bed (separate track, ducked) */} + {music ? ( + + ) : null} + {/* Layer 3: Video scenes */} {scenes.map((scene) => ( = ({ )} ))} + {/* Layer 4: TikTok-style captions */} + {captions?.words ? ( + + ) : null} ); }; diff --git a/remotion-composer/src/cinematic/types.ts b/remotion-composer/src/cinematic/types.ts index 10db753..1412a64 100644 --- a/remotion-composer/src/cinematic/types.ts +++ b/remotion-composer/src/cinematic/types.ts @@ -35,6 +35,21 @@ export interface CinematicSoundtrack { fadeOutSeconds?: number; } +export interface CinematicWordCaption { + word: string; + startMs: number; + endMs: number; +} + +export interface CinematicCaptionConfig { + words: CinematicWordCaption[]; + wordsPerPage?: number; + fontSize?: number; + color?: string; + highlightColor?: string; + backgroundColor?: string; +} + export interface CinematicRendererProps { [key: string]: unknown; scenes: CinematicScene[]; @@ -42,4 +57,6 @@ export interface CinematicRendererProps { titleWidth?: number; signalLineCount?: number; soundtrack?: CinematicSoundtrack; + music?: CinematicSoundtrack; + captions?: CinematicCaptionConfig; } diff --git a/skills/meta/video-reference-analyst.md b/skills/meta/video-reference-analyst.md index 7047fa4..d58ff8a 100644 --- a/skills/meta/video-reference-analyst.md +++ b/skills/meta/video-reference-analyst.md @@ -269,9 +269,14 @@ the user paralyzed with equal choices. **Before ANY asset generation** (sample or full production), the agent MUST: -1. Check the `agent_skills` field on every tool that will be used -2. Read each referenced skill in `.agents/skills/` -3. Apply the provider-specific prompting guidance to all generation prompts +1. Read the **Layer 2 skill** for each tool from `skills/` directory (usage guidance, input schemas, best practices) +2. Check the `agent_skills` field on every tool that will be used +3. Read each referenced **Layer 3 skill** in `.agents/skills/` (provider-specific prompting) +4. Apply the provider-specific prompting guidance to all generation prompts + +**NEVER read tool source code (*.py) to understand how to use a tool.** +Skills exist precisely so the agent doesn't need to read implementation code. +Layer 2 skills describe *what* and *when*. Layer 3 skills describe *how*. This is NOT optional. The AGENT_GUIDE says: *"Layer 3 is not optional. Every generation tool has an agent_skills field. Read them before writing