diff --git a/README.md b/README.md index 0a83093..ab99259 100644 --- a/README.md +++ b/README.md @@ -248,6 +248,7 @@ Each pipeline is a complete production workflow, from idea to finished video. | **Avatar Spokesperson** | Avatar-driven presenter videos | Corporate comms, training, announcements | | **Cinematic** | Trailer, teaser, and mood-driven edits | Brand films, teasers, promotional content | | **Clip Factory** | Batch of ranked short-form clips from one long source | Repurposing long content for social media | +| **Documentary Montage** | Thematic montage cut from a CLIP-indexed corpus of free stock footage (Pexels, Archive.org, NASA) | Video essays, mood pieces, retrieval-first B-roll edits | | **Hybrid** | Source footage + AI-generated support visuals | Enhancing existing footage with graphics | | **Localization & Dub** | Subtitle, dub, and translate existing video | Multi-language distribution | | **Podcast Repurpose** | Podcast highlights to video | Podcast marketing, audiogram videos | @@ -272,7 +273,7 @@ Most AI video tools give you a single clip from a prompt. OpenMontage gives you Edit your own talking-head footage. Generate a fully animated explainer from scratch. Cut a 2-hour podcast into a dozen social clips. Translate and dub your content into 10 languages. Build a cinematic brand teaser from stock footage and AI-generated scenes. **If a production team can make it, OpenMontage can orchestrate it.** -- **11 production pipelines** — explainers, talking heads, screen demos, cinematic trailers, animations, podcasts, localization, and more +- **12 production pipelines** — explainers, talking heads, screen demos, cinematic trailers, animations, podcasts, localization, documentary montages, and more - **52 production tools** — spanning video generation, image creation, text-to-speech, music, audio mixing, subtitles, enhancement, and analysis - **400+ agent skills** — production skills, pipeline directors, creative techniques, quality checklists, and deep technology knowledge packs that teach the agent how to use every tool like an expert - **Reference-driven creation** — paste a video you like and the agent turns it into a grounded, differentiated production plan instead of forcing you to invent the perfect prompt from scratch diff --git a/lib/clip_embedder.py b/lib/clip_embedder.py new file mode 100644 index 0000000..b9ee82e --- /dev/null +++ b/lib/clip_embedder.py @@ -0,0 +1,136 @@ +"""CLIP embedder: thin wrapper around openai/clip-vit-base-patch32 for +corpus indexing and text-to-visual similarity ranking. + +Design notes +------------ +This module intentionally does ONE thing: turn images and text into +normalised 512-d float32 vectors that can be cosine-compared. + +- Single shared model instance, lazy-loaded on first call, so the 350 MB + weights only load once per process regardless of how many places in + the codebase embed something. +- CPU by default, GPU if available. The ViT-B/32 variant runs at + ~150-300 ms per image on a modern CPU — fast enough for corpora of + a few hundred candidates without needing FAISS. +- Output vectors are L2-normalised so cosine similarity reduces to a + dot product — downstream code can `embeddings @ query_vec.T` and + interpret it as cosine similarity directly. +- Batched at the caller's request count; no internal mini-batching. + For corpora > a few hundred items, the caller should chunk. + +This file does NOT decide what to embed or how to use the embeddings. +That intelligence lives in the corpus manager and retrieval skills. +""" +from __future__ import annotations + +from pathlib import Path +from typing import Iterable, Sequence, Union + +import numpy as np + +# Import heavy deps lazily inside methods so importing this module does +# not pull torch/transformers unless someone actually uses it. + + +_MODEL = None +_PROCESSOR = None +_DEVICE: str = "cpu" +_MODEL_ID = "openai/clip-vit-base-patch32" + + +def _load() -> None: + """Load CLIP model and processor exactly once per process.""" + global _MODEL, _PROCESSOR, _DEVICE + if _MODEL is not None: + return + import torch # type: ignore + from transformers import CLIPModel, CLIPProcessor # type: ignore + + _DEVICE = "cuda" if torch.cuda.is_available() else "cpu" + _PROCESSOR = CLIPProcessor.from_pretrained(_MODEL_ID) + _MODEL = CLIPModel.from_pretrained(_MODEL_ID).to(_DEVICE) + _MODEL.eval() + + +def model_info() -> dict: + """Return metadata about the loaded model (for index provenance).""" + return { + "model_id": _MODEL_ID, + "device": _DEVICE, + "dim": 512, + } + + +def embed_images(image_paths: Sequence[Union[str, Path]]) -> np.ndarray: + """Embed a list of image files into a (N, 512) float32 matrix. + + Each row is L2-normalised. + """ + if not image_paths: + return np.zeros((0, 512), dtype=np.float32) + + import torch # type: ignore + from PIL import Image # type: ignore + + _load() + assert _MODEL is not None and _PROCESSOR is not None + + images = [] + for p in image_paths: + img = Image.open(str(p)).convert("RGB") + images.append(img) + + inputs = _PROCESSOR(images=images, return_tensors="pt").to(_DEVICE) + with torch.no_grad(): + features = _MODEL.get_image_features(**inputs) + features = features / features.norm(dim=-1, keepdim=True).clamp_min(1e-8) + arr = features.cpu().numpy().astype(np.float32, copy=False) + # Close PIL handles to avoid leaking file handles on Windows + for img in images: + img.close() + return arr + + +def embed_texts(texts: Sequence[str]) -> np.ndarray: + """Embed a list of text strings into a (N, 512) float32 matrix. + + Each row is L2-normalised. + """ + if not texts: + return np.zeros((0, 512), dtype=np.float32) + + import torch # type: ignore + + _load() + assert _MODEL is not None and _PROCESSOR is not None + + # Empty strings break the processor — substitute a placeholder so + # the alignment with caller indices stays intact. + safe_texts = [t if t and t.strip() else "untitled" for t in texts] + + inputs = _PROCESSOR( + text=safe_texts, + return_tensors="pt", + padding=True, + truncation=True, + max_length=77, + ).to(_DEVICE) + with torch.no_grad(): + features = _MODEL.get_text_features(**inputs) + features = features / features.norm(dim=-1, keepdim=True).clamp_min(1e-8) + return features.cpu().numpy().astype(np.float32, copy=False) + + +def pool_frames(frame_embeddings: np.ndarray) -> np.ndarray: + """Average a (K, 512) stack of frame embeddings into a (512,) clip vector. + + Re-normalises after the mean. This is the simplest temporal pooling + that still respects the L2 assumption the rest of the pipeline makes. + """ + if frame_embeddings.size == 0: + return np.zeros(512, dtype=np.float32) + mean = frame_embeddings.mean(axis=0) + norm = np.linalg.norm(mean) + if norm < 1e-8: + return np.zeros(512, dtype=np.float32) + return (mean / norm).astype(np.float32, copy=False) diff --git a/lib/corpus.py b/lib/corpus.py new file mode 100644 index 0000000..4aefd52 --- /dev/null +++ b/lib/corpus.py @@ -0,0 +1,424 @@ +"""Local clip corpus: project-scoped index of candidate video/image assets. + +The corpus is the heart of the documentary-montage pipeline. Instead of +hitting stock APIs every time the agent changes its mind, we download +once into a project-local corpus and query it offline. + +Directory layout +---------------- + / + clips/ # downloaded assets, named . + thumbnails/ + / + frame_00.jpg # 5 evenly-spaced frames per video (or the + frame_01.jpg # image itself as frame_00 for still assets) + ... + embeddings.npy # (N, 512) float32, L2-normalised visual + tag_embeddings.npy # (N, 512) float32, L2-normalised text + index.jsonl # one row per clip, metadata + provenance + +The JSONL + .npy split is intentional: the index is human-readable +(git-diffable, debuggable) while the embeddings are a contiguous array +for fast vector math. Row N in the JSONL aligns with row N in both .npy +files. + +The Corpus class owns add/load/save + all retrieval math. It's pure +infrastructure — no judgment calls, no creative decisions. The agent +picks WHAT to search for and WHICH results to accept; this class only +implements the operations the agent names. +""" +from __future__ import annotations + +import json +import time +from dataclasses import asdict, dataclass, field +from pathlib import Path +from typing import Any, Iterable, Optional + +import numpy as np + + +EMBED_DIM = 512 + + +@dataclass +class ClipRecord: + """One row in the corpus index. + + Fields mirror everything we want to query or attribute. Missing + fields default to None/empty so adapters can populate only what + they know. + """ + clip_id: str # unique within corpus: "_" + source: str # "pexels", "archive_org", "nasa", ... + source_id: str + source_url: str + local_path: str # relative to corpus_dir + kind: str = "video" # "video" or "image" + thumb_dir: str = "" # relative to corpus_dir + query: str = "" # the search query that surfaced this clip + creator: str = "" + license: str = "" + duration: float = 0.0 # seconds (0 for images) + width: int = 0 + height: int = 0 + motion_score: float = 0.0 # residual optical flow magnitude + dominant_colors: list[list[int]] = field(default_factory=list) + source_tags: str = "" # raw tags/description from the API + shot_type: str = "" # wide/medium/close (optional, may be empty) + time_of_day: str = "" # day/golden/night (optional) + added_at: float = 0.0 # unix timestamp + + +class Corpus: + """Append-only local clip corpus with vector search. + + Usage:: + + corp = Corpus(Path("projects/foo/corpus")) + corp.load() + corp.add(record, clip_embedding, tag_embedding) + corp.save() + ranked = corp.rank_by_text("a lonely figure walking home") + neighbours = corp.knn(clip_id="pexels_12345", k=5) + + Append-only is by design — deleting rows would break the row-to- + embedding alignment. The agent hides a clip by filtering at query + time, not by removing it from the corpus. + """ + + def __init__(self, corpus_dir: Path): + self.corpus_dir = Path(corpus_dir) + self.records: list[ClipRecord] = [] + self.clip_embeddings: np.ndarray = np.zeros((0, EMBED_DIM), dtype=np.float32) + self.tag_embeddings: np.ndarray = np.zeros((0, EMBED_DIM), dtype=np.float32) + self._id_to_row: dict[str, int] = {} + + # ------------------------------------------------------------------ + # Paths + # ------------------------------------------------------------------ + + @property + def clips_dir(self) -> Path: + return self.corpus_dir / "clips" + + @property + def thumbs_dir(self) -> Path: + return self.corpus_dir / "thumbnails" + + @property + def index_path(self) -> Path: + return self.corpus_dir / "index.jsonl" + + @property + def embed_path(self) -> Path: + return self.corpus_dir / "embeddings.npy" + + @property + def tag_embed_path(self) -> Path: + return self.corpus_dir / "tag_embeddings.npy" + + def ensure_dirs(self) -> None: + self.corpus_dir.mkdir(parents=True, exist_ok=True) + self.clips_dir.mkdir(exist_ok=True) + self.thumbs_dir.mkdir(exist_ok=True) + + # ------------------------------------------------------------------ + # Persistence + # ------------------------------------------------------------------ + + def load(self) -> None: + """Load existing corpus from disk. Silently starts empty if absent.""" + self.records = [] + self._id_to_row = {} + if self.index_path.is_file(): + with open(self.index_path, encoding="utf-8") as f: + for i, line in enumerate(f): + line = line.strip() + if not line: + continue + data = json.loads(line) + rec = ClipRecord(**data) + self.records.append(rec) + self._id_to_row[rec.clip_id] = i + + if self.embed_path.is_file(): + self.clip_embeddings = np.load(self.embed_path) + else: + self.clip_embeddings = np.zeros((0, EMBED_DIM), dtype=np.float32) + + if self.tag_embed_path.is_file(): + self.tag_embeddings = np.load(self.tag_embed_path) + else: + self.tag_embeddings = np.zeros((0, EMBED_DIM), dtype=np.float32) + + # Sanity check: if JSONL and .npy drifted out of sync (e.g. crash + # mid-add), we truncate to the shorter length so subsequent adds + # don't alias to the wrong rows. + n = min(len(self.records), self.clip_embeddings.shape[0], self.tag_embeddings.shape[0]) + if n != len(self.records): + self.records = self.records[:n] + self._id_to_row = {r.clip_id: i for i, r in enumerate(self.records)} + if self.clip_embeddings.shape[0] != n: + self.clip_embeddings = self.clip_embeddings[:n] + if self.tag_embeddings.shape[0] != n: + self.tag_embeddings = self.tag_embeddings[:n] + + def save(self) -> None: + """Persist index + both embedding stacks atomically-ish.""" + self.ensure_dirs() + + # JSONL first — if a crash interrupts the .npy writes the JSONL + # is the source of truth for what exists, and load() will + # auto-truncate the embeddings to match. + tmp_index = self.index_path.with_suffix(".jsonl.tmp") + with open(tmp_index, "w", encoding="utf-8") as f: + for rec in self.records: + f.write(json.dumps(asdict(rec)) + "\n") + tmp_index.replace(self.index_path) + + np.save(self.embed_path, self.clip_embeddings) + np.save(self.tag_embed_path, self.tag_embeddings) + + # ------------------------------------------------------------------ + # Mutation + # ------------------------------------------------------------------ + + def has(self, clip_id: str) -> bool: + return clip_id in self._id_to_row + + def add( + self, + record: ClipRecord, + clip_embedding: np.ndarray, + tag_embedding: np.ndarray, + ) -> None: + """Append a new clip to the corpus. Idempotent by clip_id.""" + if record.clip_id in self._id_to_row: + return + if clip_embedding.shape != (EMBED_DIM,): + raise ValueError( + f"clip_embedding must be ({EMBED_DIM},), got {clip_embedding.shape}" + ) + if tag_embedding.shape != (EMBED_DIM,): + raise ValueError( + f"tag_embedding must be ({EMBED_DIM},), got {tag_embedding.shape}" + ) + if record.added_at == 0.0: + record.added_at = time.time() + + idx = len(self.records) + self.records.append(record) + self._id_to_row[record.clip_id] = idx + + self.clip_embeddings = np.vstack( + [self.clip_embeddings, clip_embedding.reshape(1, -1).astype(np.float32)] + ) + self.tag_embeddings = np.vstack( + [self.tag_embeddings, tag_embedding.reshape(1, -1).astype(np.float32)] + ) + + def get(self, clip_id: str) -> Optional[ClipRecord]: + idx = self._id_to_row.get(clip_id) + if idx is None: + return None + return self.records[idx] + + def __len__(self) -> int: + return len(self.records) + + # ------------------------------------------------------------------ + # Vector math (the named retrieval operations) + # ------------------------------------------------------------------ + + def _fused_sims(self, query_vec: np.ndarray, tag_weight: float) -> np.ndarray: + """Cosine similarity fused across visual and tag channels. + + Both embedding banks are L2-normalised, so `bank @ query` is + cosine similarity directly. Fused score = (1 - w) * visual + + w * tag, where w is the tag weight (default 0.3 at call sites). + """ + if self.clip_embeddings.shape[0] == 0: + return np.zeros(0, dtype=np.float32) + visual = self.clip_embeddings @ query_vec.astype(np.float32) + tag = self.tag_embeddings @ query_vec.astype(np.float32) + return (1.0 - tag_weight) * visual + tag_weight * tag + + def rank_by_text( + self, + query_embedding: np.ndarray, + k: int = 20, + tag_weight: float = 0.3, + motion_min: Optional[float] = None, + kind: Optional[str] = None, + exclude_ids: Optional[Iterable[str]] = None, + ) -> list[tuple[ClipRecord, float]]: + """Return the top-k records scored against an embedded text query. + + Args: + query_embedding: (512,) L2-normalised text embedding. + k: how many results to return. + tag_weight: blend between visual (1-w) and tag (w) channels. + motion_min: if set, reject records with motion_score below this. + Use for "no dead clips" at the retrieval step. + kind: filter to "video" or "image" only. + exclude_ids: clip_ids to skip (already used in the current edit). + """ + if len(self.records) == 0: + return [] + + scores = self._fused_sims(query_embedding, tag_weight) + exclude = set(exclude_ids or []) + + ranked: list[tuple[int, float]] = [] + for i, s in enumerate(scores): + rec = self.records[i] + if rec.clip_id in exclude: + continue + if kind and rec.kind != kind: + continue + if motion_min is not None and rec.motion_score < motion_min: + continue + ranked.append((i, float(s))) + + ranked.sort(key=lambda x: x[1], reverse=True) + top = ranked[:k] + return [(self.records[i], s) for i, s in top] + + def knn( + self, + clip_id: str, + k: int = 5, + exclude_ids: Optional[Iterable[str]] = None, + ) -> list[tuple[ClipRecord, float]]: + """Pure k-nearest-neighbours against the visual channel only. + + Used as a building block for `find_similar_set` via MMR. + """ + if clip_id not in self._id_to_row: + return [] + seed_idx = self._id_to_row[clip_id] + seed_vec = self.clip_embeddings[seed_idx] + + sims = self.clip_embeddings @ seed_vec + exclude = set(exclude_ids or []) + exclude.add(clip_id) # never return the seed itself + + ranked: list[tuple[int, float]] = [] + for i, s in enumerate(sims): + if self.records[i].clip_id in exclude: + continue + ranked.append((i, float(s))) + + ranked.sort(key=lambda x: x[1], reverse=True) + top = ranked[:k] + return [(self.records[i], s) for i, s in top] + + def find_similar_set( + self, + seed_clip_id: str, + n: int = 5, + diversity: float = 0.3, + candidate_pool: int = 30, + exclude_ids: Optional[Iterable[str]] = None, + ) -> list[tuple[ClipRecord, float]]: + """Collection-style retrieval: `n` clips that share the seed's + visual register but don't duplicate each other. + + Uses Maximal Marginal Relevance: + + score(c) = (1 - lambda) * sim(c, seed) + - lambda * max(sim(c, already_picked)) + + where `lambda = diversity`. At diversity=0 the result is pure + k-NN; at diversity=1 it picks the most-different-from-each-other + clips regardless of seed similarity. Default 0.3 keeps the set + tight but avoids duplicates. + """ + if seed_clip_id not in self._id_to_row: + return [] + seed_idx = self._id_to_row[seed_clip_id] + seed_vec = self.clip_embeddings[seed_idx] + + exclude = set(exclude_ids or []) + exclude.add(seed_clip_id) + + # Narrow to the top-`candidate_pool` by raw similarity first. + sims_to_seed = self.clip_embeddings @ seed_vec + candidate_idxs = np.argsort(-sims_to_seed).tolist() + pool: list[int] = [] + for i in candidate_idxs: + if self.records[i].clip_id in exclude: + continue + pool.append(int(i)) + if len(pool) >= candidate_pool: + break + + if not pool: + return [] + + picked: list[int] = [] + picked_scores: list[float] = [] + + while pool and len(picked) < n: + best_i = -1 + best_score = -1e9 + for i in pool: + sim_seed = float(sims_to_seed[i]) + if picked: + # max similarity to already-picked set + picked_vecs = self.clip_embeddings[np.array(picked)] + sim_picked = float(np.max(self.clip_embeddings[i] @ picked_vecs.T)) + else: + sim_picked = 0.0 + mmr = (1.0 - diversity) * sim_seed - diversity * sim_picked + if mmr > best_score: + best_score = mmr + best_i = i + picked.append(best_i) + picked_scores.append(best_score) + pool.remove(best_i) + + return [(self.records[i], s) for i, s in zip(picked, picked_scores)] + + def diversify( + self, + candidate_ids: list[str], + n: int, + diversity: float = 0.5, + ) -> list[str]: + """Given a pre-selected candidate list, greedily pick `n` that + are mutually dissimilar. + + Used at edit-arrangement time to make sure no two adjacent slots + are visually redundant. Returns the ordered clip_ids to keep; + order matches pick order. + """ + if not candidate_ids: + return [] + idxs = [self._id_to_row[c] for c in candidate_ids if c in self._id_to_row] + if not idxs: + return [] + + picked: list[int] = [idxs[0]] + remaining = idxs[1:] + + while remaining and len(picked) < n: + best_i = -1 + best_score = -1e9 + picked_mat = self.clip_embeddings[np.array(picked)] + for i in remaining: + sim_picked = float(np.max(self.clip_embeddings[i] @ picked_mat.T)) + # We want LOW similarity, so we negate. + score = -sim_picked + # Diversity weights how hard we penalize similarity. At + # diversity=1 we always pick the most different; at + # diversity=0 we just take them in input order. + score = diversity * score + (1.0 - diversity) * (-remaining.index(i)) + if score > best_score: + best_score = score + best_i = i + picked.append(best_i) + remaining.remove(best_i) + + return [self.records[i].clip_id for i in picked] diff --git a/pipeline_defs/documentary-montage.yaml b/pipeline_defs/documentary-montage.yaml new file mode 100644 index 0000000..25bc3f4 --- /dev/null +++ b/pipeline_defs/documentary-montage.yaml @@ -0,0 +1,165 @@ +name: documentary-montage +version: "1.0" +description: > + Retrieval-first thematic montage pipeline. Builds a semantic corpus of real-world + footage from Pexels, Archive.org (Prelinger et al.), and NASA, then uses CLIP-based + retrieval to fill slot descriptions from a thematic brief. The edit arranges clips + by narrative beat with music sync and uniform color grade across mixed-era footage. + Inspired by Adam Curtis / Chris Marker / Errol Morris tone poems. +category: documentary +stability: beta +default_checkpoint_policy: guided + +# Reference video input support +reference_input: + supported: false + +orchestration: + mode: executive-producer + skill: pipelines/documentary-montage/executive-producer + budget_default_usd: 1.00 + max_revisions_per_stage: 3 + max_send_backs: 2 + max_wall_time_minutes: 60 + +extensions: + custom_scripts: true + custom_playbooks: true + custom_skills: true + custom_tools: false + +required_skills: + - pipelines/documentary-montage/executive-producer + - pipelines/documentary-montage/idea-director + - pipelines/documentary-montage/scene-director + - pipelines/documentary-montage/asset-director + - pipelines/documentary-montage/edit-director + - pipelines/documentary-montage/compose-director + - meta/reviewer + - meta/checkpoint-protocol + +compatible_playbooks: + custom_allowed: true + +stages: + - name: idea + skill: pipelines/documentary-montage/idea-director + produces: + - brief + tools_available: [] + checkpoint_required: true + human_approval_default: true + review_focus: + - Thematic question is ONE sentence + - Tone register is ONE value from the fixed list + - Duration and shape are concrete + - Music plan is resolved or explicitly silent + success_criteria: + - Schema-valid brief artifact + - thematic_question present in metadata + - music_plan present in metadata + + - name: scene_plan + skill: pipelines/documentary-montage/scene-director + required_artifacts_in: + - brief + produces: + - scene_plan + tools_available: [] + checkpoint_required: true + human_approval_default: true + review_focus: + - Slot descriptions use concrete noun-and-adjective language + - Every slot has 2-3 short search queries + - At least 2 slots are marked hero + - Slot count and target holds match the tone/duration math + - era_mix is reflected in preferred_sources distribution + success_criteria: + - Schema-valid scene_plan artifact + - metadata.slots[] present with per-slot queries and preferred_sources + - Sum of target_hold_seconds within 10% of brief.duration_seconds + + - name: assets + skill: pipelines/documentary-montage/asset-director + required_artifacts_in: + - scene_plan + - brief + produces: + - asset_manifest + required_tools: + - corpus_builder + - clip_search + optional_tools: + - music_gen + tools_available: + - corpus_builder + - clip_search + - music_gen + checkpoint_required: true + human_approval_default: false + review_focus: + - Corpus size is at least 8x the slot count + - Every slot has exactly one picked clip with score >= 0.22 + - No clip_id is picked for two slots (exclude_ids enforced) + - diversify ran clean on the final timeline + - rejected_picks log has reasons for passes + - Provenance (provider, original_url, license) present on every asset + success_criteria: + - Schema-valid asset_manifest artifact + - One video asset per scene slot + - metadata.corpus_stats present + + - name: edit + skill: pipelines/documentary-montage/edit-director + required_artifacts_in: + - scene_plan + - asset_manifest + produces: + - edit_decisions + tools_available: [] + checkpoint_required: true + human_approval_default: true + review_focus: + - Hero slots hold longest; mid-sequence cutaways shortest + - No two adjacent cuts share subject AND scale + - Transition vocabulary is at most 4 distinct values + - Music config is present OR brief explicitly says no music + - total_duration_seconds matches sum of cut durations + - Every cut has a reason + success_criteria: + - Schema-valid edit_decisions artifact + - Timeline duration within 10% of brief.duration_seconds + + - name: compose + skill: pipelines/documentary-montage/compose-director + required_artifacts_in: + - edit_decisions + - asset_manifest + - brief + produces: + - render_report + required_tools: + - video_compose + optional_tools: + - audio_mixer + - color_grade + - video_trimmer + - video_stitch + tools_available: + - video_compose + - audio_mixer + - color_grade + - video_trimmer + - video_stitch + checkpoint_required: true + human_approval_default: false + review_focus: + - Output duration matches planned within 1s + - Resolution matches target_platform canvas + - Uniform LUT applied across the timeline + - Music present iff brief planned for it + - First and last frames verified + - No silent fallback from a motion-led promise + success_criteria: + - Schema-valid render_report artifact + - Output file exists and passes ffprobe validation diff --git a/skills/pipelines/documentary-montage/asset-director.md b/skills/pipelines/documentary-montage/asset-director.md new file mode 100644 index 0000000..7c3fe59 --- /dev/null +++ b/skills/pipelines/documentary-montage/asset-director.md @@ -0,0 +1,365 @@ +# Asset Director - Documentary Montage Pipeline + +## When To Use + +The shot list exists. You now have to actually go out and find the +clips that fill each slot. This is a two-step operation: + +1. **Build the corpus** — fan the scene director's queries out across + Pexels / Archive.org / NASA and download/embed the candidates. +2. **Pick per slot** — run CLIP retrieval against the corpus with each + slot description and choose one winner per slot. + +The output is an `asset_manifest` mapping every slot to exactly one +clip with full provenance. + +## Prerequisites + +| Layer | Resource | Purpose | +|-------|----------|---------| +| Schema | `schemas/artifacts/asset_manifest.schema.json` | Artifact validation | +| Prior artifact | `state.artifacts["scene_plan"]["scene_plan"]` | Slot descriptions + queries + preferred_sources | +| Prior artifact | `state.artifacts["idea"]["brief"]` | `era_mix`, `sources_allowed`, `music_plan` | +| Tool | `corpus_builder` | Populates the retrieval index | +| Tool | `clip_search` | Ranks clips against slot descriptions | +| Tool (optional) | `music_gen`, user's `music_library/` | Score bed | + +## Mental Model + +The corpus is NOT a stock library. It is a search index the agent +builds on demand. You do not scroll through it — you query it. + +Three rules that follow from that: + +1. **Build before picking.** Never call `clip_search.rank_for_slot` + on a corpus that doesn't contain candidates for that slot's query + family. The ranking will return junk and you'll waste the slot. +2. **Grow, don't replace.** The corpus is append-only. If a slot's + retrieval is weak, add more queries and rebuild — don't start over. +3. **Pick per slot, not per clip.** Every clip only belongs to one + slot in the final edit. Use `exclude_ids` to prevent double-use. + +## Process + +### 1. Resolve The Corpus Directory + +Decide where the corpus lives. Convention: + +``` +projects//corpus/ +``` + +The same `corpus_dir` is passed to every `corpus_builder` and +`clip_search` call. The corpus is reusable across re-runs — if the +scene director adds slots later, you can grow the same corpus instead +of rebuilding from scratch. + +### 2. Fan Out The Queries Into `corpus_builder` + +Read `scene_plan.metadata.slots[]`. Collect every `queries[]` array +across every slot. De-duplicate. Group by `preferred_sources`. + +Call `corpus_builder.execute(...)` with one fan-out per source set: + +```python +# Example shape. The agent constructs this from the shot list. +corpus_builder.execute({ + "corpus_dir": "projects//corpus", + "queries": [ + {"query": "raindrop on asphalt slow motion", "kind": "video", "per_source": 8}, + {"query": "wet city street night neon", "kind": "video", "per_source": 8}, + {"query": "taxi heavy rain yellow", "kind": "video", "per_source": 6}, + # ... one entry per unique slot query + ], + "sources": ["pexels", "archive_org"], # from preferred_sources union + "filters": { + "min_duration": 3, + "max_duration": 40, + "orientation": "landscape", + "min_width": 1280, + }, + "max_new_clips": 150, # enlarge the search space + "thumbs_per_video": 5, +}) +``` + +**Rules for the fan-out:** + +- Budget the corpus for 8-12x the slot count. A 15-slot montage wants + ~150 candidates so retrieval has real choices. +- `per_source` of 4-8 per query is usually enough. Pushing to 20+ + mostly adds noise. +- If `era_mix = "vintage"`, run a separate fan-out restricted to + `["archive_org"]` with period-appropriate queries. Prelinger search + is slow — don't interleave it with the modern Pexels batch. +- If any slot has `nasa` in `preferred_sources`, run ONE small + `nasa`-only batch. NASA is slow and its results are niche. + +### 3. Sanity-Check The Corpus Before Retrieval + +Before spending tokens on slot picks, call `clip_search` with +`operation=stats`: + +```python +clip_search.execute({ + "operation": "stats", + "corpus_dir": "projects//corpus", +}) +``` + +Look at `rows`, `per_source`, `per_kind`, `mean_motion_score`. You're +checking for three failure modes: + +- `rows < 50` — corpus is too small. Grow it. +- `per_source` heavily skewed (e.g. 98% pexels, 2% archive_org) on a + vintage brief — run a targeted archive_org fan-out. +- `mean_motion_score < 1.0` — corpus is full of static clips and will + make for a slideshow. Rerun with different queries, or apply + `motion_min` at rank time. + +### 4. Rank Candidates Per Slot + +For each slot in `scene_plan.metadata.slots[]`, call `clip_search` +with `operation=rank_for_slot`: + +```python +clip_search.execute({ + "operation": "rank_for_slot", + "corpus_dir": "projects//corpus", + "query_text": slot["description"], # NOT slot["queries"] — the description is richer + "k": 30 if slot.get("hero") else 12, + "tag_weight": 0.3, + "motion_min": 1.5, + "kind": "video", + "exclude_ids": already_picked_ids, # global accumulator +}) +``` + +Key points: + +- Use the slot **description**, not the queries. The description is + the rich noun-and-adjective string the scene director wrote. CLIP + ranks it better than short search phrases. +- `tag_weight=0.3` blends visual embedding (70%) with source-tag + embedding (30%). Raise to 0.5 when Pexels URL tags are strong and + the visual channel is noisy. Lower to 0.15 for Prelinger where tags + are long prose. +- Always pass `exclude_ids` with every clip already locked to a slot, + so the same key-in-door clip doesn't win two slots. + +### 5. Pick With Judgement, Not By Score + +The top result is not always the right pick. Look at the top 3-5 and +judge each one against: + +- **Era fit.** Does a 2022 4K Pexels shot belong in an elegiac list + montage about home? Maybe. Maybe not. +- **Motion fit.** The tone table from the scene director tells you + how long the hold will be. If the clip has a 4.0s hold target and + the clip is 2s long with a fast whip pan, it won't stretch. +- **Compositional carry.** Will this clip work NEXT to the clips + picked for the adjacent slots? You don't know yet — but if slot_02 + is a wide rooftop-in-rain and the top hit for slot_03 is also a + wide rooftop-in-rain, pick the #2 instead. +- **Emotional register.** CLIP will happily match "empty city + sidewalk at night" to a bright neon Vegas cutaway. The neon shot is + WRONG for an elegiac brief. Score 0.42 does not override tone. + +**Acceptable-score rules of thumb (CLIP ViT-B/32 cosine):** + +- `>= 0.30` — strong match, usually usable. +- `0.22-0.30` — plausible, needs human judgement. +- `< 0.22` — the corpus doesn't contain what you need. Grow it, + don't force a pick. + +### 6. Grow The Corpus When Retrieval Is Weak + +If a slot's top score is below 0.22, do NOT pick the best-of-a-bad- +bunch. Instead: + +1. Rewrite the slot's queries — maybe too abstract, maybe wrong + vocabulary for the era. +2. Run another `corpus_builder.execute(...)` pass with just the new + queries for that one slot. The builder skips clips already in the + index, so this is cheap. +3. Re-rank. + +Two growth passes per slot is plenty. If three passes can't find a +score above 0.22, tell the idea director the slot is unfilmable from +open corpora and recommend either dropping the slot or letting the +user supply the footage. + +### 7. Diversify Adjacent Picks + +Once you have one candidate per slot, you have a list of clip_ids in +timeline order. Visually-redundant adjacent shots kill the edit. Run +`clip_search.diversify` across the list: + +```python +clip_search.execute({ + "operation": "diversify", + "corpus_dir": "projects//corpus", + "candidate_ids": picked_ids_in_timeline_order, + "n": len(picked_ids_in_timeline_order), + "diversity": 0.5, +}) +``` + +If `diversify` drops a clip, it's telling you two of your picks are +visually identical. Re-rank the slot whose clip got dropped with +`exclude_ids` including the surviving twin. + +### 8. Handle The Music Plan + +Read `brief.music_plan`. Execute exactly the plan the idea director +recorded — do not invent a new source here: + +- **`source=library`**: Verify the file at `music_plan.path` exists. + Record it in the asset manifest as `type=music`, `subtype=library`. +- **`source=user`**: Same, with `subtype=provided`. +- **`source=generated`**: Call the named music tool with the seed + prompt from the brief. Sample first, batch only after confirming + mood. Record provider and cost. +- **`source=none`**: Do not generate silence. Do not swap in a track + because the edit feels thin. If the user approved "no music", run + with no music. + +**Never switch music source at this stage.** That's a Decision +Communication Contract violation — changing music mode is a major +production change and needs user approval at proposal time. + +### 9. Record The Asset Manifest + +Emit one asset per slot using the canonical schema. Documentary- +montage-specific fields live in `metadata`: + +```json +{ + "version": "1.0", + "assets": [ + { + "id": "asset_slot_01", + "type": "video", + "path": "projects//corpus/clips/pexels_12345/video.mp4", + "source_tool": "corpus_builder", + "scene_id": "slot_01", + "duration_seconds": 7.2, + "resolution": "1920x1080", + "format": "mp4", + "provider": "pexels", + "license": "Pexels License (free, no attribution required)", + "original_url": "https://www.pexels.com/video/12345", + "subtype": "stock", + "generation_summary": "Retrieved via CLIP rank for slot 'raindrop on asphalt slow motion...'. Score 0.38." + }, + { + "id": "asset_music_bed", + "type": "music", + "path": "music_library/dawn_04.mp3", + "source_tool": "music_library", + "scene_id": "global", + "subtype": "library", + "license": "user-provided" + } + ], + "metadata": { + "pipeline": "documentary-montage", + "corpus_dir": "projects//corpus", + "corpus_stats": { "rows": 157, "per_source": {"pexels": 98, "archive_org": 52, "nasa": 7} }, + "rejected_picks": [ + { + "slot_id": "slot_03", + "clip_id": "pexels_99921", + "score": 0.41, + "reason": "wrong era — 2022 4K kitchen, brief is vintage" + } + ] + } +} +``` + +The `rejected_picks` log matters. The edit director reads it when a +pick feels wrong and needs to reach for the #2 option. + +### 10. Quality Gate + +- Every slot in the scene plan has exactly one asset mapped to it. +- Every picked clip has `score >= 0.22` in the rejected-picks log + (or a logged "user-approved override" note). +- No clip_id appears as the primary pick for two slots. +- `diversify` ran clean on the final list (no dropped picks, or all + dropped picks were re-filled). +- `corpus_stats` shows at least 8x the slot count in rows. +- Music asset exists OR `music_plan.source = "none"` with explicit + acknowledgement. +- For vintage briefs, at least 60% of picks come from `archive_org`. +- All file paths resolve. + +## Common Pitfalls + +- **Running `clip_search.rank_for_slot` against an empty corpus.** + You will get an empty `results` list or a cryptic shape error. + Always call `stats` after a build, before ranking. +- **Picking by score alone.** Score is an input to judgement, not the + judgement. An elegiac piece full of top-scored Pexels HD sunshine + will feel wrong regardless of scores. +- **Forgetting `exclude_ids`.** Without it, the same amazing clip + wins every slot and the montage becomes a slideshow of one image. +- **Quiet music substitution.** User said "none", agent generated + anyway because "the edit felt thin". This is a major change and + needs approval — see `skills/pipelines/documentary-montage/executive-producer.md` + cross-stage rules. +- **Growing the corpus unboundedly.** Two growth passes per weak slot + is the limit. Beyond that, the footage probably doesn't exist in + the open corpora and the slot needs to change. +- **Using slot queries as the rank text.** Queries are search phrases + for stock APIs; descriptions are semantic text for CLIP. They are + different. Rank on descriptions. +- **Losing provenance.** Every clip must carry `provider`, + `original_url`, and `license` in the manifest. These are the + non-negotiables for any downstream publishing step. + +## Retrieval Recipes + +A few retrieval moves that come up often: + +### "Find N variants of this one clip I love" + +```python +clip_search.execute({ + "operation": "find_similar_set", + "corpus_dir": "projects//corpus", + "seed_clip_id": "pexels_12345", + "n": 5, + "diversity": 0.4, + "candidate_pool": 40, +}) +``` + +Used when a slot wants "five more shots like this one" — e.g. a +catalogue of doorways all filmed in the same register. + +### "I have 20 candidates, trim to 8 non-redundant picks" + +```python +clip_search.execute({ + "operation": "diversify", + "corpus_dir": "projects//corpus", + "candidate_ids": [...], + "n": 8, + "diversity": 0.5, +}) +``` + +### "Look up one clip's full metadata" + +```python +clip_search.execute({ + "operation": "get", + "corpus_dir": "projects//corpus", + "clip_id": "archive_org_Prelinger_HomeMovies_0042", +}) +``` + +Used when the edit director wants to confirm the provider/URL before +locking the cut. diff --git a/skills/pipelines/documentary-montage/compose-director.md b/skills/pipelines/documentary-montage/compose-director.md new file mode 100644 index 0000000..1e20b13 --- /dev/null +++ b/skills/pipelines/documentary-montage/compose-director.md @@ -0,0 +1,277 @@ +# Compose Director - Documentary Montage Pipeline + +## When To Use + +The timeline exists. Every cut has an in/out, transitions are +chosen, the music bed is locked. You now have to render the piece +and apply the register-smoothing pass (uniform crop + LUT + audio +mix) that makes a mixed-era corpus feel like one film. + +The output is a single mp4 plus a `render_report` artifact. + +## Prerequisites + +| Layer | Resource | Purpose | +|-------|----------|---------| +| Schema | `schemas/artifacts/render_report.schema.json` | Artifact validation | +| Prior artifact | `state.artifacts["edit"]["edit_decisions"]` | Cuts, transitions, music, metadata hints | +| Prior artifact | `state.artifacts["assets"]["asset_manifest"]` | File paths, durations, providers | +| Tool | `video_compose` (FFmpeg + Remotion) | Primary render engine | +| Tool | `audio_mixer` | Music fade, silence window, L-cuts | +| Tool (optional) | `color_grade` | Uniform LUT across mixed-era clips | +| Tool (optional) | `video_trimmer`, `video_stitch` | Lower-level helpers if needed | + +## Mental Model + +Most pipelines treat compose as a boring export step. For +documentary montage it is a creative step: the last pass where grade +and mix reconcile footage from radically different sources into one +piece. + +Three things must happen here that cannot happen earlier: + +1. **Uniform aspect and letterbox.** Pexels 1920x1080, Prelinger + 640x480 4:3, NASA 1280x720 all need to land on one canvas. +2. **Uniform color grade.** A single LUT across the whole timeline + is what makes the 1962 home movie sit next to the 2023 kitchen + without jumping out. +3. **Audio mix.** Music level, silence window, L-cut ambient + carries, final fade — done in one pass with the timeline in hand. + +## Process + +### 0. Hard Requirement Check + +Read `brief` and `edit_decisions.metadata` for any hard requirements. +If the brief said "no narration" and a narration track somehow +appeared in the edit, STOP and ask. Do not render over a contract +violation. + +Also confirm the render engine you intend to use is actually +available — `video_compose` in FFmpeg-only mode is fine for this +pipeline (the whole piece is footage-led, no Remotion scenes needed +unless the user asked for title cards). FFmpeg alone can render this +pipeline end to end. + +### 1. Resolve The Canvas + +Read `brief.target_platform`: + +| Target | Canvas | Letterbox | +|--------|--------|-----------| +| `social_short` (Instagram/TikTok) | 1080x1920 (9:16) | Top/bottom crop; center-anchor each clip | +| `youtube` / `generic` | 1920x1080 (16:9) | None; optionally 2.35:1 top/bottom bars for cinematic feel | +| `linkedin` | 1920x1080 (16:9) | None | + +Every clip in the timeline must be scaled/cropped to this canvas. +For `social_short`, this usually means center-cropping 16:9 footage. +For `youtube` with the cinematic 2.35:1 bar treatment, pad 140px +black top and bottom on a 1920x1080 canvas. + +Commit this in `render_report.metadata.canvas` and +`render_report.metadata.letterbox`. + +### 2. Build The Concat Plan For `video_compose` + +The edit artifact gives you a list of cuts with in/out, transitions, +and source asset_ids. Walk the asset_manifest to resolve each +asset_id to a real file path. Then build the render plan. + +For a pipeline this simple, the cleanest path is: + +```python +video_compose.execute({ + "operation": "render", + "output_path": "projects//renders/final.mp4", + "canvas": { "width": 1920, "height": 1080, "fps": 24 }, + "cuts": [ + { + "source": "", + "in": 1.2, + "out": 5.2, + "scale": "fit_canvas_center_crop", + "transition_in": "fade", + "transition_in_duration": 0.8, + }, + # ... more cuts + ], + "audio": { + "music_path": "", + "music_volume": 0.7, + "music_fade_in": 1.0, + "music_fade_out": 4.0, + "silence_windows": [{"start": 54.0, "end": 56.0}], + "sfx_layers": [ + {"source": "", "start": 20.8, "duration": 1.2, "volume": 0.6} + ], + }, + "frame_treatment": { + "lut_path": "styles/luts/warm_film_100.cube", + "letterbox": "2.35:1" + } +}) +``` + +The exact field names come from the live `video_compose` schema at +render time — consult the tool's `agent_skills` if available before +writing the call. Do not invent parameters. + +### 3. Apply Grade Via LUT, Not Per Clip + +Read `edit_decisions.metadata.grade_profile`. Map it to a LUT file: + +| Profile | LUT | Suits | +|---------|-----|-------| +| `warm_film_100` | vintage film warmth, slight lift | elegiac, dreamlike | +| `cool_archive_60` | cool highlights, crushed blacks | urgent, wry | +| `neutral_doc_20` | barely-there neutral balance | reverent | +| `bleach_bypass_80` | desaturated, high contrast | wry, documentary-harsh | + +If the profile isn't in the styles library, use `neutral_doc_20` and +note it in `warnings`. Do not try to auto-grade — the LUT is the +whole point of the register-smoothing pass. + +Apply the LUT at the composition level, not per clip. One LUT, one +timeline, one consistent look. This is what makes a 1962 Prelinger +clip and a 2023 Pexels clip feel like the same film. + +### 4. Mix The Audio Once, In Compose + +The edit artifact already decided volumes, fades, silence windows, +and L-cut sfx layers. Your job is to execute them faithfully: + +- Music bed at `edit_decisions.audio.music.volume` (default 0.7). +- Fade in per `fade_in_seconds`, fade out per `fade_out_seconds`. +- Silence window = ducked to 0.0 for the window's duration, ramp + back up with a 0.2s hold-off. +- L-cut SFX layers = mix at 0.5-0.7 volume, under music. +- No narration unless explicitly present in `edit_decisions.audio.narration`. + +If the brief says "no music" and the edit correctly has no music +entry, render silent. Do NOT add ambient noise "to fill the gap". + +### 5. Render At Documentary Spec + +Recommended encoder settings for doc montage: + +| Field | Value | Why | +|-------|-------|-----| +| Codec | `libx264` (H.264) | Universal, small | +| Pixel format | `yuv420p` | Universal compatibility | +| CRF | `18` | Visually lossless for final deliverables | +| FPS | `24` | Cinematic. Do NOT upconvert 24->30. | +| Audio codec | `aac` | Universal | +| Audio bitrate | `192k` | Music-bed friendly | + +If the source clips are 30fps and the canvas is 24fps, let FFmpeg +drop frames evenly — don't blend. Motion interpolation on +mixed-source footage looks awful. + +### 6. Post-Render Verification + +After the render succeeds, actually probe the output file and check: + +- **Duration.** Should match `sum(out - in for cut in cuts) + fade + in/out` within ±0.5s. +- **Resolution.** Should match the canvas. +- **Audio presence.** If music was in the plan, the output must + have an audio stream. If silence was planned, confirm. +- **First and last frame.** Open the file, seek to 0s and to + duration-0.1s. The first frame should be a fade-in. The last + frame should be (or be fading to) black. +- **Silence window.** Seek to the silence_window start. Audio level + should drop visibly in the waveform. + +Record verifications in `render_report.verification_notes`. + +### 7. Emit The Render Report + +```json +{ + "version": "1.0", + "outputs": [ + { + "path": "projects//renders/final.mp4", + "format": "mp4", + "codec": "h264", + "audio_codec": "aac", + "resolution": "1920x1080", + "fps": 24, + "duration_seconds": 89.8, + "file_size_bytes": 18234112, + "platform_target": "youtube" + } + ], + "render_time_seconds": 42.3, + "warnings": [], + "verification_notes": [ + "Duration within +0.2s of planned", + "First frame is black fade-in as specified", + "Silence window 54-56s confirmed (music -60dB)", + "Last frame fades to black at 89.0s" + ], + "render_grammar": "cinematic-trailer", + "metadata": { + "pipeline": "documentary-montage", + "canvas": { "width": 1920, "height": 1080 }, + "letterbox": "2.35:1", + "lut": "warm_film_100", + "music_present": true + } +} +``` + +### 8. Quality Gate + +- Output file exists and plays. +- Duration within ±1s of `brief.duration_seconds`. +- Resolution matches `target_platform` canvas. +- LUT was applied (or a warning logged). +- Music is present iff the brief planned for it. +- First and last frames verified. +- Silence window (if any) verified in the waveform. +- No narration unless brief-approved. +- `render_report.warnings` lists every substitution. + +## Common Pitfalls + +- **Letting mixed-era clips render un-graded.** The piece will look + like a PowerPoint slideshow of internet clips. The LUT is + non-negotiable. +- **Upscaling to match the canvas instead of letterboxing.** + Prelinger 640x480 upscaled to 1920x1080 looks pixelated and wrong. + Center it with letterbox bars, or embrace the squared crop as a + design choice. +- **Narration or ambient SFX added "to fill the gap".** Major + change, needs user approval. +- **Per-clip color grading.** One LUT across the whole piece. Do + not try to balance each clip individually — it takes 10x the time + and makes the register LESS consistent, not more. +- **Quiet render engine swap.** If `video_compose` routes through + Remotion for some reason and the aesthetic changes, stop and + surface. This pipeline is FFmpeg-friendly and shouldn't need + Remotion unless the user asked for title cards. +- **Overriding edit decisions at render time.** If you find yourself + adjusting volumes, fades, or trims in the render call, you're + editing during compose. Go back to the edit stage, fix the + decisions, re-emit the artifact, then re-render. +- **Skipping verification.** A render that "succeeded" but is + actually silent, or fades wrong, or clips the last hero frame, is + worse than a failure. Open the file. + +## When The Render Fails + +If `video_compose` returns an error: + +1. Check the error category per the Decision Communication Contract + (auth / provider / tool bug / plan quality). +2. If it's a path error, validate every asset_id → path resolution + in the asset manifest. A single missing file fails the whole render. +3. If it's a codec error, the input clips may have exotic containers + (Archive.org sometimes serves Matroska). Try running each input + through `video_trimmer` first to normalize to mp4/h264. +4. If it's a memory or timeout error, split the render into halves + with `video_stitch` at the end. +5. Surface to the user before swapping to a lower-fidelity path. + This pipeline is footage-led; there is no generated-stills + fallback. diff --git a/skills/pipelines/documentary-montage/edit-director.md b/skills/pipelines/documentary-montage/edit-director.md new file mode 100644 index 0000000..a6fbc96 --- /dev/null +++ b/skills/pipelines/documentary-montage/edit-director.md @@ -0,0 +1,335 @@ +# Edit Director - Documentary Montage Pipeline + +## When To Use + +Every slot has a clip. You now have to turn a pile of clips into a +piece. This stage decides in-points, out-points, transitions, music +sync, and the order the clips actually run. The output is an +`edit_decisions` artifact with a concrete timeline. + +This is where documentary technique lives. If the asset director did +its job, you have the raw material. The edit is the thinking. + +## Prerequisites + +| Layer | Resource | Purpose | +|-------|----------|---------| +| Schema | `schemas/artifacts/edit_decisions.schema.json` | Artifact validation | +| Prior artifact | `state.artifacts["assets"]["asset_manifest"]` | Picked clips + music bed | +| Prior artifact | `state.artifacts["scene_plan"]["scene_plan"]` | Slot order, hero flags, target holds | +| Prior artifact | `state.artifacts["idea"]["brief"]` | Tone register, duration, shape | +| Tool (optional) | `video_analyzer` | Probe a clip's motion if you need to re-check | + +## Mental Model + +Documentary montage lives in four dimensions you have to balance: + +1. **Rhythm** — how long each hold lasts and how the holds relate. +2. **Juxtaposition** — which image follows which, and what it means. +3. **Music sync** — cuts landing on beats, dropouts earning weight. +4. **Continuity of register** — the grain, color, and era don't swing + wildly unless the swing is the point. + +The enemy is "slideshow" — a sequence of clips played back-to-back +with the same hold length and no sound design. If it feels like a +slideshow, the edit has failed, regardless of how good the clips are. + +## Process + +### 0. Guardrails — No Silent Major Changes + +Before touching the timeline, re-read the brief. If any of these are +true, STOP and surface to the user per the Decision Communication +Contract: + +- The brief approved "no narration" but the edit feels like it needs + voice-over. Narration is a MAJOR change. +- The brief approved a music track that the edit director now wants + to replace. Music swap is a MAJOR change. +- The brief approved a 90s duration but the natural cut wants 2m30s. + Duration stretch is a MAJOR change. + +Fix the edit, don't paper over it. If the edit genuinely needs +one of these, ask. + +### 1. Set The Rhythm Grid + +Read `brief.tone` and `brief.duration_seconds`. Compute the hold +table from the scene director's tone chart: + +| Tone | Base hold | Min hold | Max hold | +|------|-----------|----------|----------| +| elegiac | 4.0s | 2.5s | 7.0s | +| reverent | 3.5s | 2.0s | 6.0s | +| dreamlike | 3.0s | 1.5s | 5.5s | +| wry | 2.0s | 1.0s | 4.0s | +| urgent | 1.2s | 0.5s | 2.5s | + +**Hero slots get max hold.** Mid-sequence cutaways get base. Quick +transitions get min. + +Total hold time must sum to within ±10% of `brief.duration_seconds`. +If you overshoot, compress non-hero holds first — never cut heroes +short to fit duration. + +### 2. Arrange By Narrative Beat, Not By Score + +The scene director gave you a slot order. That order is the intent. +Don't rearrange it by CLIP score, motion score, or resolution. + +You MAY reorder slots when: + +- The music bed has a downbeat at a known timestamp and reordering + two slots lands a hero on the beat (see step 4). +- Two adjacent slots are visually identical and swapping one breaks + the monotony (but see step 7 — diversify should have caught this + already). +- The final image isn't landing. The last 5-10s carries + disproportionate weight; if the scene director's choice dies, move + a stronger candidate to the tail. + +Always log the reorder in `edit_decisions.metadata.reorder_notes` +with the reason. + +### 3. Trim Each Clip To Its Beat + +For every picked clip, decide `in_seconds` and `out_seconds`. Three +rules: + +- **Find the best sub-window, not the whole clip.** A 12-second Pexels + clip usually contains one 3-second moment that earns the hold and + 9 seconds of setup/settle. Find the moment. +- **Cut BEFORE the action's natural end.** End on a look, not on a + move-off. The cut feels intentional instead of exhausted. +- **Leave a handle at both ends.** 4-6 frames of headroom so the + composer can apply a fade or dissolve without clipping the moment. + +If a clip is too short to fill its target hold, either: + +- slow it down (speed 0.5-0.75, fine on static-ish footage, bad on + anything with sync motion or faces talking), +- let it cut early and borrow the remaining duration from the next + slot's hold, +- or swap to the #2 candidate from the rejected-picks log. + +Do NOT hold on the last frozen frame. A freeze-frame in a doc montage +reads as a technical mistake. + +### 4. Sync To The Music Bed + +Read `asset_manifest` for the music asset and load its duration. +Documentary montages earn their emotional weight from cuts landing +on musical events. Three sync moves: + +- **Downbeat cuts.** If you have bars and beats metadata (from a + provided track) or can hear them, place hero cuts on downbeats. + If not, evenly-spaced cuts on 4s intervals for a 60bpm bed are a + safe default. +- **One held silence.** Drop the music out for ~2s at the piece's + emotional center. Silence is a tool. Use it once. Use it hard. +- **Tail fade.** Music fades under the last 3-5s so the final image + can breathe without a musical resolution fighting it. + +Record the music config in `edit_decisions.audio.music` with: + +```json +{ + "asset_id": "asset_music_bed", + "volume": 0.7, + "fade_in_seconds": 1.0, + "fade_out_seconds": 4.0, + "ducking": false +} +``` + +`ducking: false` is the default for this pipeline — there's no +narration to duck under. If the user approved a narration track, set +ducking to true and let it dip during segments. + +### 5. Choose Transitions From A Small Vocabulary + +Documentary montage uses maybe four transitions total across the +entire piece: + +| Transition | Use | +|------------|-----| +| `cut` (hard) | Default. Most cuts are hard cuts. | +| `dissolve` (0.5-1.0s) | Emotional sibling clips, time passage | +| `fade_to_black` (0.5s, then back up) | Act breaks in 3-act shape, or once near the end | +| `fade_in` (first shot) / `fade_out` (last shot) | 0.5-1.0s bookends | + +**Do not use:** + +- wipes, +- push/slide transitions, +- zoom blurs, +- RGB splits, +- light leaks, +- glitch effects. + +These read as social-media edit language and will break the +documentary register. If the piece is getting boring, fix the clip +choices or the pacing, don't add transition flash. + +Record each cut's `transition_in` / `transition_out` per the schema. +Default `transition_in: "cut"` on most cuts. + +### 6. Apply Register Continuity + +Mixed-era corpora look wildly different. Pexels 2023 is clean, sharp, +color-graded. Prelinger 1962 is grainy, warm, squared-off aspect. +NASA archival is often low-res with text overlays. If you mash them +together raw, the piece looks like a Wikipedia article. + +You have two tools to smooth this: + +1. **Crop to a uniform aspect ratio.** Pick one: 16:9 cinematic + (`2.35:1` letterbox on top/bottom) for hero pieces, 9:16 for + social. Enforce in the `transform.crop` field of each cut. +2. **Flag the piece for a uniform color grade at compose time.** Put + a `grade_profile` hint in `edit_decisions.metadata`. The compose + director will apply a LUT across the whole timeline. + +Don't try to color-grade individual clips here. That's the compose +stage. Your job is to flag the need. + +### 7. Enforce Adjacent Diversity One More Time + +Walk the timeline in pairs. For each consecutive (cut_n, cut_n+1): + +- Are they the same subject at the same scale? If yes, you have a + slideshow moment. Swap one for a clip at a different scale (wide + vs close). +- Are they the same color palette (two night-blue clips back to + back)? If yes, break the pattern at least every 4 cuts. +- Are they the same motion direction (two left-to-right pans)? If + yes, flip the second's horizontal axis or reorder. + +Log any swaps you made in `metadata.diversity_swaps`. + +### 8. The L-Cut Move (Optional But Powerful) + +For any transition between two clips where the outgoing clip has +strong ambient audio (rain, footsteps, traffic), carry the audio +under the incoming clip for 0.5-1.5s. This is an L-cut and it +welds two shots together more tightly than any visual transition. + +Implement via the schema by using a short `dissolve` transition OR +by layering the outgoing clip's audio as an SFX entry in +`edit_decisions.audio.sfx` with a delayed end. + +Documentary montages with L-cuts feel 50% more coherent than ones +without. Use them on the 3-4 hardest transitions in the piece. + +### 9. Emit The Edit Decisions + +Canonical shape for this pipeline: + +```json +{ + "version": "1.0", + "cuts": [ + { + "id": "cut_01", + "source": "asset_slot_01", + "in_seconds": 1.2, + "out_seconds": 5.2, + "layer": "primary", + "transform": { "scale": 1.0, "position": "center" }, + "transition_in": "fade_in", + "transition_out": "cut", + "transition_duration": 0.8, + "reason": "opening hero — raindrop on asphalt, 4s hold, slow-motion streetlamp glow" + }, + { + "id": "cut_02", + "source": "asset_slot_02", + "in_seconds": 2.0, + "out_seconds": 5.5, + "layer": "primary", + "transition_in": "cut", + "transition_out": "cut", + "reason": "umbrella opening in doorway, hard cut from raindrop → street" + } + ], + "audio": { + "music": { + "asset_id": "asset_music_bed", + "volume": 0.7, + "fade_in_seconds": 1.0, + "fade_out_seconds": 4.0, + "ducking": false + } + }, + "metadata": { + "pipeline": "documentary-montage", + "tone": "elegiac", + "shape": "list", + "total_duration_seconds": 90.0, + "hold_table_used": { "base": 4.0, "min": 2.5, "max": 7.0 }, + "grade_profile": "warm_film_100", + "reorder_notes": [], + "diversity_swaps": [ + { "at": "cut_07-cut_08", "reason": "two wide rooftops-in-rain adjacent, swapped 08 for #2 pick" } + ], + "silence_window": { "start_seconds": 54.0, "end_seconds": 56.0 }, + "l_cuts": [ + { "from_cut": "cut_05", "to_cut": "cut_06", "carry_seconds": 1.2, "channel": "ambient_rain" } + ] + } +} +``` + +### 10. Quality Gate + +- `sum(out - in for cut in cuts)` is within ±10% of + `brief.duration_seconds`. +- Hero slots have the longest holds. +- No two adjacent cuts share subject AND scale. +- The transition vocabulary is at most 4 distinct values. +- Music config exists (or brief explicitly says no music). +- At least one `silence_window` entry for pieces >= 60s. +- Every cut has a one-line `reason` — if you can't write one, the + cut is arbitrary and should be reconsidered. +- `metadata.total_duration_seconds` matches the sum of cut durations. + +## Common Pitfalls + +- **Cutting by information density instead of rhythm.** A doc + montage is not a Wikipedia article. "But I need to show this" is + not a reason — if the image doesn't sustain a hold, it doesn't + belong. +- **Over-using dissolves.** A dissolve on every cut says "I couldn't + commit". Commit. +- **Ignoring the music bed until the end.** Music is not a sweetener + you add at compose time. It is a timing grid you cut TO. +- **Letting the final image be a weak one.** The last frame is + disproportionately remembered. If it's weak, swap it — the scene + director's slot ordering is a strong suggestion, not a contract. +- **Freeze-frame endings.** Reads as technical error. End on a + fade-to-black instead. +- **Silently adding a narration because the edit feels thin.** Major + change. Ask. +- **Hiding clip provider in the cuts.** Every `cut.source` must be + an `asset_manifest` asset_id so provenance survives. +- **Three different transition types in the first 15 seconds.** + Readers will feel the edit working. Restraint is the brand. + +## Worked Pacing Example — "A Minute in the Rain" + +90 seconds, elegiac, list shape, 15 hero-flagged slots. + +- Base hold 4.0s × 15 = 60s. Short by 30s. +- Add 30s across 3 hero slots (1, 11, 15) at +10s each: + hero_1 = 5.5s, hero_11 = 6.0s, hero_15 = 7.0s. +- Tighten slots 4, 7, 13 to 3.0s each (small cutaways). +- Insert silence_window 54.0-56.0s (right before hero_11). +- L-cut slot_10 (boot in puddle) → slot_11 (lit window across + street), carry rain-on-glass ambient 1.2s. +- First cut `fade_in` 1.0s, last cut `fade_out` 1.5s. +- All other cuts hard. +- Music fades in 1.0s, fades out 4.0s under hero_15 + black. + +This gives a 90s piece with 3 breathing points (fade_in, silence, +fade_out), a clear hero arc (slots 1 → 11 → 15), and no adjacent +scale collisions. diff --git a/skills/pipelines/documentary-montage/executive-producer.md b/skills/pipelines/documentary-montage/executive-producer.md new file mode 100644 index 0000000..86a70ab --- /dev/null +++ b/skills/pipelines/documentary-montage/executive-producer.md @@ -0,0 +1,89 @@ +# Executive Producer - Documentary Montage Pipeline + +## When To Use + +The user wants a short (30-180s) non-narrative piece built from existing +footage — a thematic collage, essay film, or Adam-Curtis-style tone +poem. The piece is NOT a narrated explainer, NOT a talking head, NOT a +single extended scene. It is an arranged sequence of real-world clips +whose meaning emerges from juxtaposition (Kuleshov effect, +Eisenstein's intellectual montage). + +This is the right pipeline when the brief includes phrases like: + +- "a montage about...", +- "show me the feeling of...", +- "like a tone poem", +- "documentary-style collage", +- "everyone who has ever..." / "the life of..." / "a portrait of...", +- "cut together from stock footage", +- "Adam Curtis", "Errol Morris", "Chris Marker", +- "no narration, just images". + +If the user asks for an explainer, a trailer with generated clips, or +a talking-head video, pick a different pipeline. + +## Philosophy + +Documentary montage is retrieval-first, not generation-first. +The corpus is the raw material; the edit is the thinking. Your job +across all stages is to: + +1. **Enlarge the search space before committing**. Build a corpus + bigger than you think you need so the edit has room to breathe. +2. **Let juxtaposition do the talking**. Two mundane clips next to + each other can mean something neither one means alone. +3. **Trust the footage**. If a clip shows a thing plainly, don't + explain it with text or voice-over. +4. **Pace is the message**. Cut on beat. Hold on images that earn it. + Short cuts = urgency, long holds = grief/weight/awe. + +## Stages + +| Stage | Director skill | Produces | +|-------|----------------|----------| +| `idea` | `idea-director.md` | brief (topic, tone, duration, shape) | +| `scene` | `scene-director.md` | shot_list (slot descriptions + queries) | +| `assets` | `asset-director.md` | asset_manifest (corpus built + per-slot picks) | +| `edit` | `edit-director.md` | edit_decisions (timeline + transitions + music) | +| `compose` | `compose-director.md` | render_report (final mp4) | + +Each director skill has its own quality gate. Read the director skill +before starting the stage. + +## Core Tools + +| Tool | Role | +|------|------| +| `corpus_builder` | Fans out across Pexels/Archive.org/NASA, downloads + embeds + indexes | +| `clip_search` | Ranks clips for a slot, finds similar sets, diversifies selections | +| `video_compose` / Remotion | Renders the final timeline | + +The agent talks to the stock sources through `corpus_builder` — never +call adapter classes directly from a skill or director. + +## Cross-Stage Rules + +- **No generated clips** unless the user explicitly asks. This pipeline + is about REAL footage, real texture, real grain. Generated B-roll + breaks the aesthetic. +- **No narration** unless the user explicitly asks. The brief should + default to image-only + music. Adding voice is a MAJOR change and + requires user approval per the Decision Communication Contract. +- **Build the corpus before picking clips**. Do not run clip_search + against an empty or half-built corpus. If retrieval results are + weak (all scores < 0.25), grow the corpus with new queries. +- **Keep a decision log of rejected picks**. When you pass on a clip + with a high score, note why (wrong era, overlit, wrong emotional + register). This helps the review stage. + +## Common Pitfalls + +- Treating the corpus as a stock library to pick from sequentially + instead of as a search index to query per slot. +- Arranging clips by score rather than by narrative beat. +- Letting visually-repetitive clips sit adjacent. Use + `clip_search` with `operation=diversify` before locking the edit. +- Over-cutting. Documentary montage lives in the hold, not the jump. +- Quietly inserting a narration track because the edit feels "thin". + Fix the edit; don't paper over it. diff --git a/skills/pipelines/documentary-montage/idea-director.md b/skills/pipelines/documentary-montage/idea-director.md new file mode 100644 index 0000000..d340750 --- /dev/null +++ b/skills/pipelines/documentary-montage/idea-director.md @@ -0,0 +1,126 @@ +# Idea Director - Documentary Montage Pipeline + +## When To Use + +You are turning a user prompt into the brief artifact that every +downstream stage will read. For this pipeline, the brief is the +thematic core: what the montage is ABOUT, what it should feel like, +and how long it should run. + +## Prerequisites + +| Layer | Resource | Purpose | +|-------|----------|---------| +| Schema | `schemas/artifacts/brief.schema.json` | Artifact validation | +| User input | Conversation history | The raw ask | +| Meta | `skills/meta/reviewer.md` | Self-review pass | + +## Process + +### 1. Extract The Thematic Question + +A documentary montage answers a question the user could not put into +a sentence. Your job is to name that question in ONE line. + +Good thematic questions: + +- "What does it feel like to come home?" +- "How did the 20th century think about the future?" +- "What happens in a city at 4am?" +- "What do all the footprints on Earth look like?" + +Bad thematic questions (too abstract or too concrete): + +- "A video about cities" (too abstract — no feeling) +- "A montage with 8 specific shots of the moon" (too concrete — that's + a shot list, not a theme) + +### 2. Fix The Tone + +Choose ONE emotional register. Write it down. Everything downstream +keys off this. + +Common registers for this pipeline: + +- **elegiac** — long holds, muted color, slow cuts (loss, memory, home) +- **urgent** — short cuts, hard sync, motion-heavy (crisis, cities, now) +- **reverent** — stately, symmetrical, patient (nature, ritual, scale) +- **wry** — ironic juxtaposition, cut on absurdity (consumer culture, + politics, mid-century optimism) +- **dreamlike** — slow dissolves, repeated motifs, non-linear (childhood, + grief, memory) + +### 3. Pick A Duration And A Shape + +Duration matters because it caps the number of beats. + +| Duration | Beats | Use | +|----------|-------|-----| +| 30-45s | 8-12 cuts | Social/Instagram/reel — one feeling, no arc | +| 60-90s | 15-25 cuts | Standard short — mini arc with a turn | +| 2-3 min | 30-50 cuts | Proper essay montage — 3-act arc possible | + +Shape options: + +- **single-image expansion** — one idea, held from many angles (good + for elegiac pieces under 60s) +- **before/after** — first half establishes, second half turns (good + for wry or urgent registers) +- **three-act** — setup → turn → release (the Adam Curtis move, needs + >90s) +- **list/catalogue** — "everyone who..." structure, no arc, just + accumulation (good for reverent or elegiac) + +### 4. Note Music Intent + +Documentary montage is inseparable from its music bed. Decide now: + +- user-provided track (put path in `music_plan.source_path`), +- music library pick (list what's in `music_library/`), +- generated (name the tool and prompt seed), +- or none (silence). + +**Warn the user if no music source is available.** Do not silently +defer this — it becomes an expensive surprise at the asset stage. + +### 5. Record The Brief + +Minimum fields the brief must carry: + +```json +{ + "topic": "A minute in the rain", + "thematic_question": "What does rain show you about a city?", + "tone": "elegiac", + "duration_seconds": 90, + "shape": "list", + "sources_allowed": ["pexels", "archive_org", "nasa"], + "generated_clips_allowed": false, + "narration": "none", + "music_plan": { "source": "library", "path": "music_library/dawn_04.mp3" }, + "era_mix": "any", + "target_platform": "social_short" +} +``` + +`era_mix` is a documentary-specific field: "modern" biases toward +Pexels, "vintage" biases toward Archive.org Prelinger, "any" leaves it +open for the scene director to decide per slot. + +### 6. Quality Gate + +- Thematic question is ONE sentence. +- Tone is ONE register from the fixed list. +- Duration and shape are concrete numbers / enum values. +- Music source is named OR the brief explicitly says "no music". +- Sources list is non-empty and at least one is `available` per the + tool registry. + +## Common Pitfalls + +- Stating multiple themes ("it's about cities AND technology AND loss"). + Pick one. The others become downstream associations. +- Jumping to shot lists. The brief is about MEANING. Shots come next. +- Ignoring duration. A 45s piece with 50 cuts is nausea. A 3-minute + piece with 12 cuts is a slideshow. +- Forgetting to ask about music. The user usually has an opinion. diff --git a/skills/pipelines/documentary-montage/scene-director.md b/skills/pipelines/documentary-montage/scene-director.md new file mode 100644 index 0000000..50c0730 --- /dev/null +++ b/skills/pipelines/documentary-montage/scene-director.md @@ -0,0 +1,301 @@ +# Scene Director - Documentary Montage Pipeline + +## When To Use + +The brief exists. You now have to turn a thematic question into a +concrete list of SLOTS the retrieval layer can fill. Each slot is an +intention ("a silhouette at a doorway at dusk") plus the queries that +will find it in the real world (Pexels/Archive.org/NASA). + +This is the most creative stage in the pipeline. Retrieval is only as +good as the slot descriptions you write. + +## Prerequisites + +| Layer | Resource | Purpose | +|-------|----------|---------| +| Schema | `schemas/artifacts/scene_plan.schema.json` | Artifact validation | +| Prior artifact | `state.artifacts["idea"]["brief"]` | Thematic question, tone, duration, shape | +| Reference | `skills/pipelines/documentary-montage/executive-producer.md` | Cross-stage rules | +| Tools | none yet — this stage is pure planning | — | + +## Mental Model + +The scene director's job is NOT "pick clips". It is "describe what the +clips need to be plainly enough that CLIP can find them". + +Think like a location scout, not a stock librarian. + +- A stock librarian says: *"rain in the city montage, 15 clips"*. +- A location scout says: *"rain streaking sideways across a bus + window at blue hour, passengers' faces in soft focus, traffic + lights bleeding red and green through the glass"*. + +The second one is what CLIP can actually rank. The first one is a +category label CLIP will match weakly and indiscriminately. + +## Process + +### 1. Turn The Shape Into A Beat Count + +Read the brief's `duration_seconds` and `shape`. Derive the number of +slots. Use these defaults unless the tone says otherwise: + +| Tone | Average hold | Slots per 60s | +|------|--------------|---------------| +| elegiac | 4.0s | ~15 | +| reverent | 3.5s | ~17 | +| dreamlike | 3.0s | ~20 | +| wry | 2.0s | ~30 | +| urgent | 1.2s | ~50 | + +Then plan the arc according to shape: + +- **list**: N uniform slots, no inflection. +- **before/after**: N/2 before slots + 1 pivot slot + N/2 after slots. +- **three-act**: setup (30%) → turn (40%) → release (30%). +- **single-image expansion**: 1 anchor image + N variations around it. + +Write the beat count down before writing any slot. + +### 2. Decompose The Thematic Question Into Concrete Beats + +Take the ONE thematic question from the brief and answer it in +sensory language. Not themes — textures. + +**Example — "What does rain show you about a city?"** + +Bad decomposition (abstract, unsearchable): + +- "establishing the mood of the city" +- "the feeling of being caught in weather" +- "the universality of rain" + +Good decomposition (concrete, searchable): + +- a single raindrop hitting dry asphalt in slow motion +- an umbrella opening in a doorway, a hand visible +- neon signs reflected upside-down in a puddle +- rain streaking across a bus window, passengers soft +- a taxi roof light pushing through heavy rain, long lens +- a storm drain swallowing leaves and water, overhead +- a street vendor pulling plastic over a produce cart +- steam rising off wet cobblestones under tungsten streetlight +- a child's rubber boot stamping into a puddle +- a lit apartment window seen through sheets of rain + +Each of those is a SHOT. Each is CLIP-rankable. Each is also *a +different angle on the same idea*, which is what gives a list-shaped +montage its weight. + +### 3. Write The Slot Description + +Every slot carries a `description` field. This is the text CLIP will +embed and rank against. Write it like a good stock-footage tag string +— nouns and adjectives, no verbs of intention, no emotion words. + +**Template:** + +``` +, , , , +``` + +**Good:** + +- `"a single raindrop hitting dry asphalt, close up, slow motion, + warm streetlamp glow"` +- `"empty city sidewalk at night after rain, reflected neon, + handheld, 1970s grain"` +- `"an umbrella opening in a doorway, hand visible, diffused + afternoon light, shallow focus"` + +**Bad:** + +- `"the feeling of arriving home"` — emotion word, no subject +- `"a warm welcoming moment"` — adjective soup, no image +- `"someone going through a door in a symbolic way"` — intent, no shot + +Rule of thumb: if you can't imagine a specific photograph from the +description, CLIP can't either. + +### 4. Write 2-3 Queries Per Slot + +The slot description is what CLIP ranks against. The queries are what +the `corpus_builder` uses to populate the candidate pool. These are +different jobs, so write them differently. + +Give each slot a `queries` array with 2-3 entries: + +1. **Literal query** — the most direct stock-search phrase. This is + what a Pexels user would type. `"raindrop on asphalt slow motion"`. +2. **Lateral query** — the same idea from a different angle or scale. + `"wet pavement close up"`. +3. **Association query** (optional, for hero slots) — an adjacent + concept that might surface texture clips the literal query misses. + `"first rain city street"`. + +Short queries beat long queries for stock search engines. 2-5 words +each. No filler words. + +### 5. Target Sources Per Slot (Era-Aware) + +Read `brief.era_mix`. Assign each slot one or more `preferred_sources` +based on what footage lives where: + +| Source | Strengths | Use when | +|--------|-----------|----------| +| `pexels` | Modern HD footage, clean shots, people, cities, nature | Default for modern/any era | +| `archive_org` | Prelinger home movies, mid-century educational film, 1940s-1980s texture | Vintage, wry, dreamlike, anything nostalgic | +| `nasa` | Earth-from-orbit, astronomy, flight, scale imagery | Reverent, anything about scale, space, planet, flight | + +If `era_mix = "vintage"`, bias slots toward `archive_org` and write +queries in period-appropriate vocabulary ("commuter", "housewife", +"suburb" not "influencer", "wfh", "coworking"). + +If `era_mix = "any"`, mix sources per slot — the scene director +decides which slot gets which source based on the beat's meaning. + +### 6. Mark Hero Slots + +Every montage has 2-3 slots the whole piece depends on: the opening +image, the turn, the final image. Mark these with `hero: true` in the +slot metadata. + +Hero slots get: + +- longer holds (2-4s instead of the tone's default), +- bigger candidate pools at asset time (k=30 instead of k=10), +- more queries (3 instead of 2). + +### 7. Leave Headroom For The Asset Stage + +Don't over-specify. The asset director's job is to rank candidates +against your description. If you nail down the description AND the +exact clip, you've done the asset director's job badly and pre-empted +its creative choices. + +Rule: describe the slot the way you would describe it to a research +assistant over the phone — specific enough to recognise, loose enough +to surprise you. + +### 8. Record The Shot List + +Use the `scene_plan.schema.json` artifact with one `scene` per slot. +For this pipeline, put documentary-montage-specific fields inside +`metadata` on each scene. The canonical shape: + +```json +{ + "version": "1.0", + "scenes": [ + { + "id": "slot_01", + "type": "broll", + "description": "a single raindrop hitting dry asphalt, close up, slow motion, warm streetlamp glow", + "start_seconds": 0.0, + "end_seconds": 3.5, + "narrative_role": "establish_context", + "hero_moment": true, + "texture_keywords": ["wet", "slow motion", "streetlamp"], + "required_assets": [ + { "type": "video", "description": "raindrop on asphalt", "source": "source" } + ] + } + ], + "metadata": { + "pipeline": "documentary-montage", + "shape": "list", + "tone": "elegiac", + "thematic_question": "What does rain show you about a city?", + "slots": [ + { + "id": "slot_01", + "description": "a single raindrop hitting dry asphalt, close up, slow motion, warm streetlamp glow", + "hero": true, + "preferred_sources": ["pexels", "archive_org"], + "queries": [ + "raindrop on asphalt slow motion", + "wet pavement close up", + "first rain city street" + ], + "min_duration": 3.0, + "target_hold_seconds": 3.5, + "era_hint": "any" + } + ] + } +} +``` + +The `scenes[]` array satisfies the schema. The `metadata.slots[]` +array is what the asset director actually reads — it carries the +retrieval-specific fields (`queries`, `preferred_sources`, `hero`, +`era_hint`) that `scene_plan.schema.json` doesn't know about. + +### 9. Quality Gate + +- Slot count matches the beat-count math from step 1. +- Every slot `description` follows the noun-and-adjective template — + no emotion words, no verbs of intention. +- Every slot has 2-3 short queries (5 words or fewer each). +- At least 2 slots are marked `hero`. +- Sum of `target_hold_seconds` is within ±10% of `brief.duration_seconds`. +- If `era_mix = "vintage"`, at least 60% of slots list `archive_org` + in `preferred_sources`. +- `metadata.thematic_question` echoes the brief verbatim (sanity check + that you didn't drift). + +## Common Pitfalls + +- **Writing slot descriptions as intentions instead of images.** "A + moment of hesitation before entering" is a screenplay direction, not + a CLIP query. "A woman standing still on a porch, hand near the + knob" is. +- **Category queries.** `"home"` and `"family"` match everything and + nothing. Push for concrete nouns: door, mat, key, hall, shoe. +- **One-query slots.** The second query is cheap insurance — if the + first query returns junk, the corpus still has something usable. +- **Forgetting duration math.** 90 elegiac seconds is ~15 holds of + ~6s. If you wrote 40 slots, you've drafted an urgent piece by + accident. +- **Skipping `era_hint` on a vintage brief.** Pexels will flood the + corpus with 2020s HD footage and bury the Prelinger material. +- **Letting the thematic question drift.** If the brief says "coming + home" and your slot list has three shots of airplanes, the piece + will be about travel, not home. Re-read the brief after drafting. + +## Worked Example — "A Minute in the Rain" + +- Duration: 90s, elegiac tone → ~15 slots at ~6s each. +- Shape: list (catalogue of weather + city). +- Thematic question: "What does rain show you about a city?" + +Sketch of slots (abbreviated): + +1. **hero** single raindrop hitting dry asphalt, slow motion +2. umbrella opening in a doorway, diffused afternoon light +3. neon sign reflected upside-down in a puddle, handheld +4. rain streaking across a bus window, passengers soft focus +5. a taxi roof light pushing through heavy rain, long lens +6. storm drain swallowing leaves and water, overhead +7. a street vendor pulling plastic over a produce cart +8. wet cobblestone alley, steam rising, tungsten streetlamp +9. rooftop antennae against a grey sky, wide shot +10. a child's rubber boot stamping a puddle, low angle +11. **hero** a lit apartment window seen through sheets of rain +12. windshield wipers at night, colored city lights beyond +13. rain beading on a parked bicycle seat, macro +14. footprints filling with water on a tiled station floor +15. **hero** first patch of blue sky breaking through grey clouds + +Each slot gets: + +- `description` in the noun-and-adjective template, +- 2-3 short queries (e.g. slot 5: `"taxi heavy rain", "yellow cab + wet street night", "city traffic downpour"`), +- `preferred_sources` (slots 1-6 → pexels+archive_org, slot 8 → + archive_org for period texture, slot 11 → pexels), +- `hero: true` on slots 1, 11, 15, +- `target_hold_seconds` summing to ~90. + +This is the artifact the asset director will run retrieval against. diff --git a/tools/video/clip_search.py b/tools/video/clip_search.py new file mode 100644 index 0000000..a141a6d --- /dev/null +++ b/tools/video/clip_search.py @@ -0,0 +1,356 @@ +"""Clip search: unified retrieval interface over a local clip corpus. + +This is the tool the documentary-montage director calls at edit time. +It loads a corpus built by `corpus_builder` and exposes every +retrieval operation the agent needs through a single dispatch +interface. + +Operations +---------- +- **rank_for_slot**: embed a text description of a scene slot and + return the top-k clips by fused visual+tag similarity. The agent's + main building block — "for this slot in the montage, what clips + match?" +- **find_similar_set**: given one seed clip, return N clips that share + the seed's visual register but are diverse from each other (MMR). + Used for "collection" shots — all the doorways, all the footsteps, + all the keys-in-locks. +- **diversify**: given a pre-selected list of clip_ids, greedily keep + the most mutually-dissimilar subset. Used at arrangement time to + prevent visually-redundant adjacent cuts. +- **get**: look up one clip_id and return its full provenance dict. +- **stats**: summary counts (rows, per-source breakdown, mean motion). + +All operations return JSON-serialisable dicts so the tool contract +stays clean across process boundaries. ClipRecords are converted via +`dataclasses.asdict`. + +The corpus is loaded fresh on every call. This keeps the tool +stateless — the agent can call it from multiple stages without +worrying about caches drifting out of sync. For a 1000-row corpus +the load cost is <50 ms. +""" +from __future__ import annotations + +import time +from dataclasses import asdict +from pathlib import Path +from typing import Any, Optional + +from tools.base_tool import ( + BaseTool, + Determinism, + ExecutionMode, + ResourceProfile, + ToolResult, + ToolRuntime, + ToolStability, + ToolStatus, + ToolTier, +) + + +class ClipSearch(BaseTool): + name = "clip_search" + version = "0.1.0" + tier = ToolTier.ANALYZE + capability = "clip_retrieval" + provider = "openmontage" + stability = ToolStability.EXPERIMENTAL + execution_mode = ExecutionMode.SYNC + determinism = Determinism.DETERMINISTIC + runtime = ToolRuntime.LOCAL + + dependencies = [ + "python:numpy", + "python:transformers", + "python:torch", + ] + install_instructions = ( + "pip install numpy transformers torch\n" + "Requires a corpus built by corpus_builder at ." + ) + agent_skills = [] + + capabilities = [ + "text_to_clip_ranking", + "visual_knn", + "mmr_diversification", + "provenance_lookup", + ] + supports = { + "fused_visual_tag_scoring": True, + "motion_filter": True, + "kind_filter": True, + "exclude_list": True, + } + best_for = [ + "picking clips for a specific slot in a montage", + "finding collection-style sets from one seed clip", + "de-duplicating a candidate list before edit arrangement", + ] + not_good_for = [ + "searching the internet (use corpus_builder to populate first)", + "editing or composing video (use video_compose)", + ] + + input_schema = { + "type": "object", + "required": ["operation", "corpus_dir"], + "properties": { + "operation": { + "type": "string", + "enum": [ + "rank_for_slot", + "find_similar_set", + "diversify", + "get", + "stats", + ], + }, + "corpus_dir": { + "type": "string", + "description": "Path to the corpus built by corpus_builder.", + }, + # rank_for_slot + "query_text": { + "type": "string", + "description": "Text description of the scene slot. " + "Embedded by CLIP for similarity ranking.", + }, + "k": {"type": "integer", "default": 10, "minimum": 1}, + "tag_weight": { + "type": "number", + "default": 0.3, + "minimum": 0.0, + "maximum": 1.0, + "description": "Blend between visual (1-w) and tag (w) channels.", + }, + "motion_min": { + "type": "number", + "description": "Reject clips with motion_score below this. " + "Use ~1.5 to filter dead-still clips.", + }, + "kind": { + "type": "string", + "enum": ["video", "image"], + "description": "Filter to only one media type.", + }, + "exclude_ids": { + "type": "array", + "items": {"type": "string"}, + "description": "Clip ids to skip (already used in this edit).", + }, + # find_similar_set + "seed_clip_id": {"type": "string"}, + "n": {"type": "integer", "default": 5, "minimum": 1}, + "diversity": { + "type": "number", + "default": 0.3, + "minimum": 0.0, + "maximum": 1.0, + }, + "candidate_pool": {"type": "integer", "default": 30}, + # diversify + "candidate_ids": {"type": "array", "items": {"type": "string"}}, + # get + "clip_id": {"type": "string"}, + }, + } + + resource_profile = ResourceProfile( + cpu_cores=1, ram_mb=1024, vram_mb=0, disk_mb=50, network_required=False + ) + side_effects = [] + user_visible_verification = [ + "Inspect returned clip_ids and visit thumb_dir/frame_02.jpg " + "to verify the retrieval matches the slot description.", + ] + + def get_status(self) -> ToolStatus: + try: + import numpy # noqa: F401 + import torch # noqa: F401 + import transformers # noqa: F401 + except ImportError: + return ToolStatus.UNAVAILABLE + return ToolStatus.AVAILABLE + + def estimate_cost(self, inputs: dict[str, Any]) -> float: + return 0.0 + + # ------------------------------------------------------------------ + # Execute + # ------------------------------------------------------------------ + + def execute(self, inputs: dict[str, Any]) -> ToolResult: + start = time.time() + try: + from lib.corpus import Corpus + + operation = inputs["operation"] + corpus_dir = Path(inputs["corpus_dir"]) + + corp = Corpus(corpus_dir) + corp.load() + + if operation == "stats": + payload = _op_stats(corp) + elif operation == "rank_for_slot": + payload = _op_rank_for_slot(corp, inputs) + elif operation == "find_similar_set": + payload = _op_find_similar_set(corp, inputs) + elif operation == "diversify": + payload = _op_diversify(corp, inputs) + elif operation == "get": + payload = _op_get(corp, inputs) + else: + return ToolResult( + success=False, + error=f"Unknown operation: {operation!r}", + ) + + return ToolResult( + success=True, + data={ + "operation": operation, + "corpus_dir": str(corpus_dir), + "corpus_size": len(corp), + **payload, + }, + duration_seconds=round(time.time() - start, 3), + cost_usd=0.0, + ) + except Exception as e: + import traceback + return ToolResult( + success=False, + error=f"{type(e).__name__}: {e}\n{traceback.format_exc()[-800:]}", + ) + + +# ---------------------------------------------------------------------- +# Operations +# ---------------------------------------------------------------------- + + +def _op_stats(corp) -> dict[str, Any]: + """Summary counts and per-source breakdown. + + Useful as a sanity check before running expensive retrieval loops — + "does the corpus I loaded actually have enough clips to satisfy the + edit plan?" + """ + import numpy as np + + if len(corp) == 0: + return { + "rows": 0, + "per_source": {}, + "per_kind": {}, + "mean_motion_score": 0.0, + "mean_duration": 0.0, + } + + per_source: dict[str, int] = {} + per_kind: dict[str, int] = {} + motion_scores: list[float] = [] + durations: list[float] = [] + for rec in corp.records: + per_source[rec.source] = per_source.get(rec.source, 0) + 1 + per_kind[rec.kind] = per_kind.get(rec.kind, 0) + 1 + motion_scores.append(rec.motion_score) + durations.append(rec.duration) + + return { + "rows": len(corp), + "per_source": per_source, + "per_kind": per_kind, + "mean_motion_score": float(np.mean(motion_scores)) if motion_scores else 0.0, + "mean_duration": float(np.mean(durations)) if durations else 0.0, + } + + +def _op_rank_for_slot(corp, inputs: dict[str, Any]) -> dict[str, Any]: + """Embed `query_text` and return top-k clips by fused similarity. + + This is the agent's main retrieval move. The returned list is + ordered best-first and every entry carries a score so the agent + can decide whether the match is strong enough (>= 0.25 is a rough + "acceptable" threshold for CLIP ViT-B/32). + """ + from lib.clip_embedder import embed_texts + + query_text = inputs.get("query_text", "").strip() + if not query_text: + raise ValueError("rank_for_slot requires 'query_text'") + + q_vec = embed_texts([query_text])[0] + + results = corp.rank_by_text( + query_embedding=q_vec, + k=int(inputs.get("k", 10)), + tag_weight=float(inputs.get("tag_weight", 0.3)), + motion_min=inputs.get("motion_min"), + kind=inputs.get("kind"), + exclude_ids=inputs.get("exclude_ids") or [], + ) + return { + "query_text": query_text, + "results": [ + {"score": score, "record": asdict(rec)} + for rec, score in results + ], + } + + +def _op_find_similar_set(corp, inputs: dict[str, Any]) -> dict[str, Any]: + """MMR-based similar-set retrieval from one seed clip.""" + seed = inputs.get("seed_clip_id") + if not seed: + raise ValueError("find_similar_set requires 'seed_clip_id'") + + results = corp.find_similar_set( + seed_clip_id=seed, + n=int(inputs.get("n", 5)), + diversity=float(inputs.get("diversity", 0.3)), + candidate_pool=int(inputs.get("candidate_pool", 30)), + exclude_ids=inputs.get("exclude_ids") or [], + ) + return { + "seed_clip_id": seed, + "results": [ + {"score": score, "record": asdict(rec)} + for rec, score in results + ], + } + + +def _op_diversify(corp, inputs: dict[str, Any]) -> dict[str, Any]: + """Pick the most mutually-dissimilar subset of a candidate list.""" + candidate_ids = inputs.get("candidate_ids") or [] + if not candidate_ids: + raise ValueError("diversify requires 'candidate_ids'") + + kept = corp.diversify( + candidate_ids=list(candidate_ids), + n=int(inputs.get("n", 5)), + diversity=float(inputs.get("diversity", 0.5)), + ) + return { + "input_count": len(candidate_ids), + "kept_count": len(kept), + "kept_ids": kept, + } + + +def _op_get(corp, inputs: dict[str, Any]) -> dict[str, Any]: + """Look up one clip_id and return its full record.""" + clip_id = inputs.get("clip_id") + if not clip_id: + raise ValueError("get requires 'clip_id'") + + rec = corp.get(clip_id) + if rec is None: + return {"clip_id": clip_id, "found": False, "record": None} + return {"clip_id": clip_id, "found": True, "record": asdict(rec)} diff --git a/tools/video/corpus_builder.py b/tools/video/corpus_builder.py new file mode 100644 index 0000000..28f7717 --- /dev/null +++ b/tools/video/corpus_builder.py @@ -0,0 +1,553 @@ +"""Corpus builder: fan out across stock sources, download, thumb, embed, index. + +This is the tool the agent calls to populate a local clip corpus for +the documentary-montage pipeline. It is deliberately the ONLY place +where adapters, embedding, and the `Corpus` class meet — everything +downstream (retrieval, selection, edit planning) reads from the corpus +on disk and never touches sources directly. + +What it does, per query, per source, per candidate +--------------------------------------------------- +1. Call `source.search(query, filters)` to get a flat list of + `Candidate`s normalised across sources. +2. Skip candidates whose `clip_id` is already in the corpus (unless + `skip_existing=false`). +3. Download the file to ``/clips/.``. +4. For videos: extract N evenly-spaced frames to + ``/thumbnails//frame_NN.jpg``, probe real + dimensions and duration, and compute a cheap motion score + (mean-abs-diff between first and middle frame). +5. For images: copy the image as ``frame_00.jpg`` in the same thumb + directory so the embedder has a consistent input. +6. Run CLIP on the thumbnails, pool frames to one 512-d vector for the + visual channel. Run CLIP on `source_tags` (falling back to the + query itself) for the tag channel. +7. Materialise a `ClipRecord` with every provenance field and append + it to the corpus via `Corpus.add()`. +8. After ALL candidates are processed, call `Corpus.save()` once. Per- + add saves would burn disk I/O for large runs. + +Caps +---- +- `max_new_clips` halts the whole run once that many new rows have + been added. The remaining candidates in the current loop iteration + are discarded. +- Per-source search errors and per-candidate processing errors are + caught and collected into `errors` in the return payload. One flaky + URL or one broken codec must not poison the whole run. + +Idempotence +----------- +Re-running with the same inputs is safe. `skip_existing=True` (default) +causes the tool to short-circuit on any `clip_id` already in the +corpus JSONL. Crash-recovery is handled by `Corpus.load()`, which +truncates the in-memory state to the shorter of the JSONL and the +`.npy` lengths. + +Agent surface +------------- +Input schema keeps the agent's decisions at the top: WHAT to search +for (`queries`), WHERE to search (`sources`), WHAT to filter +(`filters`), and HOW MUCH (`max_new_clips`). Everything else has +sensible defaults. +""" +from __future__ import annotations + +import time +import urllib.parse +from pathlib import Path +from typing import Any, Optional + +from tools.base_tool import ( + BaseTool, + Determinism, + ExecutionMode, + ResourceProfile, + ToolResult, + ToolRuntime, + ToolStability, + ToolStatus, + ToolTier, +) + + +class CorpusBuilder(BaseTool): + name = "corpus_builder" + version = "0.1.0" + tier = ToolTier.SOURCE + capability = "corpus_population" + provider = "openmontage" + stability = ToolStability.EXPERIMENTAL + execution_mode = ExecutionMode.SYNC + determinism = Determinism.DETERMINISTIC + runtime = ToolRuntime.HYBRID # local compute + network APIs + + dependencies = [ + "python:cv2", + "python:numpy", + "python:requests", + "python:PIL", + "python:transformers", + "python:torch", + ] + install_instructions = ( + "pip install opencv-python numpy requests pillow transformers torch\n" + "At least one stock source must be configured:\n" + " PEXELS_API_KEY for Pexels (free at https://www.pexels.com/api/)\n" + " archive.org and nasa work without API keys" + ) + agent_skills = [] + + capabilities = [ + "stock_fanout_search", + "corpus_population", + "clip_indexing", + "clip_embedding", + ] + supports = { + "multi_source": True, + "video_and_image": True, + "append_only": True, + "resumable": True, + } + best_for = [ + "documentary-montage retrieval corpora", + "topic-based offline clip indexing", + "collecting candidate B-roll without repeated API calls per edit", + ] + not_good_for = [ + "single-clip downloads (use pexels_video instead)", + "semantic retrieval itself (use clip_search)", + ] + fallback_tools = [] + + input_schema = { + "type": "object", + "required": ["corpus_dir", "queries"], + "properties": { + "corpus_dir": { + "type": "string", + "description": "Project-local corpus directory, e.g. projects/foo/corpus", + }, + "queries": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "required": ["query"], + "properties": { + "query": {"type": "string"}, + "kind": { + "type": "string", + "enum": ["video", "image", "any"], + "default": "video", + }, + "per_source": { + "type": "integer", + "default": 10, + "minimum": 1, + "maximum": 80, + }, + }, + }, + }, + "sources": { + "type": "array", + "items": {"type": "string"}, + "description": "Source names to use (e.g. ['pexels','archive_org']). " + "Defaults to all available.", + }, + "filters": { + "type": "object", + "properties": { + "min_duration": {"type": "number"}, + "max_duration": {"type": "number"}, + "orientation": { + "type": "string", + "enum": ["landscape", "portrait", "square"], + }, + "min_width": {"type": "integer"}, + }, + }, + "max_new_clips": { + "type": "integer", + "default": 100, + "minimum": 1, + "description": "Halt after this many NEW rows have been added.", + }, + "skip_existing": {"type": "boolean", "default": True}, + "thumbs_per_video": { + "type": "integer", + "default": 5, + "minimum": 1, + "maximum": 20, + }, + }, + } + + resource_profile = ResourceProfile( + cpu_cores=2, ram_mb=2048, vram_mb=0, disk_mb=4000, network_required=True + ) + side_effects = [ + "downloads clips to /clips", + "writes thumbnails under /thumbnails", + "appends rows to /index.jsonl + embedding .npy files", + "calls external stock APIs", + ] + user_visible_verification = [ + "Open /index.jsonl and inspect a few added rows", + "Open /thumbnails//frame_02.jpg visually", + ] + + def get_status(self) -> ToolStatus: + try: + from tools.video.stock_sources import available_sources + except Exception: + return ToolStatus.UNAVAILABLE + return ToolStatus.AVAILABLE if available_sources() else ToolStatus.UNAVAILABLE + + def estimate_cost(self, inputs: dict[str, Any]) -> float: + return 0.0 # all sources are free-tier + + # ------------------------------------------------------------------ + # Execute + # ------------------------------------------------------------------ + + def execute(self, inputs: dict[str, Any]) -> ToolResult: + start = time.time() + try: + from lib.corpus import Corpus + from tools.video.stock_sources import ( + SearchFilters, + available_sources, + get_source, + ) + + corpus_dir = Path(inputs["corpus_dir"]) + queries: list[dict] = list(inputs["queries"]) + source_names: Optional[list[str]] = inputs.get("sources") + filters_in: dict = inputs.get("filters") or {} + max_new = int(inputs.get("max_new_clips", 100)) + skip_existing = bool(inputs.get("skip_existing", True)) + thumbs_per_video = int(inputs.get("thumbs_per_video", 5)) + + # Resolve sources. If the caller passed an explicit list we + # respect it even if some are unavailable — their search + # calls will simply fail and be logged. + if source_names: + requested: list = [] + for name in source_names: + try: + s = get_source(name) + except KeyError as e: + return ToolResult(success=False, error=str(e)) + if s.is_available(): + requested.append(s) + sources = requested + else: + sources = available_sources() + + if not sources: + return ToolResult( + success=False, + error="No stock sources available. " + self.install_instructions, + ) + + corp = Corpus(corpus_dir) + corp.load() + corp.ensure_dirs() + + per_source_counts: dict[str, int] = {s.name: 0 for s in sources} + added_ids: list[str] = [] + errors: list[dict] = [] + skipped = 0 + failed = 0 + candidates_seen = 0 + + def filters_for(q_spec: dict) -> SearchFilters: + return SearchFilters( + kind=q_spec.get("kind", "video"), + per_page=int(q_spec.get("per_source", 10)), + min_duration=filters_in.get("min_duration"), + max_duration=filters_in.get("max_duration"), + orientation=filters_in.get("orientation"), + min_width=filters_in.get("min_width"), + ) + + for q_spec in queries: + if len(added_ids) >= max_new: + break + query = q_spec["query"] + f = filters_for(q_spec) + + for src in sources: + if len(added_ids) >= max_new: + break + try: + cands = src.search(query, f) + except Exception as e: + errors.append({ + "phase": "search", + "source": src.name, + "query": query, + "error": f"{type(e).__name__}: {e}", + }) + continue + + candidates_seen += len(cands) + for cand in cands: + if len(added_ids) >= max_new: + break + + if skip_existing and corp.has(cand.clip_id): + skipped += 1 + continue + + try: + rec = self._process_candidate( + cand=cand, + src=src, + corp=corp, + query=query, + thumbs_per_video=thumbs_per_video, + ) + except Exception as e: + failed += 1 + errors.append({ + "phase": "process", + "clip_id": cand.clip_id, + "error": f"{type(e).__name__}: {e}", + }) + continue + + if rec is None: + failed += 1 + continue + added_ids.append(rec.clip_id) + per_source_counts[src.name] = per_source_counts.get(src.name, 0) + 1 + + # Single save at the end. Corpus.save() writes JSONL first + # (source of truth) then both .npy files, so a crash mid-save + # still leaves a loadable corpus. + corp.save() + + elapsed = time.time() - start + return ToolResult( + success=True, + data={ + "corpus_dir": str(corpus_dir), + "queries_run": len(queries), + "candidates_seen": candidates_seen, + "clips_added": len(added_ids), + "clips_skipped_existing": skipped, + "clips_failed": failed, + "per_source_counts": per_source_counts, + "added_ids": added_ids, + "total_corpus_size": len(corp), + "errors": errors[:25], # cap log noise + }, + cost_usd=0.0, + duration_seconds=round(elapsed, 2), + ) + except Exception as e: + import traceback + return ToolResult( + success=False, + error=f"{type(e).__name__}: {e}\n{traceback.format_exc()[-800:]}", + ) + + # ------------------------------------------------------------------ + # Per-candidate pipeline + # ------------------------------------------------------------------ + + def _process_candidate( + self, + cand, + src, + corp, + query: str, + thumbs_per_video: int, + ): + """Download → thumb → embed → add one Candidate to the corpus. + + Returns the created `ClipRecord` on success, None if the clip + was rejected (download empty, thumb extraction failed, etc.). + Raises on unexpected errors (the caller logs them). + """ + import cv2 + + from lib.clip_embedder import embed_images, embed_texts, pool_frames + from lib.corpus import ClipRecord + + # Pick file extension from the URL path (sources give us + # stable .mp4/.jpg/.png URLs) with a kind-aware fallback. + ext = _guess_ext(cand) + local_rel = Path("clips") / f"{cand.clip_id}{ext}" + local_abs = corp.corpus_dir / local_rel + + # Download. Any HTTP/IO exception propagates up to the + # per-candidate try in execute(). + src.download(cand, local_abs) + if not local_abs.exists() or local_abs.stat().st_size < 1024: + # Empty / near-empty file = bad download. Clean up so a + # retry doesn't mistake it for success. + try: + if local_abs.exists(): + local_abs.unlink() + except OSError: + pass + return None + + thumb_dir_rel = Path("thumbnails") / cand.clip_id + thumb_dir_abs = corp.corpus_dir / thumb_dir_rel + thumb_dir_abs.mkdir(parents=True, exist_ok=True) + + width = cand.width + height = cand.height + duration = cand.duration + motion_score = 0.0 + + if cand.kind == "video": + thumb_paths, probe = _extract_video_thumbs( + local_abs, thumb_dir_abs, thumbs_per_video + ) + if not thumb_paths: + return None + if probe: + width = probe.get("width") or width + height = probe.get("height") or height + duration = probe.get("duration") or duration + motion_score = float(probe.get("motion_score", 0.0)) + else: + dst = thumb_dir_abs / "frame_00.jpg" + if not _save_as_jpeg(local_abs, dst): + return None + thumb_paths = [dst] + img = cv2.imread(str(local_abs)) + if img is not None: + height, width = img.shape[:2] + + # CLIP embeddings. embed_images loads the model lazily on + # first call; subsequent candidates reuse the cached model. + clip_frames = embed_images(thumb_paths) + clip_vec = pool_frames(clip_frames) + + # Tag channel: prefer source-supplied tags/description, + # fall back to the query so the row still carries SOME text + # signal. pool_frames returns zeros if empty so this never + # breaks the fused ranking math. + tag_text = cand.source_tags or query + tag_vec = embed_texts([tag_text])[0] + + rec = ClipRecord( + clip_id=cand.clip_id, + source=cand.source, + source_id=cand.source_id, + source_url=cand.source_url, + local_path=str(local_rel).replace("\\", "/"), + kind=cand.kind, + thumb_dir=str(thumb_dir_rel).replace("\\", "/"), + query=query, + creator=cand.creator, + license=cand.license, + duration=float(duration or 0.0), + width=int(width or 0), + height=int(height or 0), + motion_score=motion_score, + dominant_colors=[], + source_tags=cand.source_tags, + ) + corp.add(rec, clip_vec, tag_vec) + return rec + + +# ---------------------------------------------------------------------- +# Module-level helpers (kept outside the class so tests can hit them) +# ---------------------------------------------------------------------- + + +def _guess_ext(cand) -> str: + """Extract a sensible file extension from a candidate's URL.""" + known = {".mp4", ".mov", ".mkv", ".webm", ".ogv", ".m4v", + ".jpg", ".jpeg", ".png", ".tif", ".tiff"} + path = urllib.parse.urlparse(cand.download_url).path + ext = Path(path).suffix.lower() + if ext in known: + # Normalise .jpeg→.jpg for consistent clip paths + return ".jpg" if ext == ".jpeg" else ext + return ".mp4" if cand.kind == "video" else ".jpg" + + +def _extract_video_thumbs( + video_path: Path, out_dir: Path, n_frames: int +) -> tuple[list[Path], dict]: + """Extract `n` evenly-spaced JPEG thumbnails from a video. + + Returns ``(thumb_paths, probe_dict)``. The probe dict carries the + real dimensions, duration, and a cheap motion score (mean abs + pixel diff between frame 0 and the middle frame). Used to backfill + `ClipRecord` fields that the source API didn't give us. + """ + import cv2 + + cap = cv2.VideoCapture(str(video_path)) + if not cap.isOpened(): + return [], {} + + total = int(cap.get(cv2.CAP_PROP_FRAME_COUNT) or 0) + fps = float(cap.get(cv2.CAP_PROP_FPS) or 0) + width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH) or 0) + height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT) or 0) + duration = total / fps if fps > 0 else 0.0 + + if total < 2: + cap.release() + return [], {} + + n = max(1, min(n_frames, total)) + # Space positions at the midpoints of equal segments so we skip the + # first/last frame (often black or a splash). + positions = [int(round((i + 0.5) * total / n)) for i in range(n)] + + thumb_paths: list[Path] = [] + captured: list = [] + for idx, pos in enumerate(positions): + cap.set(cv2.CAP_PROP_POS_FRAMES, max(0, min(pos, total - 1))) + ok, frame = cap.read() + if not ok or frame is None: + continue + dst = out_dir / f"frame_{idx:02d}.jpg" + cv2.imwrite(str(dst), frame, [int(cv2.IMWRITE_JPEG_QUALITY), 88]) + thumb_paths.append(dst) + captured.append(frame) + cap.release() + + motion = 0.0 + if len(captured) >= 2: + import numpy as np + a = cv2.cvtColor(captured[0], cv2.COLOR_BGR2GRAY).astype(np.float32) + b = cv2.cvtColor(captured[len(captured) // 2], cv2.COLOR_BGR2GRAY).astype(np.float32) + motion = float(np.abs(a - b).mean()) + + return thumb_paths, { + "width": width, + "height": height, + "duration": duration, + "motion_score": motion, + } + + +def _save_as_jpeg(src_path: Path, dst_path: Path) -> bool: + """Load an arbitrary image file and re-save as JPEG. + + Handles PNG/JPEG/TIFF/WebP inputs — anything cv2.imread understands. + Returns True on success, False on unreadable input (so the caller + can reject the candidate cleanly). + """ + import cv2 + + img = cv2.imread(str(src_path)) + if img is None: + return False + cv2.imwrite(str(dst_path), img, [int(cv2.IMWRITE_JPEG_QUALITY), 88]) + return True diff --git a/tools/video/stock_sources/__init__.py b/tools/video/stock_sources/__init__.py new file mode 100644 index 0000000..e4b3bf9 --- /dev/null +++ b/tools/video/stock_sources/__init__.py @@ -0,0 +1,80 @@ +"""Stock media source adapters. + +Unified-protocol wrappers around free stock APIs (Pexels, Archive.org, +NASA, ...) used by `corpus_builder` to populate the local clip corpus. +See `base.py` for the protocol contract and the "adding a new source" +checklist. + +Registry +-------- +`all_sources()` returns one instance of every adapter known to the +package, in a stable order. `available_sources()` filters to the ones +whose `is_available()` returns True right now. The corpus builder +queries the latter during fan-out; it never imports adapters directly. + +To register a new adapter: add its class to the `_REGISTRY` tuple +below. Order matters only as a tiebreak when two sources return +matching clips. +""" +from __future__ import annotations + +from .archive_org import ArchiveOrgSource +from .base import Candidate, SearchFilters, StockSource +from .nasa import NasaSource +from .pexels import PexelsSource + +__all__ = [ + "Candidate", + "SearchFilters", + "StockSource", + "PexelsSource", + "ArchiveOrgSource", + "NasaSource", + "all_sources", + "available_sources", + "get_source", +] + +# Explicit, ordered list of every adapter class the package exposes. +# The corpus builder iterates this (filtered by availability) during +# fan-out. Order matters only as a tiebreak for identical clip ids. +_REGISTRY: tuple[type, ...] = ( + PexelsSource, + ArchiveOrgSource, + NasaSource, +) + + +def all_sources() -> list[StockSource]: + """Instantiate every registered adapter, whether available or not. + + Returned instances are cheap — adapters keep no state beyond env + var reads, so constructing them has no cost. Use this when you + want to show the user what sources exist regardless of whether + their credentials are configured. + """ + return [cls() for cls in _REGISTRY] + + +def available_sources() -> list[StockSource]: + """Return only the adapters whose `is_available()` is True. + + This is what the corpus builder uses during a normal run. An empty + list means no sources are configured — the caller should surface + that to the user with install instructions, not silently produce + an empty corpus. + """ + return [s for s in all_sources() if s.is_available()] + + +def get_source(name: str) -> StockSource: + """Look up a single adapter by its `name` attribute. + + Raises `KeyError` if no registered adapter claims that name. Useful + for tests and for agents that want to pin to a specific source + (e.g. "only Archive.org for this topic"). + """ + for s in all_sources(): + if s.name == name: + return s + raise KeyError(f"No stock source registered with name={name!r}") diff --git a/tools/video/stock_sources/archive_org.py b/tools/video/stock_sources/archive_org.py new file mode 100644 index 0000000..3ca9ca3 --- /dev/null +++ b/tools/video/stock_sources/archive_org.py @@ -0,0 +1,368 @@ +"""Archive.org stock video adapter. + +Targets public-domain film and home-movie collections on archive.org: + + - **prelinger**: Rick Prelinger's archive of ephemeral films + (industrial, educational, advertising, 1920s-1980s). The single + best source of documentary-grade B-roll for 20th century themes. + - **opensource_movies**: broader public-domain and CC-licensed film + uploads — a grab bag but useful when Prelinger is thin on a topic. + - **home_movies**: anonymous personal footage. The soul of any + nostalgic or observational documentary montage lives here. + +Archive.org requires no API key. Everything is open. + +Fetch pattern +------------- +The search API (``advancedsearch.php``) returns metadata records with +identifiers but not file lists. To get downloadable URLs we need a +second call to ``/metadata/`` per hit. This adapter pays +that round-trip cost during `search()` so the `Candidate` it returns +carries a ready-to-use `download_url`. For a per_page of 20, expect +~21 HTTP calls per search — slow but fully serial and cache-friendly. + +Dimensions are sometimes missing from file metadata. When they are, +the `Candidate` carries `width=0, height=0` and the corpus builder is +expected to probe the clip with ffprobe post-download. + +Licence +------- +Prelinger collection items are public domain. Broader +`opensource_movies` items are usually CC0 or CC-BY. We record the +collection and `licenseurl` (when present) so the agent can attribute +correctly if it wants to, but no attribution is legally required. +""" +from __future__ import annotations + +import re +from pathlib import Path +from typing import Any, Optional + +from .base import Candidate, SearchFilters + + +_SEARCH_URL = "https://archive.org/advancedsearch.php" +_METADATA_URL = "https://archive.org/metadata" +_DOWNLOAD_URL = "https://archive.org/download" + +# Default collections to bias toward. Overridable via SearchFilters.extra +# in a future refinement; for now these are baked in because they give +# the best documentary-montage hit rate. +_DEFAULT_COLLECTIONS = ("prelinger", "opensource_movies", "home_movies") + +# File formats we accept, in preference order. Archive.org runs every +# upload through a derivative pipeline so most items have multiple +# renditions — we want the best mp4. +_VIDEO_FORMAT_PRIORITY = ( + "h.264", # mp4, usually 480p or 720p + "MPEG4", # older mp4 encoding + "h.264 HD", # less common, but best quality + "512Kb MPEG4", # last resort — lowest quality derivative + "Matroska", # mkv + "WebM", # webm +) + + +class ArchiveOrgSource: + """Adapter for public-domain video on archive.org. + + Satisfies `StockSource`. Stateless, no credentials. + """ + + name = "archive_org" + + def is_available(self) -> bool: + # No API key, no config. As long as the network is up, we're + # available. The corpus builder will catch network errors in + # the per-source try block. + return True + + # ------------------------------------------------------------------ + # Public protocol + # ------------------------------------------------------------------ + + def search(self, query: str, filters: SearchFilters) -> list[Candidate]: + """Search Archive.org for matching video items. + + Two-stage: (1) advancedsearch for identifiers, (2) per-item + metadata fetch for file lists. Images are not supported — this + adapter returns an empty list for `kind="image"` since + Archive.org's image collections are a separate ecosystem (see + `nasa.py` for astronomy imagery instead). + """ + kind = (filters.kind or "video").lower() + if kind not in ("video", "any"): + return [] + + import requests # lazy + + q = self._build_query(query) + params = [ + ("q", q), + ("fl[]", "identifier"), + ("fl[]", "title"), + ("fl[]", "description"), + ("fl[]", "creator"), + ("fl[]", "date"), + ("fl[]", "subject"), + ("fl[]", "licenseurl"), + ("fl[]", "collection"), + ("rows", str(max(1, min(filters.per_page, 50)))), + ("page", str(max(1, filters.page))), + ("output", "json"), + ] + + r = requests.get(_SEARCH_URL, params=params, timeout=30) + r.raise_for_status() + data = r.json() + docs = (data.get("response") or {}).get("docs", []) or [] + + out: list[Candidate] = [] + for doc in docs: + cand = self._hydrate_candidate(doc, filters) + if cand is not None: + out.append(cand) + return out + + def download(self, candidate: Candidate, out_path: Path) -> Path: + """Stream the candidate's file to `out_path`. + + Same pattern as the Pexels adapter — no caching, corpus builder + decides. + """ + import requests # lazy + + if not candidate.download_url: + raise ValueError( + f"Candidate {candidate.clip_id} has no download_url" + ) + + out_path = Path(out_path) + out_path.parent.mkdir(parents=True, exist_ok=True) + + with requests.get( + candidate.download_url, stream=True, timeout=300 + ) as r: + r.raise_for_status() + with open(out_path, "wb") as f: + for chunk in r.iter_content(chunk_size=1 << 16): + if chunk: + f.write(chunk) + return out_path + + # ------------------------------------------------------------------ + # Internals + # ------------------------------------------------------------------ + + def _build_query(self, user_query: str) -> str: + """Wrap the user's query with mediatype + collection filters. + + Archive.org's query language is Solr-style — parentheses and + booleans work, and spaces default to AND. We quote the user + query so multi-word phrases stay intact. + """ + coll = " OR ".join(f"collection:{c}" for c in _DEFAULT_COLLECTIONS) + user = user_query.strip() + if not user: + return f"mediatype:movies AND ({coll})" + # Quote the user query as a phrase AND a loose-term search so + # we get both precise matches and relevance-ranked hits. + return f'mediatype:movies AND ({coll}) AND ({user})' + + def _hydrate_candidate( + self, doc: dict, filters: SearchFilters + ) -> Optional[Candidate]: + """Turn a search-doc identifier into a full Candidate. + + Fetches the item's file list and picks the best playable + rendition. Returns None if the item has no usable files or if + the item's duration falls outside the filter range. + """ + import requests # lazy + + identifier = doc.get("identifier") + if not identifier: + return None + + try: + r = requests.get(f"{_METADATA_URL}/{identifier}", timeout=30) + r.raise_for_status() + meta = r.json() + except Exception: + # Swallow per-item fetch failures — one bad item shouldn't + # poison the whole search. Alternative would be to raise and + # have corpus_builder catch per-source, but at this layer we + # can keep going. + return None + + files = meta.get("files") or [] + picked = _pick_video_file(files) + if picked is None: + return None + + duration = _parse_length(picked.get("length")) + if filters.min_duration is not None and duration < filters.min_duration: + return None + if filters.max_duration is not None and 0 < duration < filters.max_duration: + # 0 means "unknown" — keep those, reject only known-too-long + pass + if filters.max_duration is not None and duration > filters.max_duration: + return None + + width = _safe_int(picked.get("width")) + height = _safe_int(picked.get("height")) + if filters.min_width is not None and width and width < filters.min_width: + return None + + # Build the direct download URL. archive.org/download// + # is the stable public pattern and works without auth. + file_name = picked.get("name", "") + download_url = f"{_DOWNLOAD_URL}/{identifier}/{file_name}" + + # Tags: flatten title + description + subject. Archive.org is + # verbose, so we truncate to keep the CLIP text encoder focused + # on the most important tokens (77-token limit anyway). + title = _to_text(doc.get("title")) + description = _to_text(doc.get("description")) + subject = _to_text(doc.get("subject")) + source_tags = " ".join( + s for s in (title, description, subject) if s + ).strip() + if len(source_tags) > 500: + source_tags = source_tags[:500] + + creator = _to_text(doc.get("creator")) + collection = _to_text(doc.get("collection")) + license_url = _to_text(doc.get("licenseurl")) + license_text = license_url or _license_from_collection(collection) + + return Candidate( + source=self.name, + source_id=identifier, + source_url=f"https://archive.org/details/{identifier}", + download_url=download_url, + kind="video", + width=width, + height=height, + duration=duration, + creator=creator, + license=license_text, + source_tags=source_tags, + thumbnail_url=f"https://archive.org/services/img/{identifier}", + extra={ + "collection": collection, + "date": _to_text(doc.get("date")), + "format": picked.get("format"), + "file_name": file_name, + "file_size_bytes": _safe_int(picked.get("size")), + }, + ) + + +# ---------------------------------------------------------------------- +# Module-level helpers +# ---------------------------------------------------------------------- + + +def _pick_video_file(files: list[dict]) -> Optional[dict]: + """Pick the best playable video file from an Archive.org files list. + + Preference order is defined by `_VIDEO_FORMAT_PRIORITY`. Within a + format, we prefer the largest file (size as a quality proxy), and + reject obvious thumbnails / derivative animations. + """ + if not files: + return None + + by_format: dict[str, list[dict]] = {} + for f in files: + fmt = (f.get("format") or "").strip() + name = (f.get("name") or "").lower() + # Skip non-video + if fmt not in _VIDEO_FORMAT_PRIORITY: + continue + # Skip degenerate renditions and thumbnails regardless of format + if any(tag in name for tag in ("thumb", "preview", ".gif")): + continue + by_format.setdefault(fmt, []).append(f) + + for fmt in _VIDEO_FORMAT_PRIORITY: + bucket = by_format.get(fmt) + if not bucket: + continue + bucket.sort(key=lambda f: _safe_int(f.get("size")), reverse=True) + return bucket[0] + return None + + +_HMS_RE = re.compile(r"^(\d+):(\d+):(\d+(?:\.\d+)?)$") +_MS_RE = re.compile(r"^(\d+):(\d+(?:\.\d+)?)$") + + +def _parse_length(value: Any) -> float: + """Parse Archive.org's `length` field into seconds. + + The field is usually "HH:MM:SS.ss" or "MM:SS.ss" or a bare float + as a string. Missing / unparseable values return 0.0 which the + caller interprets as "unknown, pass any duration filter". + """ + if value is None: + return 0.0 + if isinstance(value, (int, float)): + return float(value) + s = str(value).strip() + if not s: + return 0.0 + m = _HMS_RE.match(s) + if m: + h, mn, sec = m.groups() + return int(h) * 3600 + int(mn) * 60 + float(sec) + m = _MS_RE.match(s) + if m: + mn, sec = m.groups() + return int(mn) * 60 + float(sec) + try: + return float(s) + except ValueError: + return 0.0 + + +def _safe_int(value: Any) -> int: + """Parse a value to int, tolerating strings, None, and garbage.""" + if value is None: + return 0 + try: + return int(value) + except (TypeError, ValueError): + try: + return int(float(value)) + except (TypeError, ValueError): + return 0 + + +def _to_text(value: Any) -> str: + """Flatten Archive.org's sometimes-list-sometimes-string fields. + + The search API returns `creator`, `subject`, and `description` as + either a string or a list of strings depending on the item. We + always join with spaces so the caller sees a single str. + """ + if value is None: + return "" + if isinstance(value, list): + return " ".join(str(x) for x in value if x is not None).strip() + return str(value).strip() + + +def _license_from_collection(collection: str) -> str: + """Infer licence text from a collection name when no licenseurl is set. + + Prelinger items are universally public domain; broader opensource + collections usually are too but we're less sure, so we hedge. + """ + col = collection.lower() + if "prelinger" in col: + return "Public Domain (Prelinger Archives)" + if "home_movies" in col: + return "Public Domain (archive.org home movies)" + return "Public Domain / CC (archive.org — verify per item)" diff --git a/tools/video/stock_sources/base.py b/tools/video/stock_sources/base.py new file mode 100644 index 0000000..1589105 --- /dev/null +++ b/tools/video/stock_sources/base.py @@ -0,0 +1,140 @@ +"""Unified protocol for stock media source adapters. + +Every source (Pexels, Archive.org, NASA, Wikimedia, Unsplash, ...) +implements the same small interface. The corpus builder fans out across +all enabled sources, normalises their results into `Candidate` objects, +downloads what it decides to keep, and writes everything to the local +corpus. During selection the agent never sees which source a clip came +from — it only sees `ClipRecord` rows retrieved by similarity. + +Design intent +------------- +- **Separation of concerns.** Adapters handle API shape, licensing + metadata, and downloading. `lib/corpus.py` handles indexing and + retrieval math. The agent handles WHAT to search for and WHICH + results to accept. +- **Interchangeable.** Any adapter satisfies the same protocol, so the + corpus builder can iterate `for src in sources: src.search(q, f)` + without branching on source type. +- **Dumb by design.** No ranking, no de-dup, no filtering beyond what + the API itself accepts. Judgment work happens after embedding, in + the agent. Adapters just convert "API JSON" → "normalised Candidate". + +Adding a new source +------------------- +1. Create `tools/video/stock_sources/.py` with a class that + satisfies `StockSource` — a `name` attribute plus `is_available`, + `search`, and `download` methods. +2. Normalise the API's response into `Candidate` objects. Keep every + field the corpus might later want to display or attribute (creator, + licence, source URL, description/tags for the text channel of the + CLIP fused ranking). +3. Register the class so `corpus_builder` can discover it. The + registration pattern lives alongside the first concrete adapter. +""" +from __future__ import annotations + +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Optional, Protocol, runtime_checkable + + +@dataclass +class Candidate: + """One pre-download search result, normalised across sources. + + A `Candidate` is what a source adapter returns from `search()`. It + carries everything the corpus builder needs to decide whether to + download, plus the provenance fields that later get copied into the + `ClipRecord`. + + Fields that don't apply to a given kind default to zero/empty — for + an image, `duration` is 0.0; for a source that doesn't expose fps, + `extra` may hold it or it may be absent entirely. Adapters should + populate as much as they cheaply can and leave the rest alone. + """ + + source: str # adapter name, e.g. "pexels" + source_id: str # unique within that source + source_url: str # landing page (human readable) + download_url: str # direct file URL + kind: str # "video" or "image" + width: int = 0 + height: int = 0 + duration: float = 0.0 # seconds (0 for images) + creator: str = "" # attribution name + license: str = "" # licence string or URL + source_tags: str = "" # title + description + tags joined + thumbnail_url: str = "" # for previews and image-fallback embeds + extra: dict[str, Any] = field(default_factory=dict) # source-specific junk + + @property + def clip_id(self) -> str: + """Stable ID used as the corpus row key. + + Format is ``"_"``. Matches the convention in + `lib/corpus.ClipRecord`, so the corpus builder can copy this + directly when it materialises the row. + """ + return f"{self.source}_{self.source_id}" + + +@dataclass +class SearchFilters: + """Filters a source adapter applies when searching. + + Not every source supports every filter — adapters MAY ignore fields + they don't understand. The corpus builder sets these liberally and + trusts each source to do the best it can with what it has. The + missing filters are caught later by the agent at retrieval time. + """ + + kind: str = "video" # "video", "image", or "any" + min_duration: Optional[float] = None # seconds; None = no floor + max_duration: Optional[float] = None # seconds; None = no ceiling + orientation: Optional[str] = None # "landscape" | "portrait" | "square" + min_width: Optional[int] = None # resolution floor in pixels + per_page: int = 20 + page: int = 1 + + +@runtime_checkable +class StockSource(Protocol): + """Protocol every stock source adapter must satisfy. + + Attributes + ---------- + name: + Stable key used in the corpus (becomes the prefix of each + `ClipRecord.clip_id`) and in any agent-facing source list. Do + NOT change it after a corpus has been built against it — + existing rows would orphan. + + Methods + ------- + is_available: + Cheap, non-network check. Answers "does the environment have + the keys / dependencies this source needs?" The corpus builder + silently skips unavailable sources during fan-out. + + search: + Returns a flat list of `Candidate` objects in whatever order + the source's own relevance ranking gave them. An empty list is + legal (and common for niche queries). Network errors should be + raised — the corpus builder catches and logs per-source so one + flaky API doesn't poison the whole run. + + download: + Saves the candidate's file to `out_path` (creating parent + directories if needed) and returns the final path. This is a + blocking call; any caching is the corpus builder's + responsibility, not the adapter's. + """ + + name: str + + def is_available(self) -> bool: ... + + def search(self, query: str, filters: SearchFilters) -> list[Candidate]: ... + + def download(self, candidate: Candidate, out_path: Path) -> Path: ... diff --git a/tools/video/stock_sources/nasa.py b/tools/video/stock_sources/nasa.py new file mode 100644 index 0000000..282d91b --- /dev/null +++ b/tools/video/stock_sources/nasa.py @@ -0,0 +1,330 @@ +"""NASA Image and Video Library adapter. + +Wraps ``images-api.nasa.gov`` behind the `StockSource` protocol. NASA's +catalogue is the best free source of space, mission, Earth-observation, +and scientific-visualisation footage and stills. Everything on +images.nasa.gov is free to use (public domain / NASA-created), with the +caveat that third-party material occasionally sneaks in via partner +missions — we record that caveat in the licence field and trust the +user to vet before publishing. + +No API key required. An optional `NASA_API_KEY` increases rate limits +but is not part of this adapter's availability check — the default +`DEMO_KEY` path handles typical corpus-building loads. + +Fetch pattern +------------- +Like `archive_org`, this is a two-stage fetch. The search endpoint +returns items with an `href` pointing at a per-item asset manifest (a +plain JSON array of file URLs). We follow that manifest to pick the +best rendition and build a `Candidate` with a ready-to-use +`download_url`. One extra HTTP call per hit. + +What NASA is good for +--------------------- +- space missions, rockets, astronauts, +- Earth observation (weather, city lights, glaciers, hurricanes), +- planetary-science animation, +- historical aeronautics (NACA, shuttle era), +- any "tiny human on a big planet" interstitial. + +What it is *not* good for: most documentary subjects that live on +Earth at human scale. For a "walking home" montage this adapter will +contribute atmosphere shots at best (sunrise over Earth, city lights +from orbit) — the core material comes from Pexels and Archive.org. +""" +from __future__ import annotations + +import os +import re +from pathlib import Path +from typing import Any, Optional +from urllib.parse import quote, urlparse, urlunparse + +from .base import Candidate, SearchFilters + + +_SEARCH_URL = "https://images-api.nasa.gov/search" +_UNSAFE_ID_CHARS = re.compile(r"[^A-Za-z0-9._\-]+") + + +class NasaSource: + """Adapter for images.nasa.gov. + + Satisfies `StockSource`. Stateless, no required credentials. + """ + + name = "nasa" + + def is_available(self) -> bool: + # The public API is unauthenticated; as long as the network is + # reachable we're available. `NASA_API_KEY` is optional and + # only affects rate limiting. + return True + + # ------------------------------------------------------------------ + # Public protocol + # ------------------------------------------------------------------ + + def search(self, query: str, filters: SearchFilters) -> list[Candidate]: + """Search NASA's media library for images and/or videos. + + Routes by `filters.kind`: + "video" → `media_type=video` + "image" → `media_type=image` + "any" → both (NASA accepts repeated ``media_type`` params) + """ + import requests # lazy + + kind = (filters.kind or "video").lower() + media_types: list[str] = [] + if kind in ("video", "any"): + media_types.append("video") + if kind in ("image", "any"): + media_types.append("image") + if not media_types: + return [] + + params: list[tuple[str, str]] = [("q", query)] + for mt in media_types: + params.append(("media_type", mt)) + # NASA uses `page_size` (max 100) and `page` (1-indexed). + params.append(("page_size", str(max(1, min(filters.per_page, 100))))) + params.append(("page", str(max(1, filters.page)))) + + r = requests.get(_SEARCH_URL, params=params, timeout=30) + r.raise_for_status() + data = r.json() + items = ((data.get("collection") or {}).get("items") or []) + + out: list[Candidate] = [] + for item in items: + cand = self._hydrate_candidate(item, filters) + if cand is not None: + out.append(cand) + return out + + def download(self, candidate: Candidate, out_path: Path) -> Path: + """Stream the candidate's file to `out_path`. + + Same pattern as the other adapters. + """ + import requests # lazy + + if not candidate.download_url: + raise ValueError( + f"Candidate {candidate.clip_id} has no download_url" + ) + + out_path = Path(out_path) + out_path.parent.mkdir(parents=True, exist_ok=True) + + with requests.get( + candidate.download_url, stream=True, timeout=300 + ) as r: + r.raise_for_status() + with open(out_path, "wb") as f: + for chunk in r.iter_content(chunk_size=1 << 16): + if chunk: + f.write(chunk) + return out_path + + # ------------------------------------------------------------------ + # Internals + # ------------------------------------------------------------------ + + def _hydrate_candidate( + self, item: dict, filters: SearchFilters + ) -> Optional[Candidate]: + """Turn a search-result item into a Candidate with a download URL. + + Fetches the per-item asset manifest (``item['href']`` is a + plain JSON array of file URLs) and picks the best rendition. + Returns None if the manifest is empty or no playable file + exists. + """ + import requests # lazy + + data_list = item.get("data") or [] + if not data_list: + return None + meta = data_list[0] + + nasa_id = meta.get("nasa_id") + media_type = (meta.get("media_type") or "").lower() + if not nasa_id or media_type not in ("video", "image"): + return None + + asset_href = item.get("href") + if not asset_href: + return None + + try: + r = requests.get(asset_href, timeout=30) + r.raise_for_status() + file_urls = r.json() + except Exception: + return None + + if not isinstance(file_urls, list) or not file_urls: + return None + + if media_type == "video": + download_url = _pick_video_url(file_urls) + else: + download_url = _pick_image_url(file_urls) + if not download_url: + return None + # NASA asset URLs sometimes contain unencoded spaces and other + # characters (especially older items whose nasa_id is a title). + # URL-encode the path component so `requests.get` handles them. + download_url = _encode_url_path(download_url) + + # NASA doesn't give us dimensions or duration on the search + # result. Width/height default to 0 and the corpus builder is + # expected to probe post-download. Duration is 0 for both + # images and (unfortunately) videos. + width = 0 + height = 0 + duration = 0.0 + + # Filters that need dims/duration can't apply here. Skipping + # them is the honest thing — "unknown" should not be rejected. + + title = (meta.get("title") or "").strip() + description = (meta.get("description") or "").strip() + keywords = meta.get("keywords") or [] + if isinstance(keywords, list): + kw_text = " ".join(str(k) for k in keywords if k) + else: + kw_text = str(keywords) + source_tags = " ".join( + s for s in (title, description, kw_text) if s + ).strip() + if len(source_tags) > 500: + source_tags = source_tags[:500] + + # Thumbnail: NASA exposes a "preview" link in item["links"] + thumbnail_url = "" + for link in item.get("links") or []: + if (link or {}).get("rel") == "preview": + thumbnail_url = link.get("href", "") or "" + break + + creator = (meta.get("photographer") or meta.get("center") or "").strip() + + safe_id = _sanitize_source_id(nasa_id) + return Candidate( + source=self.name, + source_id=safe_id, + source_url=f"https://images.nasa.gov/details/{quote(nasa_id, safe='')}", + download_url=download_url, + kind=media_type, + width=width, + height=height, + duration=duration, + creator=creator, + license="NASA Media Usage Guidelines (public domain with caveats)", + source_tags=source_tags, + thumbnail_url=thumbnail_url, + extra={ + "center": meta.get("center"), + "date_created": meta.get("date_created"), + "secondary_creator": meta.get("secondary_creator"), + }, + ) + + +# ---------------------------------------------------------------------- +# Module-level helpers +# ---------------------------------------------------------------------- + + +def _pick_video_url(file_urls: list[str]) -> str: + """Pick the best video rendition from a NASA asset manifest. + + NASA file URLs follow a ``~.`` naming convention + where tag is one of: ``orig``, ``large``, ``medium``, ``small``, + ``preview``, ``thumb``. Preference order (best quality first): + orig → large → medium → small. We skip previews and thumbnails. + """ + priority = ("orig", "large", "medium", "small") + buckets: dict[str, list[str]] = {p: [] for p in priority} + for url in file_urls: + lower = url.lower() + if not lower.endswith((".mp4", ".mov", ".m4v")): + continue + for p in priority: + if f"~{p}." in lower: + buckets[p].append(url) + break + for p in priority: + if buckets[p]: + return buckets[p][0] + # Fallback: any mp4 at all + for url in file_urls: + if url.lower().endswith(".mp4"): + return url + return "" + + +def _pick_image_url(file_urls: list[str]) -> str: + """Pick the best image rendition from a NASA asset manifest. + + Same pattern as videos but for .jpg/.png/.tif. Prefer ``orig``, + fall back to ``large``. We skip ``thumb`` and ``small`` because + they're too low-res for CLIP embedding quality. + """ + priority = ("orig", "large", "medium") + buckets: dict[str, list[str]] = {p: [] for p in priority} + for url in file_urls: + lower = url.lower() + if not lower.endswith((".jpg", ".jpeg", ".png", ".tif", ".tiff")): + continue + for p in priority: + if f"~{p}." in lower: + buckets[p].append(url) + break + for p in priority: + if buckets[p]: + return buckets[p][0] + # Fallback: any jpg/png + for url in file_urls: + if url.lower().endswith((".jpg", ".jpeg", ".png")): + return url + return "" + + +def _sanitize_source_id(raw: str) -> str: + """Collapse arbitrary NASA nasa_ids into a filesystem-safe token. + + Some older items use their title as the nasa_id, which means the + raw string contains spaces, apostrophes, en-dashes, and other + characters that break file paths and command-line tooling. We + replace anything outside ``[A-Za-z0-9._-]`` with an underscore, + collapse runs, and strip edges. Length is capped at 120 chars so + the resulting clip_id stays well under filesystem limits. + """ + if not raw: + return "unknown" + cleaned = _UNSAFE_ID_CHARS.sub("_", raw.strip()) + cleaned = re.sub(r"_+", "_", cleaned).strip("_.") + if not cleaned: + return "unknown" + return cleaned[:120] + + +def _encode_url_path(url: str) -> str: + """URL-encode the path component of a URL, leaving the host alone. + + NASA occasionally returns asset URLs whose path literally contains + spaces and unicode — those will 400 if passed unquoted to + ``requests.get``. We use ``urllib.parse`` to surgically re-encode + only the path segment without double-encoding the scheme/host. + """ + try: + parts = urlparse(url) + except Exception: + return url + safe_path = quote(parts.path, safe="/") + return urlunparse(parts._replace(path=safe_path)) diff --git a/tools/video/stock_sources/pexels.py b/tools/video/stock_sources/pexels.py new file mode 100644 index 0000000..7386a15 --- /dev/null +++ b/tools/video/stock_sources/pexels.py @@ -0,0 +1,289 @@ +"""Pexels stock media source adapter. + +Wraps the Pexels video and image search APIs behind the unified +`StockSource` protocol. Pexels is the workhorse for the +documentary-montage pipeline: large catalogue, fast API, free, no +attribution required, and stable URLs for cacheable downloads. + +Pexels exposes videos and images on two separate endpoints +(``/videos/search`` and ``/v1/search``), so this adapter fans out +internally and normalises both into the same `Candidate` shape. The +corpus builder never branches on kind. + +Uses `PEXELS_API_KEY` from the environment. `.env` is loaded at process +start by `tools.base_tool._load_dotenv`. +""" +from __future__ import annotations + +import os +from pathlib import Path +from typing import Any, Optional + +from .base import Candidate, SearchFilters + + +_VIDEO_SEARCH_URL = "https://api.pexels.com/videos/search" +_IMAGE_SEARCH_URL = "https://api.pexels.com/v1/search" +_PEXELS_LICENSE = "Pexels License (free, no attribution required)" + + +class PexelsSource: + """Unified Pexels adapter for videos and images. + + Satisfies `StockSource`. Stateless — all config comes from the + environment, so the corpus builder can instantiate one per run + without caching the instance. + """ + + name = "pexels" + + def is_available(self) -> bool: + return bool(os.environ.get("PEXELS_API_KEY")) + + # ------------------------------------------------------------------ + # Public protocol + # ------------------------------------------------------------------ + + def search(self, query: str, filters: SearchFilters) -> list[Candidate]: + """Search Pexels, routing to video/image endpoints by `filters.kind`. + + `kind="video"` hits only the videos endpoint, `kind="image"` + only the images endpoint, `kind="any"` queries both and + concatenates the results (videos first). The caller decides + ordering semantics downstream — this adapter does not re-rank. + """ + kind = (filters.kind or "video").lower() + out: list[Candidate] = [] + if kind in ("video", "any"): + out.extend(self._search_videos(query, filters)) + if kind in ("image", "any"): + out.extend(self._search_images(query, filters)) + return out + + def download(self, candidate: Candidate, out_path: Path) -> Path: + """Stream the candidate's file to `out_path`. + + Creates parent directories as needed. Uses streamed chunks so + large 1080p clips don't blow RAM. No caching — the corpus + builder is responsible for deciding whether to call this at + all. + """ + import requests # lazy — avoid pulling requests at import time + + if not candidate.download_url: + raise ValueError( + f"Candidate {candidate.clip_id} has no download_url" + ) + + out_path = Path(out_path) + out_path.parent.mkdir(parents=True, exist_ok=True) + + with requests.get( + candidate.download_url, stream=True, timeout=120 + ) as r: + r.raise_for_status() + with open(out_path, "wb") as f: + for chunk in r.iter_content(chunk_size=1 << 16): + if chunk: + f.write(chunk) + return out_path + + # ------------------------------------------------------------------ + # Internals + # ------------------------------------------------------------------ + + def _headers(self) -> dict[str, str]: + key = os.environ.get("PEXELS_API_KEY") + if not key: + raise RuntimeError( + "PEXELS_API_KEY not set. Get a free key at " + "https://www.pexels.com/api/ and add it to .env." + ) + return {"Authorization": key} + + def _search_videos( + self, query: str, filters: SearchFilters + ) -> list[Candidate]: + import requests # lazy + + params: dict[str, Any] = { + "query": query, + "per_page": max(1, min(filters.per_page, 80)), + "page": max(1, filters.page), + } + if filters.orientation: + params["orientation"] = filters.orientation + + r = requests.get( + _VIDEO_SEARCH_URL, + headers=self._headers(), + params=params, + timeout=30, + ) + r.raise_for_status() + data = r.json() + videos = data.get("videos", []) or [] + + out: list[Candidate] = [] + for v in videos: + # Duration filter. Pexels doesn't expose this server-side, + # so we filter client-side and accept that per_page may be + # partially consumed by clips we throw away. + duration = float(v.get("duration", 0) or 0) + if filters.min_duration is not None and duration < filters.min_duration: + continue + if filters.max_duration is not None and duration > filters.max_duration: + continue + + rend = _pick_video_rendition( + v.get("video_files", []) or [], + min_width=filters.min_width or 0, + max_width=1920, + ) + if rend is None: + continue + + user = v.get("user") or {} + tag_text = _slug_tags_from_url(v.get("url", "") or "") + + out.append( + Candidate( + source=self.name, + source_id=str(v.get("id")), + source_url=v.get("url", "") or "", + download_url=rend.get("link", "") or "", + kind="video", + width=int(rend.get("width") or v.get("width") or 0), + height=int(rend.get("height") or v.get("height") or 0), + duration=duration, + creator=user.get("name", "") or "", + license=_PEXELS_LICENSE, + source_tags=tag_text, + thumbnail_url=v.get("image", "") or "", + extra={ + "fps": rend.get("fps"), + "rendition_quality": rend.get("quality"), + "user_url": user.get("url"), + }, + ) + ) + return out + + def _search_images( + self, query: str, filters: SearchFilters + ) -> list[Candidate]: + import requests # lazy + + params: dict[str, Any] = { + "query": query, + "per_page": max(1, min(filters.per_page, 80)), + "page": max(1, filters.page), + } + if filters.orientation: + params["orientation"] = filters.orientation + + r = requests.get( + _IMAGE_SEARCH_URL, + headers=self._headers(), + params=params, + timeout=30, + ) + r.raise_for_status() + data = r.json() + photos = data.get("photos", []) or [] + + out: list[Candidate] = [] + for p in photos: + width = int(p.get("width", 0) or 0) + height = int(p.get("height", 0) or 0) + if filters.min_width is not None and width < filters.min_width: + continue + + src = p.get("src") or {} + # "large2x" is the sweet spot between detail and filesize + # for montage use — usually 1.5-2 MP. Fall back to + # "original" if the CDN gave us something weird. + download_url = src.get("large2x") or src.get("original") or "" + if not download_url: + continue + + alt = (p.get("alt") or "").strip() + + out.append( + Candidate( + source=self.name, + source_id=str(p.get("id")), + source_url=p.get("url", "") or "", + download_url=download_url, + kind="image", + width=width, + height=height, + duration=0.0, + creator=p.get("photographer", "") or "", + license=_PEXELS_LICENSE, + source_tags=alt, + thumbnail_url=src.get("medium", "") or "", + extra={ + "photographer_url": p.get("photographer_url"), + "avg_color": p.get("avg_color"), + }, + ) + ) + return out + + +# ---------------------------------------------------------------------- +# Module-level helpers (also used by tests) +# ---------------------------------------------------------------------- + + +def _pick_video_rendition( + video_files: list[dict], + min_width: int = 0, + max_width: int = 1920, +) -> Optional[dict]: + """Pick the largest mp4/mov rendition within [min_width, max_width]. + + Pexels returns HLS, SD, HD, and sometimes UHD renditions per video. + We want the biggest file that fits our cap — "biggest" as a quality + proxy, "cap" so we don't waste bandwidth downloading 4K when we'll + scale to 720p anyway. + """ + candidates = [ + f for f in video_files + if (f.get("file_type", "") or "").startswith("video/") + and min_width <= int(f.get("width") or 0) <= max_width + and f.get("link") + ] + if not candidates: + return None + candidates.sort(key=lambda f: int(f.get("width") or 0), reverse=True) + return candidates[0] + + +def _slug_tags_from_url(url: str) -> str: + """Extract a readable tag string from a Pexels video landing URL. + + Pexels video JSON does not expose tags or descriptions reliably, + but the landing URL is a slug like:: + + https://www.pexels.com/video/aerial-view-of-city-at-night-3571264/ + + The slug is the closest thing to keywords the API gives us, and it + matches what the uploader described the clip as. We strip the + trailing numeric id and rejoin on spaces so the CLIP text encoder + can work with it. + + Returns an empty string if the URL cannot be parsed — the caller + should treat that as "no tag signal for this clip". + """ + if not url: + return "" + tail = url.rstrip("/").rsplit("/", 1) + if len(tail) != 2: + return "" + slug = tail[1] + parts = slug.rsplit("-", 1) + if len(parts) == 2 and parts[1].isdigit(): + slug = parts[0] + return slug.replace("-", " ").strip() diff --git a/tools/video/video_compose.py b/tools/video/video_compose.py index b886a34..7aa9100 100644 --- a/tools/video/video_compose.py +++ b/tools/video/video_compose.py @@ -247,6 +247,32 @@ class VideoCompose(BaseTool): """Check if a file is a still image (routes to Remotion, not FFmpeg).""" return path.suffix.lower() in VideoCompose._IMAGE_EXTENSIONS + @staticmethod + def _has_audio_stream(path: Path) -> bool: + """Return True iff ffprobe reports at least one audio stream. + + Many stock video clips (especially from Pexels) ship with no audio + stream at all. If we blindly tell ffmpeg to transcode the 0:a stream + on such a file it errors out. This helper lets the segment builder + branch on stream presence so it can synthesize a silent track when + needed, keeping the concat segment layout consistent. + """ + try: + out = subprocess.check_output( + [ + "ffprobe", "-v", "error", + "-select_streams", "a", + "-show_entries", "stream=codec_type", + "-of", "default=nw=1:nk=1", + str(path), + ], + stderr=subprocess.STDOUT, + text=True, + ) + return "audio" in out + except Exception: + return False + def _compose(self, inputs: dict[str, Any]) -> ToolResult: """FFmpeg composition: concat video cuts, add audio, burn subtitles. @@ -326,23 +352,98 @@ class VideoCompose(BaseTool): ) else: # Video source: trim to segment. - # -ss before -i for input-level seeking — more reliable - # with -c copy when stream order is non-standard (e.g. - # audio-first containers from Kling/fal.ai). + # + # Semantics: + # -ss BEFORE -i → fast input-level seek to in_s + # -t AFTER -i → "play for `duration` seconds" + # (unambiguous regardless of seek mode) + # + # We MUST re-encode here — `-c copy` cannot do frame-accurate + # cuts because it snaps to keyframes. With sparse GOPs (common + # in Pexels / AI-generated clips), stream-copy can produce + # segments significantly longer than `duration`, breaking the + # target timeline. Re-encoding with libx264/AAC is slower but + # gives exact cut boundaries. Same resolution in → same + # resolution out, so same-res inputs concat cleanly. cmd = [ "ffmpeg", "-y", "-ss", str(in_s), + "-t", str(duration), "-i", str(source), - "-to", str(out_s - in_s), ] + # Normalize every segment to a consistent container so the + # concat-copy step is always safe. The concat demuxer with + # `-c copy` requires identical codec / resolution / fps / + # pix_fmt / sar across ALL segments — otherwise it throws + # "Non-monotonous DTS" or silently produces corrupt output. + # + # Default target is 1920x1080 @ 30fps, yuv420p, sar=1. If the + # source is smaller it letterboxes; if larger it downscales. + # Callers can override via edit_decisions.metadata.compose_target + # (future extension) but the defaults match the most common + # delivery profile (YouTube landscape). + vf_parts: list[str] = [ + "scale=1920:1080:force_original_aspect_ratio=decrease", + "pad=1920:1080:(ow-iw)/2:(oh-ih)/2:color=black", + "setsar=1", + "fps=30", + ] + af_parts: list[str] = [] if speed != 1.0: - vf = f"setpts={1.0/speed}*PTS" - af = self._build_atempo(speed) - cmd.extend(["-filter:v", vf, "-filter:a", af]) - cmd.extend(["-c:v", codec, "-crf", str(crf), "-c:a", "aac"]) + vf_parts.append(f"setpts={1.0/speed}*PTS") + af_parts.append(self._build_atempo(speed)) + + cmd.extend(["-filter:v", ",".join(vf_parts)]) + if af_parts: + cmd.extend(["-filter:a", ",".join(af_parts)]) + + cmd.extend([ + "-c:v", codec, + "-crf", str(crf), + "-preset", preset, + "-pix_fmt", "yuv420p", + "-r", "30", + ]) + + # Audio handling: some source clips have no audio stream + # (Pexels stock often ships silent). If we unconditionally + # ask ffmpeg to copy/encode the 0:a stream it errors out. + # Probe for an audio stream first — if present, transcode + # to AAC; if absent, synthesize a silent stereo track so + # concat segments have a consistent stream layout. + has_audio = self._has_audio_stream(source) + if has_audio: + cmd.extend(["-c:a", "aac", "-b:a", "192k", "-ar", "48000", "-ac", "2"]) else: - cmd.extend(["-c", "copy"]) + # Inject silent audio via lavfi before the output. + # We have to rebuild cmd to add the lavfi input + # before the output path and map streams explicitly. + cmd = [ + "ffmpeg", "-y", + "-ss", str(in_s), + "-t", str(duration), + "-i", str(source), + "-f", "lavfi", + "-t", str(duration), + "-i", "anullsrc=channel_layout=stereo:sample_rate=48000", + "-filter:v", ",".join(vf_parts), + ] + if af_parts: + cmd.extend(["-filter:a", ",".join(af_parts)]) + cmd.extend([ + "-map", "0:v:0", + "-map", "1:a:0", + "-c:v", codec, + "-crf", str(crf), + "-preset", preset, + "-pix_fmt", "yuv420p", + "-r", "30", + "-c:a", "aac", + "-b:a", "192k", + "-ar", "48000", + "-ac", "2", + ]) cmd.append(str(seg_path)) self.run_command(cmd)