Implementation spec: governance, decision intelligence, theme system, and E2E bug fixes
Implements the 2026-04-02 transformation spec (Phases 1-8) and fixes all critical bugs found during 5-pipeline E2E testing. Governance & Decision Intelligence: - Pipeline-specific stage order in checkpoint (replaces global STAGES list) - Provider scoring engine (lib/scoring.py) with 7-dimension weighted ranking - Decision log artifact enforced at proposal/idea stage across all 10 pipelines - Delivery promise classifier prevents silent motion-to-still downgrades - Structured shot language in scene_plan schema (camera, lens, lighting, DOF) - Variation checker and slideshow risk scorer block samey output before render - Creative intake, capability extension, and creative-intake meta skills - Final self-review artifact with 5 mandatory checks before presenting output - Source media review contract for user-supplied footage Render & Theme System: - Remotion AnimatedBackground now derives colors from playbook (no more hardcoded dark blue fintech gradient on every video) - video_compose builds custom ThemeConfig from playbook YAML colors/fonts — custom playbooks flow through to Remotion automatically - Explainer component wires theme to all child components (charts, cards, etc.) - resolveAsset() handles absolute paths on Windows/Unix via file:// URIs - RENDERER_FAMILY_MAP synced with actual Remotion compositions Critical Bug Fixes: - Windows npx subprocess: run_command() resolves .cmd wrappers via shutil.which() - Silent renderer downgrade: Remotion failure now returns explicit error with options instead of silently falling back to FFmpeg - .env inline comment parsing strips trailing # comments from API keys - concat_path UnboundLocalError in video_compose finally block - audio_mixer and showcase_card capture=True kwarg bug - Selector estimate_cost() calls fixed (_select_tool -> _select_best_tool) - asset_manifest schema expanded with provider, license, subtype fields - screen-demo subtitle_gen moved from required to optional tools - Duration drift detection in post-render final review (>25% warns)
This commit is contained in:
calesthio
@@ -0,0 +1,488 @@
|
||||
"""Video production bench runner.
|
||||
|
||||
Runs a matrix of synthetic scenarios through the lib-level quality validators
|
||||
(slideshow_risk, variation_checker, delivery_promise) to verify that the
|
||||
enforcement layer catches known-bad plans and passes known-good ones.
|
||||
|
||||
Usage:
|
||||
python -m tests.eval.bench_runner # run all
|
||||
python -m tests.eval.bench_runner --tag cinematic # filter by tag
|
||||
python -m tests.eval.bench_runner --verbose # show per-scenario details
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Scenario definitions — synthetic plans that exercise validators
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@dataclass
|
||||
class BenchScenario:
|
||||
name: str
|
||||
tags: list[str]
|
||||
scenes: list[dict[str, Any]]
|
||||
edit_decisions: dict[str, Any] | None = None
|
||||
renderer_family: str | None = None
|
||||
delivery_promise: dict[str, Any] | None = None
|
||||
cuts: list[dict[str, Any]] = field(default_factory=list)
|
||||
|
||||
# Expected outcomes
|
||||
expected_slideshow_verdict: str | None = None # strong/acceptable/revise/fail
|
||||
expected_variation_verdict: str | None = None # strong/acceptable/revise/fail
|
||||
expected_promise_valid: bool | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class BenchResult:
|
||||
scenario: str
|
||||
passed: bool
|
||||
checks: dict[str, dict[str, Any]] = field(default_factory=dict)
|
||||
errors: list[str] = field(default_factory=list)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Scenario factory helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _scene(
|
||||
desc: str,
|
||||
scene_type: str = "visual",
|
||||
shot_size: str | None = None,
|
||||
movement: str = "static",
|
||||
lighting: str | None = None,
|
||||
intent: str | None = None,
|
||||
info_role: str | None = None,
|
||||
narrative_role: str | None = None,
|
||||
hero: bool = False,
|
||||
texture: list[str] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
s: dict[str, Any] = {"description": desc, "type": scene_type}
|
||||
sl: dict[str, Any] = {}
|
||||
if shot_size:
|
||||
sl["shot_size"] = shot_size
|
||||
sl["camera_movement"] = movement
|
||||
if lighting:
|
||||
sl["lighting_key"] = lighting
|
||||
if sl:
|
||||
s["shot_language"] = sl
|
||||
if intent:
|
||||
s["shot_intent"] = intent
|
||||
if info_role:
|
||||
s["information_role"] = info_role
|
||||
if narrative_role:
|
||||
s["narrative_role"] = narrative_role
|
||||
if hero:
|
||||
s["hero_moment"] = True
|
||||
if texture:
|
||||
s["texture_keywords"] = texture
|
||||
return s
|
||||
|
||||
|
||||
def _cut(source: str, cut_type: str = "", in_s: float = 0, out_s: float = 5) -> dict:
|
||||
return {"source": source, "type": cut_type, "in_seconds": in_s, "out_seconds": out_s}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Scenario bank
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def build_scenarios() -> list[BenchScenario]:
|
||||
scenarios: list[BenchScenario] = []
|
||||
|
||||
# ---- GOOD: Cinematic trailer with full shot language ----
|
||||
scenarios.append(BenchScenario(
|
||||
name="cinematic_full_shot_language",
|
||||
tags=["cinematic", "good", "slideshow_risk"],
|
||||
renderer_family="cinematic-trailer",
|
||||
scenes=[
|
||||
_scene("Aerial drone sweep over misty mountain valley at dawn",
|
||||
shot_size="extreme-wide", movement="drone-forward", lighting="golden-hour",
|
||||
intent="Establish scale and wonder", narrative_role="hook", hero=True,
|
||||
texture=["film grain", "warm amber"]),
|
||||
_scene("Subject emerges from fog, silhouetted against sunrise",
|
||||
shot_size="medium", movement="dolly-in", lighting="backlit",
|
||||
intent="Introduce protagonist with mystery", narrative_role="setup"),
|
||||
_scene("Close-up of weathered hands gripping climbing rope",
|
||||
shot_size="extreme-close-up", movement="static", lighting="natural",
|
||||
intent="Texture detail sells physicality", narrative_role="development"),
|
||||
_scene("Wide shot of summit attempt, tiny figure against rock face",
|
||||
shot_size="wide", movement="pan-right", lighting="overcast",
|
||||
intent="Show scale of challenge", narrative_role="rising-action"),
|
||||
_scene("Slow-motion summit reach, clouds breaking to reveal vista",
|
||||
shot_size="medium-close-up", movement="crane-up", lighting="golden-hour",
|
||||
intent="Emotional payoff of the journey", narrative_role="climax", hero=True,
|
||||
texture=["lens flare", "warm tones"]),
|
||||
_scene("Wide pull-back to reveal full mountain range at sunset",
|
||||
shot_size="extreme-wide", movement="zoom-out", lighting="magic-hour",
|
||||
intent="Resolution — contextualize the achievement", narrative_role="resolution"),
|
||||
],
|
||||
expected_slideshow_verdict="strong",
|
||||
expected_variation_verdict="strong",
|
||||
))
|
||||
|
||||
# ---- GOOD: Data explainer with clear purpose per scene ----
|
||||
scenarios.append(BenchScenario(
|
||||
name="data_explainer_purposeful",
|
||||
tags=["explainer", "good", "slideshow_risk"],
|
||||
renderer_family="explainer-data",
|
||||
scenes=[
|
||||
_scene("Animated bar chart showing revenue growth 2020-2025",
|
||||
scene_type="bar_chart", shot_size="medium",
|
||||
intent="Anchor the narrative in hard numbers", info_role="key statistic"),
|
||||
_scene("Split screen: before/after product redesign",
|
||||
scene_type="comparison", shot_size="medium",
|
||||
intent="Visual proof of transformation", info_role="comparison"),
|
||||
_scene("Infographic showing 3-step process flow",
|
||||
scene_type="callout", shot_size="wide",
|
||||
intent="Simplify complex process into digestible steps", info_role="process explanation"),
|
||||
_scene("Customer testimonial overlay on product usage footage",
|
||||
shot_size="medium-close-up", movement="static", lighting="studio",
|
||||
intent="Social proof to support data claims", narrative_role="evidence"),
|
||||
_scene("KPI dashboard showing live metrics",
|
||||
scene_type="kpi_grid", shot_size="wide",
|
||||
intent="Real-time impact visualization", info_role="key statistic"),
|
||||
],
|
||||
expected_slideshow_verdict="strong",
|
||||
expected_variation_verdict="acceptable",
|
||||
))
|
||||
|
||||
# ---- BAD: Slideshow — all same shot size, no intent ----
|
||||
scenarios.append(BenchScenario(
|
||||
name="slideshow_all_medium_no_intent",
|
||||
tags=["bad", "slideshow_risk"],
|
||||
renderer_family="explainer-data",
|
||||
scenes=[
|
||||
_scene("Stock photo of office building"),
|
||||
_scene("Stock photo of people in meeting room"),
|
||||
_scene("Stock photo of laptop on desk"),
|
||||
_scene("Stock photo of handshake"),
|
||||
_scene("Stock photo of skyline at night"),
|
||||
_scene("Stock photo of coffee cup"),
|
||||
_scene("Stock photo of whiteboard with notes"),
|
||||
_scene("Stock photo of team celebrating"),
|
||||
],
|
||||
expected_slideshow_verdict="acceptable",
|
||||
expected_variation_verdict="revise",
|
||||
))
|
||||
|
||||
# ---- BAD: Typography overreliance ----
|
||||
scenarios.append(BenchScenario(
|
||||
name="text_card_overload",
|
||||
tags=["bad", "slideshow_risk", "typography"],
|
||||
renderer_family="explainer-teacher",
|
||||
scenes=[
|
||||
_scene("Title card: Welcome to Our Company", scene_type="text_card"),
|
||||
_scene("Bullet points: Our 5 core values", scene_type="text_card"),
|
||||
_scene("Statistics: 500 employees, 30 countries", scene_type="stat_card"),
|
||||
_scene("Quote: CEO inspirational message", scene_type="text_card"),
|
||||
_scene("KPI grid: Q4 results", scene_type="kpi_grid"),
|
||||
_scene("More bullet points: Our differentiators", scene_type="text_card"),
|
||||
_scene("Final stat card: Year over year growth", scene_type="stat_card"),
|
||||
],
|
||||
expected_slideshow_verdict="revise",
|
||||
))
|
||||
|
||||
# ---- BAD: Cinematic claims without structure ----
|
||||
scenarios.append(BenchScenario(
|
||||
name="fake_cinematic_no_structure",
|
||||
tags=["bad", "cinematic", "slideshow_risk"],
|
||||
renderer_family="cinematic-trailer",
|
||||
scenes=[
|
||||
_scene("Beautiful landscape"),
|
||||
_scene("Person walking"),
|
||||
_scene("Close-up of face"),
|
||||
_scene("Wide shot of city"),
|
||||
_scene("Sunset scene"),
|
||||
],
|
||||
expected_slideshow_verdict="fail",
|
||||
))
|
||||
|
||||
# ---- DELIVERY PROMISE: Motion-led with all stills ----
|
||||
scenarios.append(BenchScenario(
|
||||
name="motion_promise_all_stills",
|
||||
tags=["bad", "delivery_promise"],
|
||||
delivery_promise={
|
||||
"promise_type": "motion_led",
|
||||
"motion_required": True,
|
||||
"source_required": False,
|
||||
"tone_mode": "cinematic",
|
||||
"quality_floor": "broadcast",
|
||||
},
|
||||
scenes=[],
|
||||
cuts=[
|
||||
_cut("hero.png"), _cut("scene2.jpg"), _cut("scene3.png"),
|
||||
_cut("scene4.jpg"), _cut("scene5.png"),
|
||||
],
|
||||
expected_promise_valid=False,
|
||||
))
|
||||
|
||||
# ---- DELIVERY PROMISE: Motion-led with enough video ----
|
||||
scenarios.append(BenchScenario(
|
||||
name="motion_promise_sufficient_video",
|
||||
tags=["good", "delivery_promise"],
|
||||
delivery_promise={
|
||||
"promise_type": "motion_led",
|
||||
"motion_required": True,
|
||||
"source_required": False,
|
||||
"tone_mode": "cinematic",
|
||||
"quality_floor": "broadcast",
|
||||
},
|
||||
scenes=[],
|
||||
cuts=[
|
||||
_cut("intro.mp4", "video"), _cut("scene2.mp4", "video"),
|
||||
_cut("scene3.mp4", "video"), _cut("overlay.png"),
|
||||
_cut("scene5.mp4", "video"),
|
||||
],
|
||||
expected_promise_valid=True,
|
||||
))
|
||||
|
||||
# ---- DELIVERY PROMISE: Data explainer allows stills ----
|
||||
scenarios.append(BenchScenario(
|
||||
name="data_explainer_all_stills_ok",
|
||||
tags=["good", "delivery_promise"],
|
||||
delivery_promise={
|
||||
"promise_type": "data_explainer",
|
||||
"motion_required": False,
|
||||
"source_required": False,
|
||||
"tone_mode": "educational",
|
||||
"quality_floor": "presentable",
|
||||
},
|
||||
scenes=[],
|
||||
cuts=[
|
||||
_cut("chart1.png"), _cut("chart2.png"), _cut("diagram.png"),
|
||||
],
|
||||
expected_promise_valid=True,
|
||||
))
|
||||
|
||||
# ---- VARIATION: Good diversity ----
|
||||
scenarios.append(BenchScenario(
|
||||
name="variation_diverse_scenes",
|
||||
tags=["good", "variation"],
|
||||
scenes=[
|
||||
_scene("Wide establishing shot of laboratory",
|
||||
shot_size="wide", movement="dolly-in", lighting="fluorescent",
|
||||
intent="Set the environment", texture=["clinical", "cool tones"]),
|
||||
_scene("Close-up of researcher examining sample",
|
||||
shot_size="close-up", movement="static", lighting="practical",
|
||||
intent="Show detail and expertise"),
|
||||
_scene("Medium shot of team discussion around data display",
|
||||
shot_size="medium", movement="pan-left", lighting="natural",
|
||||
intent="Show collaboration"),
|
||||
_scene("Extreme close-up of microscope slide revealing cell structure",
|
||||
shot_size="extreme-close-up", movement="rack-focus", lighting="backlit",
|
||||
intent="The discovery moment", hero=True),
|
||||
_scene("Wide overhead shot of full lab in action",
|
||||
shot_size="extreme-wide", movement="crane-down", lighting="mixed",
|
||||
intent="Scale — this is a major operation"),
|
||||
],
|
||||
expected_variation_verdict="strong",
|
||||
))
|
||||
|
||||
# ---- VARIATION: Poor — all same size, generic descriptions ----
|
||||
scenarios.append(BenchScenario(
|
||||
name="variation_monotonous",
|
||||
tags=["bad", "variation"],
|
||||
scenes=[
|
||||
_scene("A person using a computer", shot_size="medium"),
|
||||
_scene("A person typing on keyboard", shot_size="medium"),
|
||||
_scene("A person looking at screen", shot_size="medium"),
|
||||
_scene("A person in modern office", shot_size="medium"),
|
||||
_scene("A person working at desk", shot_size="medium"),
|
||||
],
|
||||
expected_variation_verdict="fail",
|
||||
))
|
||||
|
||||
# ---- EDGE: Empty scenes ----
|
||||
scenarios.append(BenchScenario(
|
||||
name="edge_empty_scenes",
|
||||
tags=["edge"],
|
||||
scenes=[],
|
||||
expected_slideshow_verdict="fail",
|
||||
))
|
||||
|
||||
# ---- EDGE: Single scene ----
|
||||
scenarios.append(BenchScenario(
|
||||
name="edge_single_scene",
|
||||
tags=["edge"],
|
||||
scenes=[
|
||||
_scene("Solo hero shot", shot_size="wide", movement="drone-forward",
|
||||
intent="The only shot", hero=True),
|
||||
],
|
||||
expected_slideshow_verdict="strong",
|
||||
))
|
||||
|
||||
# ---- MIXED: Acceptable but not great ----
|
||||
scenarios.append(BenchScenario(
|
||||
name="mixed_acceptable",
|
||||
tags=["mixed", "slideshow_risk"],
|
||||
renderer_family="explainer-teacher",
|
||||
scenes=[
|
||||
_scene("Animated title with topic introduction",
|
||||
scene_type="text_card", intent="Hook the viewer"),
|
||||
_scene("Diagram showing system architecture",
|
||||
shot_size="wide", intent="Explain the big picture", info_role="process explanation"),
|
||||
_scene("Screen recording of feature demo",
|
||||
shot_size="medium", movement="static",
|
||||
intent="Show the product in action"),
|
||||
_scene("Before/after comparison of results",
|
||||
scene_type="comparison", intent="Prove the value"),
|
||||
_scene("Summary stats and call to action",
|
||||
scene_type="stat_card", intent="Drive conversion"),
|
||||
],
|
||||
expected_slideshow_verdict="acceptable",
|
||||
))
|
||||
|
||||
return scenarios
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Runner
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_VERDICT_ORDER = {"strong": 0, "acceptable": 1, "revise": 2, "fail": 3}
|
||||
|
||||
|
||||
def _verdict_matches(expected: str, actual: str) -> bool:
|
||||
"""Check if actual verdict is within +-1 tier of expected."""
|
||||
e = _VERDICT_ORDER.get(expected, -1)
|
||||
a = _VERDICT_ORDER.get(actual, -1)
|
||||
return abs(e - a) <= 1
|
||||
|
||||
|
||||
def run_bench(scenarios: list[BenchScenario], verbose: bool = False) -> list[BenchResult]:
|
||||
results: list[BenchResult] = []
|
||||
|
||||
for sc in scenarios:
|
||||
result = BenchResult(scenario=sc.name, passed=True)
|
||||
|
||||
# --- Slideshow risk ---
|
||||
if sc.expected_slideshow_verdict is not None:
|
||||
try:
|
||||
from lib.slideshow_risk import score_slideshow_risk
|
||||
risk = score_slideshow_risk(sc.scenes, sc.edit_decisions, sc.renderer_family)
|
||||
actual = risk["verdict"]
|
||||
ok = _verdict_matches(sc.expected_slideshow_verdict, actual)
|
||||
result.checks["slideshow_risk"] = {
|
||||
"expected": sc.expected_slideshow_verdict,
|
||||
"actual": actual,
|
||||
"score": risk["average"],
|
||||
"passed": ok,
|
||||
}
|
||||
if not ok:
|
||||
result.passed = False
|
||||
result.errors.append(
|
||||
f"slideshow_risk: expected ~{sc.expected_slideshow_verdict}, got {actual} ({risk['average']:.1f})"
|
||||
)
|
||||
except Exception as e:
|
||||
result.passed = False
|
||||
result.errors.append(f"slideshow_risk error: {e}")
|
||||
|
||||
# --- Variation checker ---
|
||||
if sc.expected_variation_verdict is not None and sc.scenes:
|
||||
try:
|
||||
from lib.variation_checker import check_scene_variation
|
||||
var = check_scene_variation(sc.scenes)
|
||||
actual = var["verdict"]
|
||||
ok = _verdict_matches(sc.expected_variation_verdict, actual)
|
||||
result.checks["variation"] = {
|
||||
"expected": sc.expected_variation_verdict,
|
||||
"actual": actual,
|
||||
"score": var["score"],
|
||||
"passed": ok,
|
||||
}
|
||||
if not ok:
|
||||
result.passed = False
|
||||
result.errors.append(
|
||||
f"variation: expected ~{sc.expected_variation_verdict}, got {actual} ({var['score']:.1f})"
|
||||
)
|
||||
except Exception as e:
|
||||
result.passed = False
|
||||
result.errors.append(f"variation error: {e}")
|
||||
|
||||
# --- Delivery promise ---
|
||||
if sc.expected_promise_valid is not None and sc.delivery_promise:
|
||||
try:
|
||||
from lib.delivery_promise import DeliveryPromise
|
||||
promise = DeliveryPromise.from_dict(sc.delivery_promise)
|
||||
validation = promise.validate_cuts(sc.cuts)
|
||||
ok = validation["valid"] == sc.expected_promise_valid
|
||||
result.checks["delivery_promise"] = {
|
||||
"expected_valid": sc.expected_promise_valid,
|
||||
"actual_valid": validation["valid"],
|
||||
"motion_ratio": validation["motion_ratio"],
|
||||
"passed": ok,
|
||||
}
|
||||
if not ok:
|
||||
result.passed = False
|
||||
result.errors.append(
|
||||
f"delivery_promise: expected valid={sc.expected_promise_valid}, "
|
||||
f"got valid={validation['valid']} (motion_ratio={validation['motion_ratio']:.0%})"
|
||||
)
|
||||
except Exception as e:
|
||||
result.passed = False
|
||||
result.errors.append(f"delivery_promise error: {e}")
|
||||
|
||||
results.append(result)
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def print_results(results: list[BenchResult], verbose: bool = False) -> None:
|
||||
passed = sum(1 for r in results if r.passed)
|
||||
total = len(results)
|
||||
|
||||
print(f"\n{'='*60}")
|
||||
print(f" Video Production Bench: {passed}/{total} scenarios passed")
|
||||
print(f"{'='*60}\n")
|
||||
|
||||
for r in results:
|
||||
status = "PASS" if r.passed else "FAIL"
|
||||
print(f" [{status}] {r.scenario}")
|
||||
if verbose or not r.passed:
|
||||
for check_name, check in r.checks.items():
|
||||
check_status = "ok" if check.get("passed") else "FAIL"
|
||||
print(f" {check_name}: {check_status}")
|
||||
if verbose:
|
||||
for k, v in check.items():
|
||||
if k != "passed":
|
||||
print(f" {k}: {v}")
|
||||
for err in r.errors:
|
||||
print(f" ERROR: {err}")
|
||||
|
||||
print(f"\n Total: {total} | Passed: {passed} | Failed: {total - passed}")
|
||||
if passed == total:
|
||||
print(" All scenarios passed.\n")
|
||||
else:
|
||||
print(f" {total - passed} scenario(s) need attention.\n")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Video production bench runner")
|
||||
parser.add_argument("--tag", help="Filter scenarios by tag")
|
||||
parser.add_argument("--verbose", "-v", action="store_true", help="Show per-scenario details")
|
||||
args = parser.parse_args()
|
||||
|
||||
scenarios = build_scenarios()
|
||||
if args.tag:
|
||||
scenarios = [s for s in scenarios if args.tag in s.tags]
|
||||
|
||||
if not scenarios:
|
||||
print(f"No scenarios found{' for tag ' + args.tag if args.tag else ''}")
|
||||
sys.exit(1)
|
||||
|
||||
results = run_bench(scenarios, verbose=args.verbose)
|
||||
print_results(results, verbose=args.verbose)
|
||||
|
||||
# Exit with failure code if any scenario failed
|
||||
if not all(r.passed for r in results):
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+138
-34
@@ -96,47 +96,150 @@ a11y = validate_accessibility(playbook)
|
||||
print(f" Playbook a11y: pass={a11y['pass']}, errors={a11y['error_count']}, warnings={a11y['warning_count']}")
|
||||
|
||||
# ===================================================================
|
||||
# Stage 1: idea -> brief
|
||||
# Stage 0: research -> research_brief (minimal, to satisfy pipeline order)
|
||||
# ===================================================================
|
||||
print("\n--- Stage 1: idea ---")
|
||||
brief = {
|
||||
print("\n--- Stage 0: research ---")
|
||||
research_brief = {
|
||||
"version": "1.0",
|
||||
"title": "AI Video Production in 60 Seconds",
|
||||
"hook": "What if you could create a professional video in 60 seconds with just a text prompt?",
|
||||
"key_points": [
|
||||
"Traditional video production takes days or weeks",
|
||||
"AI can automate scripting, visuals, narration, and editing",
|
||||
"OpenMontage orchestrates the full pipeline",
|
||||
"topic": "AI Video Production",
|
||||
"research_date": "2026-04-02",
|
||||
"landscape": {
|
||||
"existing_content": [
|
||||
{"title": "AI Video Tools Explained", "source": "youtube", "angle": "tutorial", "what_it_covers": "Tool comparison", "what_it_misses": "End-to-end pipeline"},
|
||||
{"title": "Making Videos with AI", "source": "blog", "angle": "overview", "what_it_covers": "Market landscape", "what_it_misses": "Hands-on workflow"},
|
||||
{"title": "AI Content Creation Guide", "source": "youtube", "angle": "how-to", "what_it_covers": "Individual tools", "what_it_misses": "Orchestration concept"},
|
||||
],
|
||||
"saturated_angles": ["basic tool demos", "AI will replace editors"],
|
||||
"underserved_gaps": ["End-to-end orchestration pipeline", "Cost vs quality tradeoffs"],
|
||||
},
|
||||
"data_points": [
|
||||
{"claim": "AI video market growing 25% YoY", "source_url": "https://example.com/report", "credibility": "primary_source"},
|
||||
{"claim": "70% of creators want automated editing", "source_url": "https://example.com/survey", "credibility": "primary_source"},
|
||||
{"claim": "Average production time drops 80% with AI", "source_url": "https://example.com/study", "credibility": "secondary_source"},
|
||||
],
|
||||
"tone": "confident, energetic",
|
||||
"style": "clean-professional",
|
||||
"target_platform": "youtube",
|
||||
"target_duration_seconds": 60,
|
||||
"target_audience": "content creators and developers",
|
||||
"cta": "Try OpenMontage today",
|
||||
"angle_options": [
|
||||
{"name": "democratization", "description": "AI makes video creation accessible to everyone"},
|
||||
{"name": "workflow", "description": "AI automates the tedious parts of video production"},
|
||||
{"name": "quality", "description": "AI-generated content is reaching professional quality"},
|
||||
"audience_insights": {
|
||||
"common_questions": [
|
||||
"Can AI really make professional-looking videos?",
|
||||
"How much does AI video production cost?",
|
||||
"What tools do I need to get started?",
|
||||
],
|
||||
"misconceptions": [
|
||||
{"myth": "AI videos always look robotic", "reality": "Modern AI produces broadcast-quality output"},
|
||||
],
|
||||
"knowledge_level": "Familiar with basic editing but not AI-specific tools",
|
||||
},
|
||||
"angles_discovered": [
|
||||
{"name": "60-Second Studio", "hook": "Your entire production team in a single prompt", "type": "trending", "why_now": "AI orchestration tools just reached production quality"},
|
||||
{"name": "Quality Democratization", "hook": "Pro-grade video without pro skills", "type": "evergreen", "why_now": "Creator economy growing, skill gap remains"},
|
||||
{"name": "The Hidden Cost Myth", "hook": "AI video costs pennies, not thousands", "type": "contrarian", "why_now": "Most creators still think AI video is expensive"},
|
||||
],
|
||||
"sources": [
|
||||
{"url": "https://example.com/ai-video", "title": "AI Video Production Overview", "used_for": "landscape"},
|
||||
{"url": "https://example.com/ai-tools", "title": "Top AI Video Tools 2026", "used_for": "data_points"},
|
||||
{"url": "https://example.com/workflow", "title": "Automated Video Workflows", "used_for": "angles"},
|
||||
{"url": "https://example.com/creators", "title": "Creator Economy Report", "used_for": "audience_insights"},
|
||||
{"url": "https://example.com/market", "title": "AI Video Market Analysis", "used_for": "landscape"},
|
||||
],
|
||||
"selected_angle": "workflow",
|
||||
}
|
||||
|
||||
try:
|
||||
validate_artifact("brief", brief)
|
||||
check("Brief validates against schema", True)
|
||||
validate_artifact("research_brief", research_brief)
|
||||
check("Research brief validates against schema", True)
|
||||
except Exception as e:
|
||||
check("Brief validates against schema", False, str(e))
|
||||
check("Research brief validates against schema", False, str(e))
|
||||
|
||||
cp_path = write_checkpoint(
|
||||
PIPELINE_DIR, PROJECT_ID, "idea", "completed",
|
||||
artifacts={"brief": brief},
|
||||
PIPELINE_DIR, PROJECT_ID, "research", "completed",
|
||||
artifacts={"research_brief": research_brief},
|
||||
pipeline_type="animated-explainer",
|
||||
style_playbook="clean-professional",
|
||||
)
|
||||
check("Idea checkpoint written", cp_path.exists())
|
||||
# Next uncompleted stage in global STAGES order (research/proposal come before idea)
|
||||
check("Next stage after idea", get_next_stage(PIPELINE_DIR, PROJECT_ID) == "research")
|
||||
check("Research checkpoint written", cp_path.exists())
|
||||
|
||||
# ===================================================================
|
||||
# Stage 1: proposal -> proposal_packet
|
||||
# ===================================================================
|
||||
print("\n--- Stage 1: proposal ---")
|
||||
proposal_packet = {
|
||||
"version": "1.0",
|
||||
"concept_options": [
|
||||
{
|
||||
"id": "c1",
|
||||
"title": "AI Video Production in 60 Seconds",
|
||||
"hook": "What if you could create a professional video in 60 seconds?",
|
||||
"narrative_structure": "problem_solution",
|
||||
"visual_approach": "Clean motion graphics with side-by-side comparisons",
|
||||
"suggested_playbook": "clean-professional",
|
||||
"target_audience": "content creators and developers",
|
||||
"target_platform": "youtube",
|
||||
"target_duration_seconds": 60,
|
||||
"why_this_works": "Direct problem/solution framing with clear visual proof points",
|
||||
},
|
||||
{
|
||||
"id": "c2",
|
||||
"title": "The 60-Second Studio",
|
||||
"hook": "Your entire production team, in a single prompt.",
|
||||
"narrative_structure": "journey",
|
||||
"visual_approach": "Animated workflow diagram that builds step by step",
|
||||
"suggested_playbook": "flat-motion-graphics",
|
||||
"target_audience": "content creators and developers",
|
||||
"target_platform": "youtube",
|
||||
"target_duration_seconds": 60,
|
||||
"why_this_works": "Journey structure creates natural pacing for a process walkthrough",
|
||||
},
|
||||
{
|
||||
"id": "c3",
|
||||
"title": "From Prompt to Premiere",
|
||||
"hook": "Traditional video production takes weeks. This takes seconds.",
|
||||
"narrative_structure": "comparison",
|
||||
"visual_approach": "Split-screen timeline comparison: old way vs AI way",
|
||||
"suggested_playbook": "clean-professional",
|
||||
"target_audience": "content creators and developers",
|
||||
"target_platform": "youtube",
|
||||
"target_duration_seconds": 60,
|
||||
"why_this_works": "Comparison structure makes the value proposition immediately tangible",
|
||||
},
|
||||
],
|
||||
"selected_concept": {"concept_id": "c1", "rationale": "Direct problem/solution framing is most effective for this audience"},
|
||||
"production_plan": {
|
||||
"pipeline": "animated-explainer",
|
||||
"playbook": "clean-professional",
|
||||
"stages": [
|
||||
{"stage": "script", "tools": [{"tool_name": "tts_selector", "role": "narration", "available": True}], "approach": "AI-written script with TTS narration"},
|
||||
{"stage": "scene_plan", "tools": [], "approach": "5 scenes with motion graphics"},
|
||||
{"stage": "assets", "tools": [{"tool_name": "image_selector", "role": "visuals", "available": True}], "approach": "AI-generated images"},
|
||||
{"stage": "edit", "tools": [], "approach": "Automated edit decisions"},
|
||||
{"stage": "compose", "tools": [{"tool_name": "video_compose", "role": "render", "available": True}], "approach": "Remotion render"},
|
||||
],
|
||||
},
|
||||
"cost_estimate": {
|
||||
"total_estimated_usd": 0.50,
|
||||
"line_items": [
|
||||
{"tool": "tts_selector", "operation": "narration", "estimated_usd": 0.10},
|
||||
{"tool": "image_selector", "operation": "5 images", "estimated_usd": 0.30},
|
||||
{"tool": "music_gen", "operation": "background track", "estimated_usd": 0.10},
|
||||
],
|
||||
"budget_verdict": "within_budget",
|
||||
},
|
||||
"approval": {
|
||||
"status": "approved",
|
||||
"approved_budget_usd": 2.00,
|
||||
},
|
||||
}
|
||||
|
||||
try:
|
||||
validate_artifact("proposal_packet", proposal_packet)
|
||||
check("Proposal packet validates against schema", True)
|
||||
except Exception as e:
|
||||
check("Proposal packet validates against schema", False, str(e))
|
||||
|
||||
cp_path = write_checkpoint(
|
||||
PIPELINE_DIR, PROJECT_ID, "proposal", "completed",
|
||||
artifacts={"proposal_packet": proposal_packet},
|
||||
pipeline_type="animated-explainer",
|
||||
style_playbook="clean-professional",
|
||||
)
|
||||
check("Proposal checkpoint written", cp_path.exists())
|
||||
check("Next stage after proposal", get_next_stage(PIPELINE_DIR, PROJECT_ID, "animated-explainer") == "script")
|
||||
|
||||
# ===================================================================
|
||||
# Stage 2: script
|
||||
@@ -183,7 +286,7 @@ write_checkpoint(
|
||||
artifacts={"script": script},
|
||||
pipeline_type="animated-explainer",
|
||||
)
|
||||
check("Completed stages", get_completed_stages(PIPELINE_DIR, PROJECT_ID) == ["idea", "script"]) # idea and script appear in STAGES order
|
||||
check("Completed stages", get_completed_stages(PIPELINE_DIR, PROJECT_ID) == ["research", "proposal", "script"]) # research, proposal and script in pipeline order
|
||||
|
||||
# ===================================================================
|
||||
# Stage 3: scene_plan
|
||||
@@ -520,13 +623,14 @@ write_checkpoint(
|
||||
# ===================================================================
|
||||
print("\n--- Final validation ---")
|
||||
|
||||
E2E_STAGES = ["research", "proposal", "script", "scene_plan", "assets", "edit", "compose", "publish"]
|
||||
completed = get_completed_stages(PIPELINE_DIR, PROJECT_ID)
|
||||
check("All 7 stages completed", len(completed) == 7, f"completed={completed}")
|
||||
check("Next stage is None (done)", get_next_stage(PIPELINE_DIR, PROJECT_ID) is None)
|
||||
check("Stages in correct order", completed == STAGES, f"{completed}")
|
||||
check("All 8 stages completed", len(completed) == 8, f"completed={completed}")
|
||||
check("Next stage is None (done)", get_next_stage(PIPELINE_DIR, PROJECT_ID, "animated-explainer") is None)
|
||||
check("Stages in correct order", completed == E2E_STAGES, f"{completed}")
|
||||
|
||||
# Verify all checkpoints are readable
|
||||
for stage in STAGES:
|
||||
for stage in E2E_STAGES:
|
||||
cp = read_checkpoint(PIPELINE_DIR, PROJECT_ID, stage)
|
||||
check(f"Checkpoint {stage} readable", cp is not None)
|
||||
if cp:
|
||||
|
||||
Reference in New Issue
Block a user