Implementation spec: governance, decision intelligence, theme system, and E2E bug fixes

Implements the 2026-04-02 transformation spec (Phases 1-8) and fixes all
critical bugs found during 5-pipeline E2E testing.

Governance & Decision Intelligence:
- Pipeline-specific stage order in checkpoint (replaces global STAGES list)
- Provider scoring engine (lib/scoring.py) with 7-dimension weighted ranking
- Decision log artifact enforced at proposal/idea stage across all 10 pipelines
- Delivery promise classifier prevents silent motion-to-still downgrades
- Structured shot language in scene_plan schema (camera, lens, lighting, DOF)
- Variation checker and slideshow risk scorer block samey output before render
- Creative intake, capability extension, and creative-intake meta skills
- Final self-review artifact with 5 mandatory checks before presenting output
- Source media review contract for user-supplied footage

Render & Theme System:
- Remotion AnimatedBackground now derives colors from playbook (no more hardcoded
  dark blue fintech gradient on every video)
- video_compose builds custom ThemeConfig from playbook YAML colors/fonts —
  custom playbooks flow through to Remotion automatically
- Explainer component wires theme to all child components (charts, cards, etc.)
- resolveAsset() handles absolute paths on Windows/Unix via file:// URIs
- RENDERER_FAMILY_MAP synced with actual Remotion compositions

Critical Bug Fixes:
- Windows npx subprocess: run_command() resolves .cmd wrappers via shutil.which()
- Silent renderer downgrade: Remotion failure now returns explicit error with
  options instead of silently falling back to FFmpeg
- .env inline comment parsing strips trailing # comments from API keys
- concat_path UnboundLocalError in video_compose finally block
- audio_mixer and showcase_card capture=True kwarg bug
- Selector estimate_cost() calls fixed (_select_tool -> _select_best_tool)
- asset_manifest schema expanded with provider, license, subtype fields
- screen-demo subtitle_gen moved from required to optional tools
- Duration drift detection in post-render final review (>25% warns)

lib/checkpoint.py

@@ -16,7 +16,16 @@ import jsonschema
 
 from schemas.artifacts import ARTIFACT_NAMES, validate_artifact
 
-STAGES = ["research", "proposal", "idea", "script", "scene_plan", "assets", "edit", "compose", "publish"]
+# All known stages across all pipelines (used only for artifact name lookup).
+ALL_KNOWN_STAGES = frozenset([
+    "research", "proposal", "idea", "script", "scene_plan",
+    "assets", "edit", "compose", "publish",
+])
+
+# Backward-compatible alias — existing code / tests that import STAGES still work.
+# New code should use get_pipeline_stages(pipeline_type) instead.
+STAGES = ["research", "proposal", "idea", "script", "scene_plan",
+          "assets", "edit", "compose", "publish"]
 
 CANONICAL_STAGE_ARTIFACTS = {
     "research": "research_brief",
@@ -30,6 +39,40 @@ CANONICAL_STAGE_ARTIFACTS = {
     "publish": "publish_log",
 }
 
+# Additional artifacts that may be produced alongside canonical ones.
+# These are not stage-defining but are required by governance contracts.
+SUPPLEMENTARY_ARTIFACTS = {
+    "source_media_review",  # Required before first planning stage when user media exists
+    "final_review",         # Required by compose stage before presenting to user
+}
+
+
+def get_pipeline_stages(pipeline_type: str | None) -> list[str]:
+    """Return the ordered stage list for a specific pipeline.
+
+    Falls back to STAGES (deterministic canonical order) when pipeline_type
+    is not provided or the manifest cannot be loaded.
+
+    Previous versions used a set intersection here, which produced
+    nondeterministic ordering. The fallback now uses a stable list.
+    """
+    if pipeline_type is None:
+        # Deterministic canonical fallback — sorted to ensure stable ordering
+        import logging
+        logging.getLogger(__name__).warning(
+            "get_pipeline_stages called without pipeline_type — "
+            "using canonical fallback order. Pass pipeline_type for correctness."
+        )
+        return list(STAGES)
+
+    try:
+        from lib.pipeline_loader import load_pipeline, get_stage_order
+        manifest = load_pipeline(pipeline_type)
+        return get_stage_order(manifest)
+    except (FileNotFoundError, Exception):
+        # Graceful fallback: return all known stages in canonical order
+        return list(STAGES)
+
 CHECKPOINT_SCHEMA_PATH = (
     Path(__file__).resolve().parent.parent
     / "schemas"
@@ -76,13 +119,26 @@ def _validate_artifacts_for_stage(
 
 
 def validate_checkpoint(checkpoint: dict[str, Any]) -> None:
-    """Validate checkpoint structure and canonical artifact payloads."""
+    """Validate checkpoint structure and canonical artifact payloads.
+
+    Uses pipeline_type (if present) to resolve the valid stage list.
+    Falls back to ALL_KNOWN_STAGES when pipeline_type is absent.
+    """
     stage = checkpoint.get("stage")
     status = checkpoint.get("status")
     artifacts = checkpoint.get("artifacts")
+    pipeline_type = checkpoint.get("pipeline_type")
 
-    if not isinstance(stage, str) or stage not in STAGES:
-        raise CheckpointValidationError(f"Invalid stage: {stage!r}")
+    valid_stages = (
+        set(get_pipeline_stages(pipeline_type)) if pipeline_type
+        else ALL_KNOWN_STAGES
+    )
+
+    if not isinstance(stage, str) or stage not in valid_stages:
+        raise CheckpointValidationError(
+            f"Invalid stage: {stage!r} for pipeline {pipeline_type!r}. "
+            f"Valid stages: {sorted(valid_stages)}"
+        )
     if not isinstance(status, str):
         raise CheckpointValidationError(f"Invalid status: {status!r}")
     if not isinstance(artifacts, dict):
@@ -100,6 +156,40 @@ def _checkpoint_path(pipeline_dir: Path, project_id: str, stage: str) -> Path:
     return pipeline_dir / project_id / f"checkpoint_{stage}.json"
 
 
+def _decision_log_path(pipeline_dir: Path, project_id: str) -> Path:
+    return pipeline_dir / project_id / "decision_log.json"
+
+
+def _merge_decision_log(
+    pipeline_dir: Path, project_id: str, new_log: dict[str, Any]
+) -> None:
+    """Append new decisions to the project-level decision log.
+
+    Each stage may produce decisions. This function merges them into a
+    single cumulative file so reviewers and the bench can inspect the
+    full audit trail.
+    """
+    path = _decision_log_path(pipeline_dir, project_id)
+    if path.exists():
+        with open(path) as f:
+            existing = json.load(f)
+    else:
+        existing = {
+            "version": "1.0",
+            "project_id": project_id,
+            "decisions": [],
+        }
+
+    existing_ids = {d["decision_id"] for d in existing.get("decisions", [])}
+    for decision in new_log.get("decisions", []):
+        if decision.get("decision_id") not in existing_ids:
+            existing["decisions"].append(decision)
+
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with open(path, "w") as f:
+        json.dump(existing, f, indent=2)
+
+
 def write_checkpoint(
     pipeline_dir: Path,
     project_id: str,
@@ -118,12 +208,20 @@ def write_checkpoint(
     metadata: Optional[dict] = None,
 ) -> Path:
     """Write a checkpoint file for a pipeline stage."""
-    if stage not in STAGES:
-        raise ValueError(f"Invalid stage: {stage!r}. Must be one of {STAGES}")
+    valid_stages = (
+        set(get_pipeline_stages(pipeline_type)) if pipeline_type
+        else ALL_KNOWN_STAGES
+    )
+    if stage not in valid_stages:
+        raise ValueError(
+            f"Invalid stage: {stage!r} for pipeline {pipeline_type!r}. "
+            f"Valid stages: {sorted(valid_stages)}"
+        )
 
     checkpoint = {
         "version": "1.0",
         "project_id": project_id,
+        "pipeline_type": pipeline_type or "unknown",
         "stage": stage,
         "status": status,
         "timestamp": datetime.now(timezone.utc).isoformat(),
@@ -132,9 +230,6 @@ def write_checkpoint(
         "human_approved": human_approved,
         "artifacts": artifacts,
     }
-
-    if pipeline_type is not None:
-        checkpoint["pipeline_type"] = pipeline_type
     if style_playbook is not None:
         checkpoint["style_playbook"] = style_playbook
     if review is not None:
@@ -146,6 +241,26 @@ def write_checkpoint(
     if metadata is not None:
         checkpoint["metadata"] = metadata
 
+    # Merge decision_log: if this checkpoint carries new decisions,
+    # append them to the project-level decision log file, then write the
+    # reference back into relevant artifacts so downstream consumers can find it.
+    if "decision_log" in artifacts and isinstance(artifacts["decision_log"], dict):
+        _merge_decision_log(pipeline_dir, project_id, artifacts["decision_log"])
+        log_ref = str(_decision_log_path(pipeline_dir, project_id))
+
+        # Write decision_log_ref into proposal_packet and render_report
+        # artifacts if they are present in this checkpoint.
+        for artifact_key in ("proposal_packet", "render_report"):
+            if artifact_key in artifacts and isinstance(artifacts[artifact_key], dict):
+                plan_or_top = artifacts[artifact_key]
+                # proposal_packet stores it under production_plan
+                if artifact_key == "proposal_packet":
+                    plan = plan_or_top.get("production_plan")
+                    if isinstance(plan, dict):
+                        plan["decision_log_ref"] = log_ref
+                else:
+                    plan_or_top["decision_log_ref"] = log_ref
+
     validate_checkpoint(checkpoint)
 
     path = _checkpoint_path(pipeline_dir, project_id, stage)
@@ -191,20 +306,35 @@ def get_latest_checkpoint(
     return checkpoint
 
 
-def get_completed_stages(pipeline_dir: Path, project_id: str) -> list[str]:
-    """Return list of stages that have a completed checkpoint."""
+def get_completed_stages(
+    pipeline_dir: Path, project_id: str, pipeline_type: str | None = None
+) -> list[str]:
+    """Return list of stages that have a completed checkpoint.
+
+    When pipeline_type is provided, only checks stages defined in that
+    pipeline's manifest — preventing false positives from leftover
+    checkpoints of a different pipeline type.
+    """
+    stages_to_check = get_pipeline_stages(pipeline_type)
     completed = []
-    for stage in STAGES:
+    for stage in stages_to_check:
         cp = read_checkpoint(pipeline_dir, project_id, stage)
         if cp and cp.get("status") == "completed":
             completed.append(stage)
     return completed
 
 
-def get_next_stage(pipeline_dir: Path, project_id: str) -> Optional[str]:
-    """Determine the next stage to run based on completed checkpoints."""
-    completed = set(get_completed_stages(pipeline_dir, project_id))
-    for stage in STAGES:
+def get_next_stage(
+    pipeline_dir: Path, project_id: str, pipeline_type: str | None = None
+) -> Optional[str]:
+    """Determine the next stage to run based on completed checkpoints.
+
+    Uses pipeline-specific stage order so that pipelines with different
+    stage sequences (e.g. cinematic vs explainer) progress correctly.
+    """
+    stages = get_pipeline_stages(pipeline_type) if pipeline_type else STAGES
+    completed = set(get_completed_stages(pipeline_dir, project_id, pipeline_type))
+    for stage in stages:
         if stage not in completed:
             return stage
     return None