Initial release — OpenMontage: the first open-source agentic video production system
11 production pipelines, 47 tools, 124 agent skills. Supports cloud APIs (fal.ai, OpenAI, ElevenLabs, Suno, HeyGen, Runway) and free local providers (diffusers, Piper TTS, WAN 2.1, Hunyuan, CogVideo). Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
calesthio
@@ -0,0 +1,210 @@
|
||||
"""Checkpoint writer/reader for pipeline state persistence.
|
||||
|
||||
Each stage writes a checkpoint after completion. The orchestrator uses
|
||||
checkpoints to resume pipelines and to present state at human checkpoints.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from functools import lru_cache
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any, Optional
|
||||
|
||||
import jsonschema
|
||||
|
||||
from schemas.artifacts import ARTIFACT_NAMES, validate_artifact
|
||||
|
||||
STAGES = ["research", "proposal", "idea", "script", "scene_plan", "assets", "edit", "compose", "publish"]
|
||||
|
||||
CANONICAL_STAGE_ARTIFACTS = {
|
||||
"research": "research_brief",
|
||||
"proposal": "proposal_packet",
|
||||
"idea": "brief",
|
||||
"script": "script",
|
||||
"scene_plan": "scene_plan",
|
||||
"assets": "asset_manifest",
|
||||
"edit": "edit_decisions",
|
||||
"compose": "render_report",
|
||||
"publish": "publish_log",
|
||||
}
|
||||
|
||||
CHECKPOINT_SCHEMA_PATH = (
|
||||
Path(__file__).resolve().parent.parent
|
||||
/ "schemas"
|
||||
/ "checkpoints"
|
||||
/ "checkpoint.schema.json"
|
||||
)
|
||||
|
||||
|
||||
class CheckpointValidationError(ValueError):
|
||||
"""Raised when a checkpoint or its canonical artifacts are invalid."""
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def _load_checkpoint_schema() -> dict[str, Any]:
|
||||
with open(CHECKPOINT_SCHEMA_PATH) as f:
|
||||
return json.load(f)
|
||||
|
||||
|
||||
def _validate_artifacts_for_stage(
|
||||
stage: str,
|
||||
status: str,
|
||||
artifacts: dict[str, Any],
|
||||
) -> None:
|
||||
required_artifact = CANONICAL_STAGE_ARTIFACTS[stage]
|
||||
if status in {"completed", "awaiting_human"} and required_artifact not in artifacts:
|
||||
raise CheckpointValidationError(
|
||||
f"Stage {stage!r} with status {status!r} must include "
|
||||
f"canonical artifact {required_artifact!r}"
|
||||
)
|
||||
|
||||
for artifact_name, artifact_data in artifacts.items():
|
||||
if artifact_name not in ARTIFACT_NAMES:
|
||||
continue
|
||||
if not isinstance(artifact_data, dict):
|
||||
raise CheckpointValidationError(
|
||||
f"Artifact {artifact_name!r} must be a JSON object matching its schema"
|
||||
)
|
||||
try:
|
||||
validate_artifact(artifact_name, artifact_data)
|
||||
except Exception as exc:
|
||||
raise CheckpointValidationError(
|
||||
f"Artifact {artifact_name!r} failed schema validation: {exc}"
|
||||
) from exc
|
||||
|
||||
|
||||
def validate_checkpoint(checkpoint: dict[str, Any]) -> None:
|
||||
"""Validate checkpoint structure and canonical artifact payloads."""
|
||||
stage = checkpoint.get("stage")
|
||||
status = checkpoint.get("status")
|
||||
artifacts = checkpoint.get("artifacts")
|
||||
|
||||
if not isinstance(stage, str) or stage not in STAGES:
|
||||
raise CheckpointValidationError(f"Invalid stage: {stage!r}")
|
||||
if not isinstance(status, str):
|
||||
raise CheckpointValidationError(f"Invalid status: {status!r}")
|
||||
if not isinstance(artifacts, dict):
|
||||
raise CheckpointValidationError("Checkpoint artifacts must be a dictionary")
|
||||
|
||||
_validate_artifacts_for_stage(stage, status, artifacts)
|
||||
|
||||
try:
|
||||
jsonschema.validate(instance=checkpoint, schema=_load_checkpoint_schema())
|
||||
except jsonschema.ValidationError as exc:
|
||||
raise CheckpointValidationError(f"Checkpoint failed schema validation: {exc.message}") from exc
|
||||
|
||||
|
||||
def _checkpoint_path(pipeline_dir: Path, project_id: str, stage: str) -> Path:
|
||||
return pipeline_dir / project_id / f"checkpoint_{stage}.json"
|
||||
|
||||
|
||||
def write_checkpoint(
|
||||
pipeline_dir: Path,
|
||||
project_id: str,
|
||||
stage: str,
|
||||
status: str,
|
||||
artifacts: dict[str, Any],
|
||||
*,
|
||||
pipeline_type: Optional[str] = None,
|
||||
style_playbook: Optional[str] = None,
|
||||
checkpoint_policy: str = "guided",
|
||||
human_approval_required: bool = False,
|
||||
human_approved: bool = False,
|
||||
review: Optional[dict] = None,
|
||||
cost_snapshot: Optional[dict] = None,
|
||||
error: Optional[str] = None,
|
||||
metadata: Optional[dict] = None,
|
||||
) -> Path:
|
||||
"""Write a checkpoint file for a pipeline stage."""
|
||||
if stage not in STAGES:
|
||||
raise ValueError(f"Invalid stage: {stage!r}. Must be one of {STAGES}")
|
||||
|
||||
checkpoint = {
|
||||
"version": "1.0",
|
||||
"project_id": project_id,
|
||||
"stage": stage,
|
||||
"status": status,
|
||||
"timestamp": datetime.now(timezone.utc).isoformat(),
|
||||
"checkpoint_policy": checkpoint_policy,
|
||||
"human_approval_required": human_approval_required,
|
||||
"human_approved": human_approved,
|
||||
"artifacts": artifacts,
|
||||
}
|
||||
|
||||
if pipeline_type is not None:
|
||||
checkpoint["pipeline_type"] = pipeline_type
|
||||
if style_playbook is not None:
|
||||
checkpoint["style_playbook"] = style_playbook
|
||||
if review is not None:
|
||||
checkpoint["review"] = review
|
||||
if cost_snapshot is not None:
|
||||
checkpoint["cost_snapshot"] = cost_snapshot
|
||||
if error is not None:
|
||||
checkpoint["error"] = error
|
||||
if metadata is not None:
|
||||
checkpoint["metadata"] = metadata
|
||||
|
||||
validate_checkpoint(checkpoint)
|
||||
|
||||
path = _checkpoint_path(pipeline_dir, project_id, stage)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(path, "w") as f:
|
||||
json.dump(checkpoint, f, indent=2)
|
||||
|
||||
return path
|
||||
|
||||
|
||||
def read_checkpoint(
|
||||
pipeline_dir: Path, project_id: str, stage: str
|
||||
) -> Optional[dict[str, Any]]:
|
||||
"""Read a checkpoint file. Returns None if not found."""
|
||||
path = _checkpoint_path(pipeline_dir, project_id, stage)
|
||||
if not path.exists():
|
||||
return None
|
||||
with open(path) as f:
|
||||
checkpoint = json.load(f)
|
||||
validate_checkpoint(checkpoint)
|
||||
return checkpoint
|
||||
|
||||
|
||||
def get_latest_checkpoint(
|
||||
pipeline_dir: Path, project_id: str
|
||||
) -> Optional[dict[str, Any]]:
|
||||
"""Find the most recent checkpoint for a project (by file mtime)."""
|
||||
project_dir = pipeline_dir / project_id
|
||||
if not project_dir.exists():
|
||||
return None
|
||||
|
||||
checkpoints = sorted(
|
||||
project_dir.glob("checkpoint_*.json"),
|
||||
key=lambda p: p.stat().st_mtime,
|
||||
reverse=True,
|
||||
)
|
||||
if not checkpoints:
|
||||
return None
|
||||
|
||||
with open(checkpoints[0]) as f:
|
||||
checkpoint = json.load(f)
|
||||
validate_checkpoint(checkpoint)
|
||||
return checkpoint
|
||||
|
||||
|
||||
def get_completed_stages(pipeline_dir: Path, project_id: str) -> list[str]:
|
||||
"""Return list of stages that have a completed checkpoint."""
|
||||
completed = []
|
||||
for stage in STAGES:
|
||||
cp = read_checkpoint(pipeline_dir, project_id, stage)
|
||||
if cp and cp.get("status") == "completed":
|
||||
completed.append(stage)
|
||||
return completed
|
||||
|
||||
|
||||
def get_next_stage(pipeline_dir: Path, project_id: str) -> Optional[str]:
|
||||
"""Determine the next stage to run based on completed checkpoints."""
|
||||
completed = set(get_completed_stages(pipeline_dir, project_id))
|
||||
for stage in STAGES:
|
||||
if stage not in completed:
|
||||
return stage
|
||||
return None
|
||||
Reference in New Issue
Block a user