Fix video generation pipeline gaps found during UAT

Four gaps found during user acceptance testing of the reference-video
production workflow:

- video_selector: expose aspect_ratio, duration, reference_image_path,
  reference_image_url, and image_url in schema so agents can discover
  these critical params. Auto-upload local images when the selected
  provider requires a URL.
- _shared.py: add upload_image_fal() for local→URL image bridging via
  fal.ai storage. Fix upload_image_heygen() to try v2 presigned upload
  before falling back to fal.ai (old /v1/asset endpoint returns 404).
- kling_video: call probe_output() so response includes output_path,
  duration_seconds, file_size_mb, video dimensions, and codec info.
This commit is contained in:
calesthio
2026-04-04 11:55:54 -07:00
parent 61c704149e
commit 65a6b32ebd
3 changed files with 109 additions and 8 deletions
+68 -8
View File
@@ -355,22 +355,82 @@ def poll_heygen(execution_id: str, api_key: str, timeout: int = 600) -> str:
raise TimeoutError(f"HeyGen execution {execution_id} timed out after {timeout}s")
def upload_image_fal(image_path: str) -> str:
"""Upload a local image to fal.ai storage and return a public URL."""
import requests
api_key = os.environ.get("FAL_KEY") or os.environ.get("FAL_AI_API_KEY")
if not api_key:
raise RuntimeError("FAL_KEY or FAL_AI_API_KEY required for image upload")
path = Path(image_path)
if not path.exists():
raise FileNotFoundError(f"Image not found: {image_path}")
suffix = path.suffix.lower()
content_type = {"png": "image/png", "jpg": "image/jpeg", "jpeg": "image/jpeg", "webp": "image/webp"}.get(
suffix.lstrip("."), "image/png"
)
# Initiate upload
init_resp = requests.post(
"https://rest.alpha.fal.ai/storage/upload/initiate",
headers={"Authorization": f"Key {api_key}", "Content-Type": "application/json"},
json={"content_type": content_type, "file_name": path.name},
timeout=30,
)
init_resp.raise_for_status()
data = init_resp.json()
# Upload file content
put_resp = requests.put(
data["upload_url"],
headers={"Content-Type": content_type},
data=path.read_bytes(),
timeout=60,
)
put_resp.raise_for_status()
return data["file_url"]
def upload_image_heygen(image_path: str, api_key: str) -> str:
"""Upload a local image to HeyGen and return a public URL.
Tries the v2 presigned-upload endpoint first, falls back to fal.ai storage.
"""
import requests
path = Path(image_path)
if not path.exists():
raise FileNotFoundError(f"Image not found: {image_path}")
with path.open("rb") as handle:
response = requests.post(
"https://api.heygen.com/v1/asset",
headers={"X-Api-Key": api_key},
files={"file": (path.name, handle, "image/png")},
timeout=60,
# Try HeyGen v2 presigned upload
try:
resp = requests.post(
"https://api.heygen.com/v2/assets/upload",
headers={"X-Api-Key": api_key, "Content-Type": "application/json"},
json={"content_type": "image/png", "file_name": path.name},
timeout=30,
)
response.raise_for_status()
return response.json().get("data", {}).get("url", "")
if resp.status_code == 200:
data = resp.json().get("data", {})
upload_url = data.get("upload_url")
file_url = data.get("url") or data.get("file_url")
if upload_url and file_url:
put_resp = requests.put(
upload_url,
headers={"Content-Type": "image/png"},
data=path.read_bytes(),
timeout=60,
)
put_resp.raise_for_status()
return file_url
except Exception:
pass
# Fallback to fal.ai storage upload
return upload_image_fal(image_path)
def generate_heygen_video(inputs: dict[str, Any]) -> ToolResult:
+8
View File
@@ -189,13 +189,21 @@ class KlingVideo(BaseTool):
except Exception as e:
return ToolResult(success=False, error=f"Kling video generation failed: {e}")
from tools.video._shared import probe_output
probed = probe_output(output_path)
return ToolResult(
success=True,
data={
"provider": "kling",
"model": f"fal-ai/{model_path}",
"prompt": inputs["prompt"],
"operation": operation,
"aspect_ratio": inputs.get("aspect_ratio", "16:9"),
"output": str(output_path),
"output_path": str(output_path),
"format": "mp4",
**probed,
},
artifacts=[str(output_path)],
cost_usd=self.estimate_cost(inputs),
+33
View File
@@ -50,6 +50,28 @@ class VideoSelector(BaseTool):
},
"allowed_providers": {"type": "array", "items": {"type": "string"}},
"operation": {"type": "string", "enum": ["text_to_video", "image_to_video", "rank"], "default": "text_to_video"},
"aspect_ratio": {
"type": "string",
"enum": ["16:9", "9:16", "1:1"],
"default": "16:9",
"description": "Video aspect ratio. Passed through to the selected provider.",
},
"duration": {
"type": "string",
"description": "Duration hint (e.g., '5', '10'). Passed through to the selected provider.",
},
"reference_image_path": {
"type": "string",
"description": "Local path to a reference image for image_to_video. Auto-uploaded if the provider requires a URL.",
},
"reference_image_url": {
"type": "string",
"description": "URL of a reference image for image_to_video.",
},
"image_url": {
"type": "string",
"description": "Alias for reference_image_url (used by some providers like Kling via fal.ai).",
},
"output_path": {"type": "string"},
},
}
@@ -123,6 +145,17 @@ class VideoSelector(BaseTool):
if "query" in required and "query" not in adapted:
adapted["query"] = adapted.get("prompt", "")
# Auto-resolve reference_image_path to a URL for providers that need it
if adapted.get("operation") == "image_to_video" and adapted.get("reference_image_path"):
tool_props = getattr(tool, "input_schema", {}).get("properties", {})
# If the provider uses image_url (not reference_image_path), upload and convert
if "image_url" in tool_props and "image_url" not in adapted:
try:
from tools.video._shared import upload_image_fal
adapted["image_url"] = upload_image_fal(adapted["reference_image_path"])
except Exception as e:
return ToolResult(success=False, error=f"Failed to upload reference image: {e}")
result = tool.execute(adapted)
if result.success:
result.data.setdefault("selected_tool", tool.name)