Fix video generation pipeline gaps found during UAT
Four gaps found during user acceptance testing of the reference-video production workflow: - video_selector: expose aspect_ratio, duration, reference_image_path, reference_image_url, and image_url in schema so agents can discover these critical params. Auto-upload local images when the selected provider requires a URL. - _shared.py: add upload_image_fal() for local→URL image bridging via fal.ai storage. Fix upload_image_heygen() to try v2 presigned upload before falling back to fal.ai (old /v1/asset endpoint returns 404). - kling_video: call probe_output() so response includes output_path, duration_seconds, file_size_mb, video dimensions, and codec info.
This commit is contained in:
+68
-8
@@ -355,22 +355,82 @@ def poll_heygen(execution_id: str, api_key: str, timeout: int = 600) -> str:
|
||||
raise TimeoutError(f"HeyGen execution {execution_id} timed out after {timeout}s")
|
||||
|
||||
|
||||
def upload_image_fal(image_path: str) -> str:
|
||||
"""Upload a local image to fal.ai storage and return a public URL."""
|
||||
import requests
|
||||
|
||||
api_key = os.environ.get("FAL_KEY") or os.environ.get("FAL_AI_API_KEY")
|
||||
if not api_key:
|
||||
raise RuntimeError("FAL_KEY or FAL_AI_API_KEY required for image upload")
|
||||
|
||||
path = Path(image_path)
|
||||
if not path.exists():
|
||||
raise FileNotFoundError(f"Image not found: {image_path}")
|
||||
|
||||
suffix = path.suffix.lower()
|
||||
content_type = {"png": "image/png", "jpg": "image/jpeg", "jpeg": "image/jpeg", "webp": "image/webp"}.get(
|
||||
suffix.lstrip("."), "image/png"
|
||||
)
|
||||
|
||||
# Initiate upload
|
||||
init_resp = requests.post(
|
||||
"https://rest.alpha.fal.ai/storage/upload/initiate",
|
||||
headers={"Authorization": f"Key {api_key}", "Content-Type": "application/json"},
|
||||
json={"content_type": content_type, "file_name": path.name},
|
||||
timeout=30,
|
||||
)
|
||||
init_resp.raise_for_status()
|
||||
data = init_resp.json()
|
||||
|
||||
# Upload file content
|
||||
put_resp = requests.put(
|
||||
data["upload_url"],
|
||||
headers={"Content-Type": content_type},
|
||||
data=path.read_bytes(),
|
||||
timeout=60,
|
||||
)
|
||||
put_resp.raise_for_status()
|
||||
|
||||
return data["file_url"]
|
||||
|
||||
|
||||
def upload_image_heygen(image_path: str, api_key: str) -> str:
|
||||
"""Upload a local image to HeyGen and return a public URL.
|
||||
|
||||
Tries the v2 presigned-upload endpoint first, falls back to fal.ai storage.
|
||||
"""
|
||||
import requests
|
||||
|
||||
path = Path(image_path)
|
||||
if not path.exists():
|
||||
raise FileNotFoundError(f"Image not found: {image_path}")
|
||||
|
||||
with path.open("rb") as handle:
|
||||
response = requests.post(
|
||||
"https://api.heygen.com/v1/asset",
|
||||
headers={"X-Api-Key": api_key},
|
||||
files={"file": (path.name, handle, "image/png")},
|
||||
timeout=60,
|
||||
# Try HeyGen v2 presigned upload
|
||||
try:
|
||||
resp = requests.post(
|
||||
"https://api.heygen.com/v2/assets/upload",
|
||||
headers={"X-Api-Key": api_key, "Content-Type": "application/json"},
|
||||
json={"content_type": "image/png", "file_name": path.name},
|
||||
timeout=30,
|
||||
)
|
||||
response.raise_for_status()
|
||||
return response.json().get("data", {}).get("url", "")
|
||||
if resp.status_code == 200:
|
||||
data = resp.json().get("data", {})
|
||||
upload_url = data.get("upload_url")
|
||||
file_url = data.get("url") or data.get("file_url")
|
||||
if upload_url and file_url:
|
||||
put_resp = requests.put(
|
||||
upload_url,
|
||||
headers={"Content-Type": "image/png"},
|
||||
data=path.read_bytes(),
|
||||
timeout=60,
|
||||
)
|
||||
put_resp.raise_for_status()
|
||||
return file_url
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Fallback to fal.ai storage upload
|
||||
return upload_image_fal(image_path)
|
||||
|
||||
|
||||
def generate_heygen_video(inputs: dict[str, Any]) -> ToolResult:
|
||||
|
||||
@@ -189,13 +189,21 @@ class KlingVideo(BaseTool):
|
||||
except Exception as e:
|
||||
return ToolResult(success=False, error=f"Kling video generation failed: {e}")
|
||||
|
||||
from tools.video._shared import probe_output
|
||||
|
||||
probed = probe_output(output_path)
|
||||
return ToolResult(
|
||||
success=True,
|
||||
data={
|
||||
"provider": "kling",
|
||||
"model": f"fal-ai/{model_path}",
|
||||
"prompt": inputs["prompt"],
|
||||
"operation": operation,
|
||||
"aspect_ratio": inputs.get("aspect_ratio", "16:9"),
|
||||
"output": str(output_path),
|
||||
"output_path": str(output_path),
|
||||
"format": "mp4",
|
||||
**probed,
|
||||
},
|
||||
artifacts=[str(output_path)],
|
||||
cost_usd=self.estimate_cost(inputs),
|
||||
|
||||
@@ -50,6 +50,28 @@ class VideoSelector(BaseTool):
|
||||
},
|
||||
"allowed_providers": {"type": "array", "items": {"type": "string"}},
|
||||
"operation": {"type": "string", "enum": ["text_to_video", "image_to_video", "rank"], "default": "text_to_video"},
|
||||
"aspect_ratio": {
|
||||
"type": "string",
|
||||
"enum": ["16:9", "9:16", "1:1"],
|
||||
"default": "16:9",
|
||||
"description": "Video aspect ratio. Passed through to the selected provider.",
|
||||
},
|
||||
"duration": {
|
||||
"type": "string",
|
||||
"description": "Duration hint (e.g., '5', '10'). Passed through to the selected provider.",
|
||||
},
|
||||
"reference_image_path": {
|
||||
"type": "string",
|
||||
"description": "Local path to a reference image for image_to_video. Auto-uploaded if the provider requires a URL.",
|
||||
},
|
||||
"reference_image_url": {
|
||||
"type": "string",
|
||||
"description": "URL of a reference image for image_to_video.",
|
||||
},
|
||||
"image_url": {
|
||||
"type": "string",
|
||||
"description": "Alias for reference_image_url (used by some providers like Kling via fal.ai).",
|
||||
},
|
||||
"output_path": {"type": "string"},
|
||||
},
|
||||
}
|
||||
@@ -123,6 +145,17 @@ class VideoSelector(BaseTool):
|
||||
if "query" in required and "query" not in adapted:
|
||||
adapted["query"] = adapted.get("prompt", "")
|
||||
|
||||
# Auto-resolve reference_image_path to a URL for providers that need it
|
||||
if adapted.get("operation") == "image_to_video" and adapted.get("reference_image_path"):
|
||||
tool_props = getattr(tool, "input_schema", {}).get("properties", {})
|
||||
# If the provider uses image_url (not reference_image_path), upload and convert
|
||||
if "image_url" in tool_props and "image_url" not in adapted:
|
||||
try:
|
||||
from tools.video._shared import upload_image_fal
|
||||
adapted["image_url"] = upload_image_fal(adapted["reference_image_path"])
|
||||
except Exception as e:
|
||||
return ToolResult(success=False, error=f"Failed to upload reference image: {e}")
|
||||
|
||||
result = tool.execute(adapted)
|
||||
if result.success:
|
||||
result.data.setdefault("selected_tool", tool.name)
|
||||
|
||||
Reference in New Issue
Block a user