From 65a6b32ebd61dc405fd1dd969f03dd274dc777b0 Mon Sep 17 00:00:00 2001 From: calesthio Date: Sat, 4 Apr 2026 11:55:54 -0700 Subject: [PATCH] Fix video generation pipeline gaps found during UAT MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four gaps found during user acceptance testing of the reference-video production workflow: - video_selector: expose aspect_ratio, duration, reference_image_path, reference_image_url, and image_url in schema so agents can discover these critical params. Auto-upload local images when the selected provider requires a URL. - _shared.py: add upload_image_fal() for local→URL image bridging via fal.ai storage. Fix upload_image_heygen() to try v2 presigned upload before falling back to fal.ai (old /v1/asset endpoint returns 404). - kling_video: call probe_output() so response includes output_path, duration_seconds, file_size_mb, video dimensions, and codec info. --- tools/video/_shared.py | 76 +++++++++++++++++++++++++++++++---- tools/video/kling_video.py | 8 ++++ tools/video/video_selector.py | 33 +++++++++++++++ 3 files changed, 109 insertions(+), 8 deletions(-) diff --git a/tools/video/_shared.py b/tools/video/_shared.py index 8d07db0..459a85c 100644 --- a/tools/video/_shared.py +++ b/tools/video/_shared.py @@ -355,22 +355,82 @@ def poll_heygen(execution_id: str, api_key: str, timeout: int = 600) -> str: raise TimeoutError(f"HeyGen execution {execution_id} timed out after {timeout}s") +def upload_image_fal(image_path: str) -> str: + """Upload a local image to fal.ai storage and return a public URL.""" + import requests + + api_key = os.environ.get("FAL_KEY") or os.environ.get("FAL_AI_API_KEY") + if not api_key: + raise RuntimeError("FAL_KEY or FAL_AI_API_KEY required for image upload") + + path = Path(image_path) + if not path.exists(): + raise FileNotFoundError(f"Image not found: {image_path}") + + suffix = path.suffix.lower() + content_type = {"png": "image/png", "jpg": "image/jpeg", "jpeg": "image/jpeg", "webp": "image/webp"}.get( + suffix.lstrip("."), "image/png" + ) + + # Initiate upload + init_resp = requests.post( + "https://rest.alpha.fal.ai/storage/upload/initiate", + headers={"Authorization": f"Key {api_key}", "Content-Type": "application/json"}, + json={"content_type": content_type, "file_name": path.name}, + timeout=30, + ) + init_resp.raise_for_status() + data = init_resp.json() + + # Upload file content + put_resp = requests.put( + data["upload_url"], + headers={"Content-Type": content_type}, + data=path.read_bytes(), + timeout=60, + ) + put_resp.raise_for_status() + + return data["file_url"] + + def upload_image_heygen(image_path: str, api_key: str) -> str: + """Upload a local image to HeyGen and return a public URL. + + Tries the v2 presigned-upload endpoint first, falls back to fal.ai storage. + """ import requests path = Path(image_path) if not path.exists(): raise FileNotFoundError(f"Image not found: {image_path}") - with path.open("rb") as handle: - response = requests.post( - "https://api.heygen.com/v1/asset", - headers={"X-Api-Key": api_key}, - files={"file": (path.name, handle, "image/png")}, - timeout=60, + # Try HeyGen v2 presigned upload + try: + resp = requests.post( + "https://api.heygen.com/v2/assets/upload", + headers={"X-Api-Key": api_key, "Content-Type": "application/json"}, + json={"content_type": "image/png", "file_name": path.name}, + timeout=30, ) - response.raise_for_status() - return response.json().get("data", {}).get("url", "") + if resp.status_code == 200: + data = resp.json().get("data", {}) + upload_url = data.get("upload_url") + file_url = data.get("url") or data.get("file_url") + if upload_url and file_url: + put_resp = requests.put( + upload_url, + headers={"Content-Type": "image/png"}, + data=path.read_bytes(), + timeout=60, + ) + put_resp.raise_for_status() + return file_url + except Exception: + pass + + # Fallback to fal.ai storage upload + return upload_image_fal(image_path) def generate_heygen_video(inputs: dict[str, Any]) -> ToolResult: diff --git a/tools/video/kling_video.py b/tools/video/kling_video.py index a386ac8..02ef69d 100644 --- a/tools/video/kling_video.py +++ b/tools/video/kling_video.py @@ -189,13 +189,21 @@ class KlingVideo(BaseTool): except Exception as e: return ToolResult(success=False, error=f"Kling video generation failed: {e}") + from tools.video._shared import probe_output + + probed = probe_output(output_path) return ToolResult( success=True, data={ "provider": "kling", "model": f"fal-ai/{model_path}", "prompt": inputs["prompt"], + "operation": operation, + "aspect_ratio": inputs.get("aspect_ratio", "16:9"), "output": str(output_path), + "output_path": str(output_path), + "format": "mp4", + **probed, }, artifacts=[str(output_path)], cost_usd=self.estimate_cost(inputs), diff --git a/tools/video/video_selector.py b/tools/video/video_selector.py index dba23c9..9e18ed0 100644 --- a/tools/video/video_selector.py +++ b/tools/video/video_selector.py @@ -50,6 +50,28 @@ class VideoSelector(BaseTool): }, "allowed_providers": {"type": "array", "items": {"type": "string"}}, "operation": {"type": "string", "enum": ["text_to_video", "image_to_video", "rank"], "default": "text_to_video"}, + "aspect_ratio": { + "type": "string", + "enum": ["16:9", "9:16", "1:1"], + "default": "16:9", + "description": "Video aspect ratio. Passed through to the selected provider.", + }, + "duration": { + "type": "string", + "description": "Duration hint (e.g., '5', '10'). Passed through to the selected provider.", + }, + "reference_image_path": { + "type": "string", + "description": "Local path to a reference image for image_to_video. Auto-uploaded if the provider requires a URL.", + }, + "reference_image_url": { + "type": "string", + "description": "URL of a reference image for image_to_video.", + }, + "image_url": { + "type": "string", + "description": "Alias for reference_image_url (used by some providers like Kling via fal.ai).", + }, "output_path": {"type": "string"}, }, } @@ -123,6 +145,17 @@ class VideoSelector(BaseTool): if "query" in required and "query" not in adapted: adapted["query"] = adapted.get("prompt", "") + # Auto-resolve reference_image_path to a URL for providers that need it + if adapted.get("operation") == "image_to_video" and adapted.get("reference_image_path"): + tool_props = getattr(tool, "input_schema", {}).get("properties", {}) + # If the provider uses image_url (not reference_image_path), upload and convert + if "image_url" in tool_props and "image_url" not in adapted: + try: + from tools.video._shared import upload_image_fal + adapted["image_url"] = upload_image_fal(adapted["reference_image_path"]) + except Exception as e: + return ToolResult(success=False, error=f"Failed to upload reference image: {e}") + result = tool.execute(adapted) if result.success: result.data.setdefault("selected_tool", tool.name)