Add reference video input analysis workflow
This commit is contained in:
calesthio
@@ -44,6 +44,7 @@ class FrameSampler(BaseTool):
|
||||
"extract_frames_interval",
|
||||
"extract_frames_count",
|
||||
"extract_frames_timestamps",
|
||||
"extract_frames_scene_guided",
|
||||
]
|
||||
|
||||
input_schema = {
|
||||
@@ -53,7 +54,7 @@ class FrameSampler(BaseTool):
|
||||
"input_path": {"type": "string"},
|
||||
"strategy": {
|
||||
"type": "string",
|
||||
"enum": ["interval", "count", "timestamps"],
|
||||
"enum": ["interval", "count", "timestamps", "scene_guided"],
|
||||
},
|
||||
"interval_seconds": {
|
||||
"type": "number",
|
||||
@@ -70,6 +71,23 @@ class FrameSampler(BaseTool):
|
||||
"items": {"type": "number"},
|
||||
"description": "Specific timestamps in seconds (for timestamps strategy)",
|
||||
},
|
||||
"scene_boundaries": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"start_seconds": {"type": "number"},
|
||||
"end_seconds": {"type": "number"},
|
||||
},
|
||||
},
|
||||
"description": "Scene boundary list (for scene_guided strategy)",
|
||||
},
|
||||
"max_frames": {
|
||||
"type": "integer",
|
||||
"minimum": 1,
|
||||
"default": 20,
|
||||
"description": "Max frames to extract (for scene_guided strategy)",
|
||||
},
|
||||
"output_dir": {"type": "string"},
|
||||
"format": {"type": "string", "enum": ["png", "jpg"], "default": "jpg"},
|
||||
"quality": {"type": "integer", "minimum": 1, "maximum": 31, "default": 2},
|
||||
@@ -101,6 +119,8 @@ class FrameSampler(BaseTool):
|
||||
frames = self._extract_count(input_path, output_dir, fmt, quality, inputs)
|
||||
elif strategy == "timestamps":
|
||||
frames = self._extract_timestamps(input_path, output_dir, fmt, quality, inputs)
|
||||
elif strategy == "scene_guided":
|
||||
frames = self._extract_scene_guided(input_path, output_dir, fmt, quality, inputs)
|
||||
else:
|
||||
return ToolResult(success=False, error=f"Unknown strategy: {strategy}")
|
||||
except Exception as e:
|
||||
@@ -208,6 +228,54 @@ class FrameSampler(BaseTool):
|
||||
|
||||
return frames
|
||||
|
||||
def _extract_scene_guided(
|
||||
self,
|
||||
input_path: Path,
|
||||
output_dir: Path,
|
||||
fmt: str,
|
||||
quality: int,
|
||||
inputs: dict,
|
||||
) -> list[dict]:
|
||||
"""Extract keyframes guided by scene boundaries.
|
||||
|
||||
Extracts the first frame of each scene plus a midpoint frame for scenes
|
||||
longer than 3 seconds. This captures all visual transitions with a
|
||||
bounded, predictable number of frames — much better than uniform FPS.
|
||||
"""
|
||||
scene_boundaries = inputs.get("scene_boundaries", [])
|
||||
max_frames = inputs.get("max_frames", 20)
|
||||
|
||||
if not scene_boundaries:
|
||||
# No scene data — fall back to count-based
|
||||
return self._extract_count(input_path, output_dir, fmt, quality, {
|
||||
"count": min(max_frames, 15),
|
||||
})
|
||||
|
||||
# Compute timestamps: first frame + midpoint for long scenes
|
||||
timestamps = []
|
||||
for scene in scene_boundaries:
|
||||
start = scene.get("start_seconds", 0)
|
||||
end = scene.get("end_seconds", 0)
|
||||
duration = end - start
|
||||
|
||||
# First frame of scene (offset slightly to avoid black frames)
|
||||
timestamps.append(start + 0.1)
|
||||
|
||||
# Midpoint for scenes > 3 seconds
|
||||
if duration > 3.0:
|
||||
timestamps.append(start + duration / 2)
|
||||
|
||||
# Deduplicate, sort, limit
|
||||
timestamps = sorted(set(round(t, 3) for t in timestamps))
|
||||
if len(timestamps) > max_frames:
|
||||
step = len(timestamps) / max_frames
|
||||
timestamps = [timestamps[int(i * step)] for i in range(max_frames)]
|
||||
|
||||
# Extract via timestamps strategy
|
||||
return self._extract_timestamps(
|
||||
input_path, output_dir, fmt, quality, {"timestamps": timestamps}
|
||||
)
|
||||
|
||||
def _get_duration(self, input_path: Path) -> float:
|
||||
"""Get video duration in seconds via ffprobe."""
|
||||
cmd = [
|
||||
|
||||
@@ -0,0 +1,216 @@
|
||||
"""YouTube transcript fetcher tool wrapping youtube-transcript-api.
|
||||
|
||||
Extracts transcripts/captions from YouTube videos without downloading the video.
|
||||
Instant, free, no API key needed. Falls back to yt-dlp subtitle download.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
from tools.base_tool import (
|
||||
BaseTool,
|
||||
Determinism,
|
||||
ExecutionMode,
|
||||
ResourceProfile,
|
||||
ToolResult,
|
||||
ToolStability,
|
||||
ToolStatus,
|
||||
ToolTier,
|
||||
ToolRuntime,
|
||||
)
|
||||
|
||||
|
||||
class TranscriptFetcher(BaseTool):
|
||||
name = "transcript_fetcher"
|
||||
version = "0.1.0"
|
||||
tier = ToolTier.ANALYZE
|
||||
capability = "analysis"
|
||||
provider = "youtube-transcript-api"
|
||||
stability = ToolStability.PRODUCTION
|
||||
execution_mode = ExecutionMode.SYNC
|
||||
determinism = Determinism.DETERMINISTIC
|
||||
runtime = ToolRuntime.LOCAL
|
||||
|
||||
dependencies = ["python:youtube_transcript_api"]
|
||||
install_instructions = (
|
||||
"Install youtube-transcript-api: pip install youtube-transcript-api"
|
||||
)
|
||||
agent_skills = []
|
||||
|
||||
capabilities = [
|
||||
"fetch_transcript",
|
||||
"list_transcripts",
|
||||
]
|
||||
|
||||
best_for = [
|
||||
"fast YouTube transcript extraction",
|
||||
"caption-based analysis without video download",
|
||||
"getting timestamped text from YouTube videos",
|
||||
]
|
||||
|
||||
not_good_for = [
|
||||
"non-YouTube platforms (Instagram, TikTok)",
|
||||
"videos without any captions",
|
||||
"speaker diarization (use transcriber tool instead)",
|
||||
]
|
||||
|
||||
input_schema = {
|
||||
"type": "object",
|
||||
"required": ["url_or_video_id"],
|
||||
"properties": {
|
||||
"url_or_video_id": {
|
||||
"type": "string",
|
||||
"description": "YouTube URL or video ID",
|
||||
},
|
||||
"languages": {
|
||||
"type": "array",
|
||||
"items": {"type": "string"},
|
||||
"default": ["en"],
|
||||
"description": "Preferred languages in priority order",
|
||||
},
|
||||
"include_auto_generated": {
|
||||
"type": "boolean",
|
||||
"default": True,
|
||||
"description": "Whether to include auto-generated captions",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
output_schema = {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"transcript": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"text": {"type": "string"},
|
||||
"start": {"type": "number"},
|
||||
"duration": {"type": "number"},
|
||||
},
|
||||
},
|
||||
},
|
||||
"full_text": {"type": "string"},
|
||||
"language": {"type": "string"},
|
||||
"is_auto_generated": {"type": "boolean"},
|
||||
"word_count": {"type": "integer"},
|
||||
"source": {"type": "string"},
|
||||
},
|
||||
}
|
||||
|
||||
resource_profile = ResourceProfile(
|
||||
cpu_cores=1, ram_mb=256, vram_mb=0, disk_mb=10,
|
||||
network_required=True,
|
||||
)
|
||||
idempotency_key_fields = ["url_or_video_id", "languages"]
|
||||
side_effects = []
|
||||
fallback = "transcriber"
|
||||
user_visible_verification = [
|
||||
"Spot-check transcript accuracy against video audio",
|
||||
]
|
||||
|
||||
def _extract_video_id(self, url_or_id: str) -> str:
|
||||
"""Extract YouTube video ID from URL or return as-is if already an ID."""
|
||||
# Already a bare ID (11 chars, alphanumeric + - _)
|
||||
if re.match(r"^[A-Za-z0-9_-]{11}$", url_or_id):
|
||||
return url_or_id
|
||||
|
||||
# Standard YouTube URLs
|
||||
patterns = [
|
||||
r"(?:youtube\.com/watch\?.*v=|youtu\.be/|youtube\.com/embed/|youtube\.com/shorts/)([A-Za-z0-9_-]{11})",
|
||||
]
|
||||
for pattern in patterns:
|
||||
match = re.search(pattern, url_or_id)
|
||||
if match:
|
||||
return match.group(1)
|
||||
|
||||
# If nothing matched, try using the whole string as ID
|
||||
return url_or_id.strip()
|
||||
|
||||
def execute(self, inputs: dict[str, Any]) -> ToolResult:
|
||||
video_id = self._extract_video_id(inputs["url_or_video_id"])
|
||||
languages = inputs.get("languages", ["en"])
|
||||
include_auto = inputs.get("include_auto_generated", True)
|
||||
|
||||
start = time.time()
|
||||
|
||||
try:
|
||||
from youtube_transcript_api import YouTubeTranscriptApi
|
||||
|
||||
ytt = YouTubeTranscriptApi()
|
||||
|
||||
# Fetch transcript using the instance-based API (v1.0+)
|
||||
transcript_result = ytt.fetch(video_id, languages=languages)
|
||||
|
||||
# Build segments and full text from snippets
|
||||
segments = []
|
||||
full_text_parts = []
|
||||
for snippet in transcript_result.snippets:
|
||||
segments.append({
|
||||
"text": snippet.text,
|
||||
"start": round(snippet.start, 3),
|
||||
"duration": round(snippet.duration, 3),
|
||||
})
|
||||
full_text_parts.append(snippet.text)
|
||||
|
||||
full_text = " ".join(full_text_parts)
|
||||
word_count = len(full_text.split())
|
||||
|
||||
# Get auto-generated status and language from the result
|
||||
is_auto = getattr(transcript_result, "is_generated", False)
|
||||
detected_lang = getattr(transcript_result, "language", languages[0])
|
||||
# If language is an object, get the code
|
||||
if hasattr(detected_lang, "code"):
|
||||
detected_lang = detected_lang.code
|
||||
elif not isinstance(detected_lang, str):
|
||||
detected_lang = languages[0]
|
||||
|
||||
elapsed = time.time() - start
|
||||
|
||||
return ToolResult(
|
||||
success=True,
|
||||
data={
|
||||
"transcript": segments,
|
||||
"full_text": full_text,
|
||||
"language": detected_lang,
|
||||
"is_auto_generated": is_auto,
|
||||
"word_count": word_count,
|
||||
"source": "youtube_captions",
|
||||
"video_id": video_id,
|
||||
"segment_count": len(segments),
|
||||
},
|
||||
duration_seconds=round(elapsed, 2),
|
||||
)
|
||||
|
||||
except ImportError:
|
||||
return ToolResult(
|
||||
success=False,
|
||||
error="youtube-transcript-api not installed. Run: pip install youtube-transcript-api",
|
||||
)
|
||||
except Exception as e:
|
||||
elapsed = time.time() - start
|
||||
error_str = str(e)
|
||||
|
||||
# Provide helpful error messages
|
||||
if "Could not retrieve" in error_str or "TranscriptsDisabled" in error_str:
|
||||
return ToolResult(
|
||||
success=False,
|
||||
error=(
|
||||
f"No captions available for video {video_id}. "
|
||||
"This video may not have captions enabled. "
|
||||
"Fallback: download the video and use the transcriber tool "
|
||||
"with Whisper for local transcription."
|
||||
),
|
||||
data={"video_id": video_id, "fallback_suggested": "transcriber"},
|
||||
duration_seconds=round(elapsed, 2),
|
||||
)
|
||||
|
||||
return ToolResult(
|
||||
success=False,
|
||||
error=f"Transcript fetch failed: {error_str}",
|
||||
data={"video_id": video_id},
|
||||
duration_seconds=round(elapsed, 2),
|
||||
)
|
||||
@@ -0,0 +1,678 @@
|
||||
"""Video analyzer tool — comprehensive reference video analysis.
|
||||
|
||||
Orchestrates multiple analysis tools to produce a VideoAnalysisBrief from a
|
||||
video URL or local file. Runs entirely locally with zero API keys: yt-dlp for
|
||||
download, youtube-transcript-api for captions, PySceneDetect/FFmpeg for scene
|
||||
detection, FFmpeg for frame extraction, and faster-whisper for transcription.
|
||||
|
||||
The agent's own vision model analyzes extracted keyframes — this tool provides
|
||||
the structured data; the agent provides the visual interpretation.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from tools.base_tool import (
|
||||
BaseTool,
|
||||
Determinism,
|
||||
ExecutionMode,
|
||||
ResourceProfile,
|
||||
ToolResult,
|
||||
ToolStability,
|
||||
ToolStatus,
|
||||
ToolTier,
|
||||
ToolRuntime,
|
||||
)
|
||||
|
||||
|
||||
class VideoAnalyzer(BaseTool):
|
||||
name = "video_analyzer"
|
||||
version = "0.1.0"
|
||||
tier = ToolTier.ANALYZE
|
||||
capability = "analysis"
|
||||
provider = "multi"
|
||||
stability = ToolStability.BETA
|
||||
execution_mode = ExecutionMode.SYNC
|
||||
determinism = Determinism.DETERMINISTIC
|
||||
runtime = ToolRuntime.LOCAL
|
||||
|
||||
dependencies = ["cmd:ffmpeg"]
|
||||
install_instructions = (
|
||||
"Core: FFmpeg is required (https://ffmpeg.org/download.html)\n"
|
||||
"For URL downloads: pip install yt-dlp\n"
|
||||
"For YouTube transcripts: pip install youtube-transcript-api\n"
|
||||
"For local transcription: pip install faster-whisper\n"
|
||||
"For scene detection: pip install scenedetect[opencv]\n"
|
||||
"All dependencies are free and local — no API keys needed."
|
||||
)
|
||||
agent_skills = ["video-understand", "ffmpeg"]
|
||||
|
||||
capabilities = [
|
||||
"analyze_reference_video",
|
||||
"extract_structure",
|
||||
"extract_style",
|
||||
"extract_transcript",
|
||||
]
|
||||
|
||||
best_for = [
|
||||
"comprehensive video analysis",
|
||||
"reference video understanding",
|
||||
"style extraction from example video",
|
||||
"understanding video structure and pacing",
|
||||
]
|
||||
|
||||
not_good_for = [
|
||||
"editing or modifying video",
|
||||
"generating new video content",
|
||||
]
|
||||
|
||||
input_schema = {
|
||||
"type": "object",
|
||||
"required": ["source"],
|
||||
"properties": {
|
||||
"source": {
|
||||
"type": "string",
|
||||
"description": "Video file path or URL (YouTube, Shorts, Instagram, TikTok)",
|
||||
},
|
||||
"analysis_depth": {
|
||||
"type": "string",
|
||||
"enum": ["transcript_only", "standard", "deep"],
|
||||
"default": "standard",
|
||||
"description": (
|
||||
"transcript_only: transcript + metadata only. "
|
||||
"standard: + scene detection + keyframes + audio energy. "
|
||||
"deep: + intra-scene sampling + detailed style extraction."
|
||||
),
|
||||
},
|
||||
"max_keyframes": {
|
||||
"type": "integer",
|
||||
"default": 20,
|
||||
"minimum": 1,
|
||||
"maximum": 50,
|
||||
"description": "Maximum keyframes to extract",
|
||||
},
|
||||
"output_dir": {
|
||||
"type": "string",
|
||||
"description": "Directory for analysis outputs (default: auto-generated)",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
output_schema = {
|
||||
"type": "object",
|
||||
"description": "VideoAnalysisBrief artifact — see schemas/artifacts/video_analysis_brief.schema.json",
|
||||
}
|
||||
|
||||
resource_profile = ResourceProfile(
|
||||
cpu_cores=2, ram_mb=2048, vram_mb=0, disk_mb=3000,
|
||||
network_required=False, # Only needed for URL sources
|
||||
)
|
||||
idempotency_key_fields = ["source", "analysis_depth"]
|
||||
side_effects = [
|
||||
"downloads video to output_dir (if URL)",
|
||||
"writes keyframe images to output_dir/keyframes/",
|
||||
"writes analysis JSON to output_dir/video_analysis_brief.json",
|
||||
]
|
||||
fallback_tools = []
|
||||
user_visible_verification = [
|
||||
"Review keyframe images for representative coverage",
|
||||
"Check transcript accuracy against video",
|
||||
"Verify scene boundaries look correct",
|
||||
]
|
||||
|
||||
def _is_url(self, source: str) -> bool:
|
||||
"""Check if source is a URL vs local file."""
|
||||
return source.startswith(("http://", "https://", "www."))
|
||||
|
||||
def _detect_platform(self, source: str) -> str:
|
||||
"""Detect platform from URL."""
|
||||
if not self._is_url(source):
|
||||
return "local_file"
|
||||
s = source.lower()
|
||||
if "youtube.com/shorts" in s:
|
||||
return "shorts"
|
||||
if "youtube.com" in s or "youtu.be" in s:
|
||||
return "youtube"
|
||||
if "instagram.com" in s:
|
||||
return "instagram"
|
||||
if "tiktok.com" in s:
|
||||
return "tiktok"
|
||||
return "other_url"
|
||||
|
||||
def _is_youtube(self, platform: str) -> bool:
|
||||
return platform in ("youtube", "shorts")
|
||||
|
||||
def execute(self, inputs: dict[str, Any]) -> ToolResult:
|
||||
source = inputs["source"]
|
||||
depth = inputs.get("analysis_depth", "standard")
|
||||
max_keyframes = inputs.get("max_keyframes", 20)
|
||||
|
||||
# Setup output directory
|
||||
if inputs.get("output_dir"):
|
||||
output_dir = Path(inputs["output_dir"])
|
||||
else:
|
||||
output_dir = Path("projects/_analysis") / f"analysis_{int(time.time())}"
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
platform = self._detect_platform(source)
|
||||
is_url = self._is_url(source)
|
||||
start = time.time()
|
||||
|
||||
# Initialize brief structure
|
||||
brief = {
|
||||
"version": "1.0",
|
||||
"source": {
|
||||
"type": platform,
|
||||
"duration_seconds": 0,
|
||||
},
|
||||
"content_analysis": {
|
||||
"summary": "",
|
||||
"topics": [],
|
||||
"target_audience": "general",
|
||||
},
|
||||
"structure_analysis": {
|
||||
"total_scenes": 0,
|
||||
"scenes": [],
|
||||
"pacing_profile": {},
|
||||
},
|
||||
}
|
||||
|
||||
if is_url:
|
||||
brief["source"]["url"] = source
|
||||
else:
|
||||
brief["source"]["local_path"] = source
|
||||
|
||||
# Track what succeeded and what failed
|
||||
steps_completed = []
|
||||
steps_failed = []
|
||||
|
||||
# ─── STEP 1: Get metadata + download (if URL) ───
|
||||
video_path = None
|
||||
audio_path = None
|
||||
metadata = {}
|
||||
|
||||
if is_url:
|
||||
try:
|
||||
from tools.analysis.video_downloader import VideoDownloader
|
||||
downloader = VideoDownloader()
|
||||
|
||||
if depth == "transcript_only" and self._is_youtube(platform):
|
||||
# Only get metadata, skip video download
|
||||
dl_result = downloader.execute({
|
||||
"url": source,
|
||||
"output_dir": str(output_dir),
|
||||
"format": "metadata_only",
|
||||
})
|
||||
else:
|
||||
dl_result = downloader.execute({
|
||||
"url": source,
|
||||
"output_dir": str(output_dir),
|
||||
"format": "video",
|
||||
"max_resolution": "720p",
|
||||
})
|
||||
|
||||
if dl_result.success:
|
||||
metadata = dl_result.data.get("metadata", {})
|
||||
video_path = dl_result.data.get("video_path")
|
||||
audio_path = dl_result.data.get("audio_path")
|
||||
brief["source"]["title"] = metadata.get("title", "")
|
||||
brief["source"]["duration_seconds"] = metadata.get("duration", 0)
|
||||
brief["source"]["resolution"] = metadata.get("resolution", "")
|
||||
brief["source"]["platform_metadata"] = {
|
||||
"uploader": metadata.get("uploader", ""),
|
||||
"upload_date": metadata.get("upload_date", ""),
|
||||
"view_count": metadata.get("view_count", 0),
|
||||
"like_count": metadata.get("like_count", 0),
|
||||
"description": metadata.get("description", ""),
|
||||
}
|
||||
steps_completed.append("metadata")
|
||||
if video_path:
|
||||
steps_completed.append("download")
|
||||
else:
|
||||
steps_failed.append(f"download: {dl_result.error}")
|
||||
except Exception as e:
|
||||
steps_failed.append(f"download: {e}")
|
||||
else:
|
||||
# Local file
|
||||
local_path = Path(source)
|
||||
if not local_path.exists():
|
||||
return ToolResult(
|
||||
success=False,
|
||||
error=f"Local file not found: {source}",
|
||||
)
|
||||
video_path = str(local_path)
|
||||
# Get duration via ffprobe
|
||||
try:
|
||||
duration = self._get_duration(local_path)
|
||||
brief["source"]["duration_seconds"] = duration
|
||||
brief["source"]["title"] = local_path.stem
|
||||
steps_completed.append("metadata")
|
||||
except Exception as e:
|
||||
steps_failed.append(f"metadata: {e}")
|
||||
|
||||
# ─── STEP 2: Get transcript ───
|
||||
transcript_data = None
|
||||
|
||||
# Try youtube-transcript-api first (instant, for YouTube)
|
||||
if self._is_youtube(platform):
|
||||
try:
|
||||
from youtube_transcript_api import YouTubeTranscriptApi
|
||||
|
||||
from tools.analysis.transcript_fetcher import TranscriptFetcher
|
||||
fetcher = TranscriptFetcher()
|
||||
|
||||
# Auto-detect available languages instead of hardcoding "en"
|
||||
languages_to_try = ["en"]
|
||||
try:
|
||||
ytt = YouTubeTranscriptApi()
|
||||
available = ytt.list(fetcher._extract_video_id(source))
|
||||
# Build priority list: manual first, then auto-generated
|
||||
lang_codes = []
|
||||
for t in available:
|
||||
code = t.language_code if hasattr(t, "language_code") else str(t)
|
||||
if code not in lang_codes:
|
||||
lang_codes.append(code)
|
||||
if lang_codes:
|
||||
languages_to_try = lang_codes
|
||||
except Exception:
|
||||
pass # Fall through to default ["en"]
|
||||
|
||||
tf_result = fetcher.execute({
|
||||
"url_or_video_id": source,
|
||||
"languages": languages_to_try,
|
||||
"include_auto_generated": True,
|
||||
})
|
||||
if tf_result.success:
|
||||
transcript_data = tf_result.data
|
||||
brief["narration_transcript"] = {
|
||||
"full_text": transcript_data.get("full_text", ""),
|
||||
"segments": transcript_data.get("transcript", []),
|
||||
"language": transcript_data.get("language", "en"),
|
||||
"word_count": transcript_data.get("word_count", 0),
|
||||
}
|
||||
steps_completed.append("transcript_youtube")
|
||||
except Exception as e:
|
||||
steps_failed.append(f"transcript_youtube: {e}")
|
||||
|
||||
# Fallback: If transcript failed and we don't have audio yet,
|
||||
# download the video to get audio for Whisper transcription
|
||||
if transcript_data is None and audio_path is None and video_path is None and is_url:
|
||||
try:
|
||||
from tools.analysis.video_downloader import VideoDownloader
|
||||
downloader = VideoDownloader()
|
||||
dl_result = downloader.execute({
|
||||
"url": source,
|
||||
"output_dir": str(output_dir),
|
||||
"format": "video",
|
||||
"max_resolution": "720p",
|
||||
})
|
||||
if dl_result.success:
|
||||
video_path = dl_result.data.get("video_path")
|
||||
audio_path = dl_result.data.get("audio_path")
|
||||
if video_path:
|
||||
steps_completed.append("download_for_whisper")
|
||||
# Also update metadata if we didn't have it
|
||||
if not metadata:
|
||||
metadata = dl_result.data.get("metadata", {})
|
||||
brief["source"]["title"] = metadata.get("title", "")
|
||||
brief["source"]["duration_seconds"] = metadata.get("duration", 0)
|
||||
except Exception as e:
|
||||
steps_failed.append(f"download_for_whisper: {e}")
|
||||
|
||||
# Fallback: Whisper transcription on audio
|
||||
if transcript_data is None and audio_path:
|
||||
try:
|
||||
from tools.analysis.transcriber import Transcriber
|
||||
transcriber = Transcriber()
|
||||
# Let Whisper auto-detect language instead of assuming English
|
||||
tr_inputs = {
|
||||
"input_path": audio_path,
|
||||
"model_size": "base",
|
||||
"output_dir": str(output_dir),
|
||||
}
|
||||
# Only set language if we know it from transcript attempt
|
||||
detected_lang = brief.get("narration_transcript", {}).get("language")
|
||||
if detected_lang and detected_lang != "en":
|
||||
tr_inputs["language"] = detected_lang
|
||||
# else: let Whisper auto-detect
|
||||
|
||||
tr_result = transcriber.execute(tr_inputs)
|
||||
if tr_result.success:
|
||||
segments = tr_result.data.get("segments", [])
|
||||
full_text = " ".join(s.get("text", "") for s in segments)
|
||||
brief["narration_transcript"] = {
|
||||
"full_text": full_text,
|
||||
"segments": [
|
||||
{
|
||||
"start": s.get("start", 0),
|
||||
"end": s.get("end", 0),
|
||||
"text": s.get("text", ""),
|
||||
}
|
||||
for s in segments
|
||||
],
|
||||
"language": tr_result.data.get("language", "en"),
|
||||
"word_count": len(full_text.split()),
|
||||
}
|
||||
transcript_data = brief["narration_transcript"]
|
||||
steps_completed.append("transcript_whisper")
|
||||
except Exception as e:
|
||||
steps_failed.append(f"transcript_whisper: {e}")
|
||||
|
||||
# For transcript_only depth, we're done
|
||||
if depth == "transcript_only":
|
||||
brief["_analysis_meta"] = {
|
||||
"depth": depth,
|
||||
"steps_completed": steps_completed,
|
||||
"steps_failed": steps_failed,
|
||||
"duration_seconds": round(time.time() - start, 2),
|
||||
}
|
||||
self._save_brief(brief, output_dir)
|
||||
return ToolResult(
|
||||
success=True,
|
||||
data=brief,
|
||||
artifacts=[str(output_dir / "video_analysis_brief.json")],
|
||||
duration_seconds=round(time.time() - start, 2),
|
||||
)
|
||||
|
||||
# ─── STEP 3: Scene detection (standard + deep) ───
|
||||
scenes = []
|
||||
if video_path:
|
||||
try:
|
||||
from tools.analysis.scene_detect import SceneDetect
|
||||
detector = SceneDetect()
|
||||
sd_result = detector.execute({
|
||||
"input_path": video_path,
|
||||
"method": "content",
|
||||
"min_scene_length_seconds": 0.5,
|
||||
"output_path": str(output_dir / "scenes.json"),
|
||||
})
|
||||
if sd_result.success:
|
||||
scenes = sd_result.data.get("scenes", [])
|
||||
steps_completed.append("scene_detect")
|
||||
except Exception as e:
|
||||
steps_failed.append(f"scene_detect: {e}")
|
||||
|
||||
# Build scene list for the brief
|
||||
if scenes:
|
||||
brief["structure_analysis"]["total_scenes"] = len(scenes)
|
||||
brief_scenes = []
|
||||
for scene in scenes:
|
||||
brief_scenes.append({
|
||||
"scene_index": scene.get("index", scene.get("scene_index", 0)),
|
||||
"start_time": scene.get("start_seconds", 0),
|
||||
"end_time": scene.get("end_seconds", 0),
|
||||
"description": "", # Agent fills this via vision
|
||||
"visual_type": "other", # Agent classifies via vision
|
||||
"energy_level": "medium",
|
||||
})
|
||||
brief["structure_analysis"]["scenes"] = brief_scenes
|
||||
|
||||
# Compute pacing profile
|
||||
durations = [
|
||||
s.get("end_seconds", 0) - s.get("start_seconds", 0)
|
||||
for s in scenes
|
||||
]
|
||||
total_duration = brief["source"]["duration_seconds"] or sum(durations)
|
||||
if durations:
|
||||
brief["structure_analysis"]["pacing_profile"] = {
|
||||
"avg_scene_duration_seconds": round(sum(durations) / len(durations), 2),
|
||||
"shortest_scene_seconds": round(min(durations), 2),
|
||||
"longest_scene_seconds": round(max(durations), 2),
|
||||
"cuts_per_minute": round(len(durations) / (total_duration / 60), 2) if total_duration > 0 else 0,
|
||||
"pacing_style": self._classify_pacing(durations),
|
||||
}
|
||||
|
||||
# ─── STEP 4: Keyframe extraction (scene-guided) ───
|
||||
keyframes = []
|
||||
keyframe_dir = output_dir / "keyframes"
|
||||
if video_path and scenes:
|
||||
try:
|
||||
# Extract keyframes at scene boundaries + midpoints
|
||||
timestamps = self._compute_keyframe_timestamps(scenes, max_keyframes, depth)
|
||||
|
||||
from tools.analysis.frame_sampler import FrameSampler
|
||||
sampler = FrameSampler()
|
||||
fs_result = sampler.execute({
|
||||
"input_path": video_path,
|
||||
"strategy": "timestamps",
|
||||
"timestamps": timestamps,
|
||||
"output_dir": str(keyframe_dir),
|
||||
"format": "jpg",
|
||||
"quality": 2,
|
||||
})
|
||||
if fs_result.success:
|
||||
for frame in fs_result.data.get("frames", []):
|
||||
# Map each frame to its scene
|
||||
scene_idx = self._timestamp_to_scene(
|
||||
frame["timestamp_seconds"], scenes
|
||||
)
|
||||
keyframes.append({
|
||||
"timestamp": frame["timestamp_seconds"],
|
||||
"scene_index": scene_idx,
|
||||
"path": frame["path"],
|
||||
"description": "", # Agent fills via vision
|
||||
})
|
||||
steps_completed.append("keyframes")
|
||||
except Exception as e:
|
||||
steps_failed.append(f"keyframes: {e}")
|
||||
elif video_path and not scenes:
|
||||
# No scene detection — fall back to count-based extraction
|
||||
try:
|
||||
from tools.analysis.frame_sampler import FrameSampler
|
||||
sampler = FrameSampler()
|
||||
fs_result = sampler.execute({
|
||||
"input_path": video_path,
|
||||
"strategy": "count",
|
||||
"count": min(max_keyframes, 15),
|
||||
"output_dir": str(keyframe_dir),
|
||||
"format": "jpg",
|
||||
"quality": 2,
|
||||
})
|
||||
if fs_result.success:
|
||||
for frame in fs_result.data.get("frames", []):
|
||||
keyframes.append({
|
||||
"timestamp": frame["timestamp_seconds"],
|
||||
"scene_index": 0,
|
||||
"path": frame["path"],
|
||||
"description": "",
|
||||
})
|
||||
steps_completed.append("keyframes_uniform")
|
||||
except Exception as e:
|
||||
steps_failed.append(f"keyframes_uniform: {e}")
|
||||
|
||||
brief["keyframes"] = keyframes
|
||||
|
||||
# ─── STEP 5: Audio energy analysis ───
|
||||
if audio_path or video_path:
|
||||
audio_source = audio_path or video_path
|
||||
try:
|
||||
from tools.analysis.audio_energy import AudioEnergy
|
||||
energy = AudioEnergy()
|
||||
ae_result = energy.execute({
|
||||
"input_path": audio_source,
|
||||
"video_duration_seconds": brief["source"]["duration_seconds"],
|
||||
})
|
||||
if ae_result.success:
|
||||
# Store energy profile summary in style_profile
|
||||
if "style_profile" not in brief:
|
||||
brief["style_profile"] = {}
|
||||
brief["style_profile"]["audio_energy_profile"] = {
|
||||
"recommended_offset": ae_result.data.get("recommended_offset_seconds", 0),
|
||||
"has_energy_data": True,
|
||||
}
|
||||
steps_completed.append("audio_energy")
|
||||
except Exception as e:
|
||||
steps_failed.append(f"audio_energy: {e}")
|
||||
|
||||
# ─── STEP 6: Build replication guidance ───
|
||||
brief["replication_guidance"] = {
|
||||
"suggested_pipeline": self._suggest_pipeline(brief),
|
||||
"suggested_playbook": "flat-motion-graphics",
|
||||
"key_elements_to_replicate": [], # Agent fills via analysis
|
||||
"elements_requiring_custom_work": [],
|
||||
"estimated_complexity": self._estimate_complexity(brief),
|
||||
"motion_required": self._needs_motion(brief),
|
||||
"creative_differentiation_seeds": [], # Agent fills
|
||||
}
|
||||
|
||||
# ─── STEP 7: Initialize style_profile ───
|
||||
if "style_profile" not in brief:
|
||||
brief["style_profile"] = {}
|
||||
|
||||
# Narration style from transcript
|
||||
if transcript_data:
|
||||
duration = brief["source"]["duration_seconds"]
|
||||
wc = transcript_data.get("word_count", 0) if isinstance(transcript_data, dict) else brief.get("narration_transcript", {}).get("word_count", 0)
|
||||
wpm = round(wc / (duration / 60), 1) if duration > 0 else 0
|
||||
brief["style_profile"]["narration_style"] = {
|
||||
"has_narration": wc > 20,
|
||||
"speaker_count": 1, # Agent refines via analysis
|
||||
"delivery_style": "", # Agent fills
|
||||
"words_per_minute": wpm,
|
||||
}
|
||||
|
||||
# Initialize remaining style fields for agent to fill
|
||||
brief["style_profile"].setdefault("color_palette", {
|
||||
"primary_colors": [],
|
||||
"accent_colors": [],
|
||||
"overall_mood": "",
|
||||
})
|
||||
brief["style_profile"].setdefault("typography_observed", "")
|
||||
brief["style_profile"].setdefault("transition_types", [])
|
||||
brief["style_profile"].setdefault("music_style", "")
|
||||
brief["style_profile"].setdefault("subtitle_style", "")
|
||||
brief["style_profile"].setdefault("production_quality", "prosumer")
|
||||
brief["style_profile"].setdefault("closest_playbook", "")
|
||||
brief["style_profile"].setdefault("playbook_delta", "")
|
||||
|
||||
# ─── Finalize ───
|
||||
brief["_analysis_meta"] = {
|
||||
"depth": depth,
|
||||
"steps_completed": steps_completed,
|
||||
"steps_failed": steps_failed,
|
||||
"keyframe_count": len(keyframes),
|
||||
"scene_count": len(scenes),
|
||||
"has_transcript": transcript_data is not None,
|
||||
"duration_seconds": round(time.time() - start, 2),
|
||||
}
|
||||
|
||||
self._save_brief(brief, output_dir)
|
||||
|
||||
elapsed = time.time() - start
|
||||
artifacts = [str(output_dir / "video_analysis_brief.json")]
|
||||
if keyframe_dir.exists():
|
||||
artifacts.append(str(keyframe_dir))
|
||||
|
||||
return ToolResult(
|
||||
success=True,
|
||||
data=brief,
|
||||
artifacts=artifacts,
|
||||
duration_seconds=round(elapsed, 2),
|
||||
)
|
||||
|
||||
# ─── Helpers ───
|
||||
|
||||
def _get_duration(self, video_path: Path) -> float:
|
||||
"""Get video duration via ffprobe."""
|
||||
cmd = [
|
||||
"ffprobe", "-v", "quiet",
|
||||
"-show_entries", "format=duration",
|
||||
"-of", "json",
|
||||
str(video_path),
|
||||
]
|
||||
result = self.run_command(cmd)
|
||||
data = json.loads(result.stdout)
|
||||
return float(data.get("format", {}).get("duration", 0))
|
||||
|
||||
def _compute_keyframe_timestamps(
|
||||
self, scenes: list[dict], max_frames: int, depth: str
|
||||
) -> list[float]:
|
||||
"""Compute optimal keyframe timestamps from scene boundaries."""
|
||||
timestamps = []
|
||||
|
||||
for scene in scenes:
|
||||
start = scene.get("start_seconds", 0)
|
||||
end = scene.get("end_seconds", 0)
|
||||
duration = end - start
|
||||
|
||||
# First frame of each scene
|
||||
timestamps.append(start + 0.1)
|
||||
|
||||
# Midpoint for scenes > 3 seconds
|
||||
if duration > 3.0:
|
||||
timestamps.append(start + duration / 2)
|
||||
|
||||
# For deep analysis, add more intra-scene samples
|
||||
if depth == "deep" and duration > 6.0:
|
||||
timestamps.append(start + duration * 0.25)
|
||||
timestamps.append(start + duration * 0.75)
|
||||
|
||||
# Deduplicate, sort, and limit
|
||||
timestamps = sorted(set(round(t, 3) for t in timestamps))
|
||||
if len(timestamps) > max_frames:
|
||||
# Uniform subsample to max_frames
|
||||
step = len(timestamps) / max_frames
|
||||
timestamps = [timestamps[int(i * step)] for i in range(max_frames)]
|
||||
|
||||
return timestamps
|
||||
|
||||
def _timestamp_to_scene(self, ts: float, scenes: list[dict]) -> int:
|
||||
"""Map a timestamp to its scene index."""
|
||||
for scene in scenes:
|
||||
start = scene.get("start_seconds", 0)
|
||||
end = scene.get("end_seconds", 0)
|
||||
if start <= ts <= end:
|
||||
return scene.get("index", scene.get("scene_index", 0))
|
||||
return 0
|
||||
|
||||
def _classify_pacing(self, durations: list[float]) -> str:
|
||||
"""Classify pacing style from scene durations."""
|
||||
if not durations:
|
||||
return "variable"
|
||||
avg = sum(durations) / len(durations)
|
||||
if avg > 10:
|
||||
return "slow_contemplative"
|
||||
if avg > 5:
|
||||
return "steady_educational"
|
||||
if avg > 2:
|
||||
return "dynamic_social"
|
||||
return "rapid_fire"
|
||||
|
||||
def _suggest_pipeline(self, brief: dict) -> str:
|
||||
"""Suggest the best pipeline based on content analysis."""
|
||||
platform = brief["source"]["type"]
|
||||
pacing = brief["structure_analysis"].get("pacing_profile", {}).get("pacing_style", "")
|
||||
|
||||
if platform in ("shorts", "tiktok", "instagram"):
|
||||
return "animation" # Short-form → animation pipeline works well
|
||||
if pacing in ("slow_contemplative",):
|
||||
return "cinematic"
|
||||
return "animated-explainer"
|
||||
|
||||
def _estimate_complexity(self, brief: dict) -> str:
|
||||
"""Estimate how complex it would be to recreate this style."""
|
||||
scenes = brief["structure_analysis"]["total_scenes"]
|
||||
duration = brief["source"]["duration_seconds"]
|
||||
|
||||
if duration > 300 or scenes > 30:
|
||||
return "complex"
|
||||
if duration > 120 or scenes > 15:
|
||||
return "moderate"
|
||||
return "simple"
|
||||
|
||||
def _needs_motion(self, brief: dict) -> bool:
|
||||
"""Determine if motion (video gen or Remotion) is required."""
|
||||
pacing = brief["structure_analysis"].get("pacing_profile", {}).get("pacing_style", "")
|
||||
return pacing in ("dynamic_social", "rapid_fire")
|
||||
|
||||
def _save_brief(self, brief: dict, output_dir: Path) -> None:
|
||||
"""Save the VideoAnalysisBrief to disk."""
|
||||
out_path = output_dir / "video_analysis_brief.json"
|
||||
# Remove non-serializable items
|
||||
clean_brief = {k: v for k, v in brief.items()}
|
||||
with open(out_path, "w", encoding="utf-8") as f:
|
||||
json.dump(clean_brief, f, indent=2, default=str)
|
||||
@@ -0,0 +1,355 @@
|
||||
"""Video downloader tool wrapping yt-dlp.
|
||||
|
||||
Downloads video, audio, or subtitles from YouTube, Shorts, Instagram Reels,
|
||||
TikTok, and 1000+ other sites. Designed for reference video analysis — downloads
|
||||
at analysis quality (720p), not production quality.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from tools.base_tool import (
|
||||
BaseTool,
|
||||
Determinism,
|
||||
ExecutionMode,
|
||||
ResourceProfile,
|
||||
ToolResult,
|
||||
ToolStability,
|
||||
ToolStatus,
|
||||
ToolTier,
|
||||
ToolRuntime,
|
||||
)
|
||||
|
||||
|
||||
class VideoDownloader(BaseTool):
|
||||
name = "video_downloader"
|
||||
version = "0.1.0"
|
||||
tier = ToolTier.SOURCE
|
||||
capability = "source_ingest"
|
||||
provider = "yt-dlp"
|
||||
stability = ToolStability.PRODUCTION
|
||||
execution_mode = ExecutionMode.SYNC
|
||||
determinism = Determinism.DETERMINISTIC
|
||||
runtime = ToolRuntime.LOCAL
|
||||
|
||||
dependencies = ["python:yt_dlp"]
|
||||
install_instructions = (
|
||||
"Install yt-dlp: pip install yt-dlp\n"
|
||||
"For YouTube support, also install Deno (JS runtime): "
|
||||
"https://deno.land/#installation\n"
|
||||
"Without Deno, YouTube downloads may fail but other platforms still work."
|
||||
)
|
||||
agent_skills = ["video-download"]
|
||||
|
||||
capabilities = [
|
||||
"download_video",
|
||||
"download_audio",
|
||||
"download_subtitles",
|
||||
"extract_metadata",
|
||||
]
|
||||
|
||||
best_for = [
|
||||
"downloading reference video from URL",
|
||||
"extracting audio from online video",
|
||||
"downloading subtitles from YouTube",
|
||||
"getting video metadata without downloading",
|
||||
]
|
||||
|
||||
not_good_for = [
|
||||
"downloading entire playlists",
|
||||
"downloading DRM-protected content",
|
||||
]
|
||||
|
||||
input_schema = {
|
||||
"type": "object",
|
||||
"required": ["url", "output_dir"],
|
||||
"properties": {
|
||||
"url": {"type": "string", "description": "Video URL to download"},
|
||||
"output_dir": {"type": "string", "description": "Directory for downloaded files"},
|
||||
"format": {
|
||||
"type": "string",
|
||||
"enum": ["video", "audio_only", "subtitles_only", "metadata_only"],
|
||||
"default": "video",
|
||||
"description": "What to download",
|
||||
},
|
||||
"max_resolution": {
|
||||
"type": "string",
|
||||
"enum": ["360p", "480p", "720p", "1080p"],
|
||||
"default": "720p",
|
||||
"description": "Maximum video resolution (for analysis, 720p is sufficient)",
|
||||
},
|
||||
"max_duration_seconds": {
|
||||
"type": "integer",
|
||||
"default": 600,
|
||||
"description": "Reject videos longer than this (safety limit)",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
output_schema = {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"video_path": {"type": ["string", "null"]},
|
||||
"audio_path": {"type": ["string", "null"]},
|
||||
"subtitle_path": {"type": ["string", "null"]},
|
||||
"metadata": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"title": {"type": "string"},
|
||||
"duration": {"type": "number"},
|
||||
"uploader": {"type": "string"},
|
||||
"upload_date": {"type": "string"},
|
||||
"description": {"type": "string"},
|
||||
"view_count": {"type": "integer"},
|
||||
"like_count": {"type": "integer"},
|
||||
},
|
||||
},
|
||||
"platform": {"type": "string"},
|
||||
},
|
||||
}
|
||||
|
||||
resource_profile = ResourceProfile(
|
||||
cpu_cores=1, ram_mb=512, vram_mb=0, disk_mb=2000,
|
||||
network_required=True,
|
||||
)
|
||||
idempotency_key_fields = ["url", "format", "max_resolution"]
|
||||
side_effects = ["downloads media files to output_dir"]
|
||||
resume_support_value = "from_start"
|
||||
user_visible_verification = [
|
||||
"Check downloaded file plays correctly",
|
||||
"Verify resolution matches requested max",
|
||||
]
|
||||
|
||||
# --- Resolution mapping ---
|
||||
_RES_MAP = {
|
||||
"360p": 360,
|
||||
"480p": 480,
|
||||
"720p": 720,
|
||||
"1080p": 1080,
|
||||
}
|
||||
|
||||
def _detect_platform(self, url: str) -> str:
|
||||
"""Detect platform from URL."""
|
||||
url_lower = url.lower()
|
||||
if "youtube.com/shorts" in url_lower or "youtu.be" in url_lower and "/shorts" in url_lower:
|
||||
return "shorts"
|
||||
if "youtube.com" in url_lower or "youtu.be" in url_lower:
|
||||
return "youtube"
|
||||
if "instagram.com" in url_lower:
|
||||
return "instagram"
|
||||
if "tiktok.com" in url_lower:
|
||||
return "tiktok"
|
||||
if "vimeo.com" in url_lower:
|
||||
return "vimeo"
|
||||
if "twitter.com" in url_lower or "x.com" in url_lower:
|
||||
return "twitter"
|
||||
return "other_url"
|
||||
|
||||
def _extract_metadata(self, url: str) -> dict:
|
||||
"""Extract metadata without downloading."""
|
||||
import yt_dlp
|
||||
|
||||
ydl_opts = {
|
||||
"quiet": True,
|
||||
"no_warnings": True,
|
||||
"skip_download": True,
|
||||
}
|
||||
try:
|
||||
with yt_dlp.YoutubeDL(ydl_opts) as ydl:
|
||||
info = ydl.extract_info(url, download=False)
|
||||
if info is None:
|
||||
return {"error": "No info extracted", "title": "", "duration": 0}
|
||||
return {
|
||||
"title": info.get("title", ""),
|
||||
"duration": info.get("duration", 0),
|
||||
"uploader": info.get("uploader", info.get("channel", "")),
|
||||
"upload_date": info.get("upload_date", ""),
|
||||
"description": (info.get("description", "") or "")[:500],
|
||||
"view_count": info.get("view_count", 0),
|
||||
"like_count": info.get("like_count", 0),
|
||||
"resolution": f"{info.get('width', 0)}x{info.get('height', 0)}",
|
||||
"fps": info.get("fps", 0),
|
||||
}
|
||||
except Exception as e:
|
||||
return {"error": str(e), "title": "", "duration": 0}
|
||||
|
||||
def execute(self, inputs: dict[str, Any]) -> ToolResult:
|
||||
url = inputs["url"]
|
||||
output_dir = Path(inputs["output_dir"])
|
||||
dl_format = inputs.get("format", "video")
|
||||
max_res = inputs.get("max_resolution", "720p")
|
||||
max_duration = inputs.get("max_duration_seconds", 600)
|
||||
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
platform = self._detect_platform(url)
|
||||
start = time.time()
|
||||
|
||||
# Step 1: Always get metadata first
|
||||
metadata = self._extract_metadata(url)
|
||||
|
||||
# Check duration limit
|
||||
duration = metadata.get("duration", 0)
|
||||
if duration and duration > max_duration:
|
||||
return ToolResult(
|
||||
success=False,
|
||||
error=(
|
||||
f"Video is {duration}s, exceeds max_duration_seconds={max_duration}. "
|
||||
f"Increase the limit or use a shorter video."
|
||||
),
|
||||
data={"metadata": metadata, "platform": platform},
|
||||
)
|
||||
|
||||
if dl_format == "metadata_only":
|
||||
return ToolResult(
|
||||
success=True,
|
||||
data={
|
||||
"video_path": None,
|
||||
"audio_path": None,
|
||||
"subtitle_path": None,
|
||||
"metadata": metadata,
|
||||
"platform": platform,
|
||||
},
|
||||
duration_seconds=round(time.time() - start, 2),
|
||||
)
|
||||
|
||||
video_path = None
|
||||
audio_path = None
|
||||
subtitle_path = None
|
||||
|
||||
try:
|
||||
if dl_format == "video":
|
||||
video_path, audio_path = self._download_video(
|
||||
url, output_dir, max_res
|
||||
)
|
||||
elif dl_format == "audio_only":
|
||||
audio_path = self._download_audio(url, output_dir)
|
||||
elif dl_format == "subtitles_only":
|
||||
subtitle_path = self._download_subtitles(url, output_dir)
|
||||
except Exception as e:
|
||||
elapsed = time.time() - start
|
||||
return ToolResult(
|
||||
success=False,
|
||||
error=f"Download failed: {e}",
|
||||
data={"metadata": metadata, "platform": platform},
|
||||
duration_seconds=round(elapsed, 2),
|
||||
)
|
||||
|
||||
elapsed = time.time() - start
|
||||
artifacts = [p for p in [video_path, audio_path, subtitle_path] if p]
|
||||
|
||||
return ToolResult(
|
||||
success=True,
|
||||
data={
|
||||
"video_path": video_path,
|
||||
"audio_path": audio_path,
|
||||
"subtitle_path": subtitle_path,
|
||||
"metadata": metadata,
|
||||
"platform": platform,
|
||||
},
|
||||
artifacts=artifacts,
|
||||
duration_seconds=round(elapsed, 2),
|
||||
)
|
||||
|
||||
def _download_video(
|
||||
self, url: str, output_dir: Path, max_res: str
|
||||
) -> tuple[str | None, str | None]:
|
||||
"""Download video + extract audio track."""
|
||||
import yt_dlp
|
||||
|
||||
height = self._RES_MAP.get(max_res, 720)
|
||||
video_out = str(output_dir / "reference_video.%(ext)s")
|
||||
|
||||
ydl_opts = {
|
||||
"format": f"bestvideo[height<={height}]+bestaudio/best[height<={height}]/best",
|
||||
"merge_output_format": "mp4",
|
||||
"outtmpl": video_out,
|
||||
"noplaylist": True,
|
||||
"quiet": True,
|
||||
"no_warnings": True,
|
||||
}
|
||||
with yt_dlp.YoutubeDL(ydl_opts) as ydl:
|
||||
ydl.download([url])
|
||||
|
||||
# Find the downloaded video file
|
||||
video_path = self._find_downloaded(output_dir, "reference_video", ["mp4", "mkv", "webm"])
|
||||
|
||||
# Extract audio separately for transcription
|
||||
audio_path = None
|
||||
if video_path:
|
||||
audio_out = output_dir / "reference_audio.wav"
|
||||
try:
|
||||
audio_cmd = [
|
||||
"ffmpeg", "-y",
|
||||
"-i", video_path,
|
||||
"-vn",
|
||||
"-acodec", "pcm_s16le",
|
||||
"-ar", "16000",
|
||||
"-ac", "1",
|
||||
str(audio_out),
|
||||
]
|
||||
self.run_command(audio_cmd, timeout=120)
|
||||
if audio_out.exists():
|
||||
audio_path = str(audio_out)
|
||||
except Exception:
|
||||
pass # Audio extraction is optional
|
||||
|
||||
return video_path, audio_path
|
||||
|
||||
def _download_audio(self, url: str, output_dir: Path) -> str | None:
|
||||
"""Download audio only."""
|
||||
import yt_dlp
|
||||
|
||||
audio_out = str(output_dir / "reference_audio.%(ext)s")
|
||||
ydl_opts = {
|
||||
"format": "bestaudio/best",
|
||||
"postprocessors": [{
|
||||
"key": "FFmpegExtractAudio",
|
||||
"preferredcodec": "wav",
|
||||
"preferredquality": "0",
|
||||
}],
|
||||
"outtmpl": audio_out,
|
||||
"noplaylist": True,
|
||||
"quiet": True,
|
||||
"no_warnings": True,
|
||||
}
|
||||
with yt_dlp.YoutubeDL(ydl_opts) as ydl:
|
||||
ydl.download([url])
|
||||
return self._find_downloaded(output_dir, "reference_audio", ["wav", "mp3", "m4a", "opus"])
|
||||
|
||||
def _download_subtitles(self, url: str, output_dir: Path) -> str | None:
|
||||
"""Download subtitles only."""
|
||||
import yt_dlp
|
||||
|
||||
sub_out = str(output_dir / "reference_subs.%(ext)s")
|
||||
ydl_opts = {
|
||||
"writesubtitles": True,
|
||||
"writeautomaticsub": True,
|
||||
"subtitleslangs": ["en"],
|
||||
"subtitlesformat": "srt",
|
||||
"skip_download": True,
|
||||
"outtmpl": sub_out,
|
||||
"noplaylist": True,
|
||||
"quiet": True,
|
||||
"no_warnings": True,
|
||||
}
|
||||
try:
|
||||
with yt_dlp.YoutubeDL(ydl_opts) as ydl:
|
||||
ydl.download([url])
|
||||
except Exception:
|
||||
pass
|
||||
return self._find_downloaded(output_dir, "reference_subs", ["srt", "vtt", "ass"])
|
||||
|
||||
def _find_downloaded(
|
||||
self, output_dir: Path, prefix: str, extensions: list[str]
|
||||
) -> str | None:
|
||||
"""Find a downloaded file by prefix and possible extensions."""
|
||||
for ext in extensions:
|
||||
candidates = list(output_dir.glob(f"{prefix}*.{ext}"))
|
||||
if candidates:
|
||||
return str(candidates[0])
|
||||
return None
|
||||
Reference in New Issue
Block a user