772 lines
26 KiB
Python
772 lines
26 KiB
Python
import hashlib
|
|
import json
|
|
import logging
|
|
import os
|
|
import time
|
|
import uuid
|
|
from collections.abc import AsyncGenerator
|
|
from contextlib import asynccontextmanager
|
|
from datetime import datetime, timezone
|
|
|
|
import httpx
|
|
from fastapi import FastAPI, File, Form, HTTPException, UploadFile
|
|
from fastapi.responses import JSONResponse
|
|
|
|
from .buster_client import (
|
|
build_frame_evidence,
|
|
call_vllm_chat,
|
|
frame_to_data_url_b64jpeg,
|
|
parse_verdict_and_explanation,
|
|
)
|
|
from .runtime_config import RuntimeConfigClient
|
|
from .schemas import AnalyzeResponse, ChunkResult, SemanticAnalysisResponse, SemanticMeta, Usage
|
|
from .settings import settings
|
|
from .video_sampling import sample_frames_chunked, sample_frames_uniform
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
# Runtime config (reachable from handlers via app.state.runtime_config)
|
|
runtime_config = RuntimeConfigClient(
|
|
dashboard_url=settings.dashboard_url,
|
|
live_log_logger_name="video_analysis",
|
|
live_log_key="video.log.level",
|
|
)
|
|
|
|
|
|
@asynccontextmanager
|
|
async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
|
|
"""App lifespan: start runtime config polling."""
|
|
logger.info("Starting Video Analysis API")
|
|
await runtime_config.start()
|
|
app.state.runtime_config = runtime_config
|
|
yield
|
|
logger.info("Shutting down Video Analysis API")
|
|
await runtime_config.stop()
|
|
|
|
|
|
app = FastAPI(
|
|
title="Video Analysis API",
|
|
description="Deepfake detection and semantic video analysis",
|
|
version="0.1.0",
|
|
servers=[{"url": settings.external_url, "description": "Video Analysis API"}],
|
|
lifespan=lifespan,
|
|
)
|
|
|
|
|
|
|
|
# ---------------------------------------------------------------- observability
|
|
|
|
|
|
# Prometheus /metrics + OTel tracing (no-op if deps missing or OTEL endpoint unset)
|
|
|
|
|
|
try:
|
|
|
|
|
|
from prometheus_fastapi_instrumentator import Instrumentator as _Inst # type: ignore
|
|
|
|
|
|
_Inst(should_group_status_codes=True).instrument(app).expose(app, endpoint="/metrics", include_in_schema=False)
|
|
|
|
|
|
except ImportError:
|
|
|
|
|
|
pass
|
|
|
|
|
|
import os as _os # noqa: E402
|
|
|
|
|
|
_otel_ep = _os.environ.get("OTEL_EXPORTER_OTLP_ENDPOINT")
|
|
|
|
|
|
if _otel_ep:
|
|
|
|
|
|
try:
|
|
|
|
|
|
from opentelemetry import trace as _trace # type: ignore
|
|
|
|
|
|
from opentelemetry.sdk.resources import Resource as _R # type: ignore
|
|
|
|
|
|
from opentelemetry.sdk.trace import TracerProvider as _TP # type: ignore
|
|
|
|
|
|
from opentelemetry.sdk.trace.export import BatchSpanProcessor as _BSP # type: ignore
|
|
|
|
|
|
from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter as _Exp # type: ignore
|
|
|
|
|
|
from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor as _FInst # type: ignore
|
|
|
|
|
|
_provider = _TP(resource=_R.create({"service.name": _os.environ.get("OTEL_SERVICE_NAME", "didiAI-video-api")}))
|
|
|
|
|
|
_provider.add_span_processor(_BSP(_Exp(endpoint=_otel_ep, insecure=True)))
|
|
|
|
|
|
_trace.set_tracer_provider(_provider)
|
|
|
|
|
|
_FInst.instrument_app(app)
|
|
|
|
|
|
print(f"[otel] didiAI-video-api instrumented -> {_otel_ep}")
|
|
|
|
|
|
except ImportError as _e:
|
|
|
|
|
|
print(f"[otel] skip: {_e}")
|
|
|
|
|
|
|
|
def safe_mkdir(path: str) -> None:
|
|
os.makedirs(path, exist_ok=True)
|
|
|
|
|
|
def write_json(path: str, obj) -> None:
|
|
with open(path, "w", encoding="utf-8") as f:
|
|
json.dump(obj, f, ensure_ascii=False, indent=2)
|
|
|
|
|
|
def sha256_file(path: str, chunk_size: int = 8 * 1024 * 1024) -> str:
|
|
h = hashlib.sha256()
|
|
with open(path, "rb") as f:
|
|
while True:
|
|
b = f.read(chunk_size)
|
|
if not b:
|
|
break
|
|
h.update(b)
|
|
return h.hexdigest()
|
|
|
|
|
|
@app.get("/health")
|
|
def health():
|
|
return {"status": "ok"}
|
|
|
|
|
|
@app.post("/analyze/video", response_model=AnalyzeResponse)
|
|
def analyze_video(file: UploadFile = File(...)):
|
|
# Defensive (Settings already requires it)
|
|
if not settings.vllm_base_url:
|
|
raise HTTPException(status_code=500, detail="VIDEO_ANALYSIS_VLLM_BASE_URL is not set")
|
|
|
|
request_id = str(uuid.uuid4())
|
|
started_at = datetime.now(timezone.utc).isoformat()
|
|
|
|
# Make run_dir absolute under runs_dir (inside container: /app/runs/<uuid>)
|
|
run_dir = os.path.join(settings.runs_dir, request_id)
|
|
safe_mkdir(run_dir)
|
|
|
|
# Save upload to disk for reproducibility
|
|
video_path = os.path.join(run_dir, file.filename or "upload.mp4")
|
|
t0 = time.time()
|
|
with open(video_path, "wb") as out:
|
|
out.write(file.file.read())
|
|
save_time_s = round(time.time() - t0, 4)
|
|
|
|
video_hash = sha256_file(video_path)
|
|
prompt = settings.analysis_prompt
|
|
|
|
# Sampling
|
|
frames, meta = sample_frames_uniform(video_path, num_frames=settings.frames)
|
|
|
|
# Encode frames
|
|
t_enc0 = time.time()
|
|
data_urls = [
|
|
frame_to_data_url_b64jpeg(f, max_side=settings.max_side, jpeg_quality=settings.jpeg_quality)
|
|
for f in frames
|
|
]
|
|
encode_time_s = round(time.time() - t_enc0, 4)
|
|
|
|
request_record = {
|
|
"request_id": request_id,
|
|
"started_at_utc": started_at,
|
|
"video": {
|
|
"path": video_path,
|
|
"sha256": video_hash,
|
|
"fps": meta.get("fps"),
|
|
"total_frames": meta.get("total_frames"),
|
|
"duration_s": meta.get("duration_s"),
|
|
"width": meta.get("width"),
|
|
"height": meta.get("height"),
|
|
},
|
|
"sampling_policy": {
|
|
"frames_requested": meta.get("frames_requested"),
|
|
"sampled": meta.get("sampled"),
|
|
"indices": meta.get("indices"),
|
|
"timestamps_s": meta.get("timestamps_s"),
|
|
"sampling_method": meta.get("sampling_method"),
|
|
"max_side": settings.max_side,
|
|
"jpeg_quality": settings.jpeg_quality,
|
|
},
|
|
"runtime": {
|
|
"vllm_base_url": settings.vllm_base_url,
|
|
"model": settings.vllm_model,
|
|
},
|
|
"generation_params": {
|
|
"max_tokens": settings.max_tokens,
|
|
"temperature": settings.temperature,
|
|
"repetition_penalty": settings.repetition_penalty,
|
|
},
|
|
"latency_s": {"upload_save_time_s": save_time_s},
|
|
"prompt": prompt,
|
|
}
|
|
write_json(os.path.join(run_dir, "request.json"), request_record)
|
|
|
|
# Call vLLM
|
|
raw_resp, infer_time_s = call_vllm_chat(
|
|
base_url=settings.vllm_base_url,
|
|
model=settings.vllm_model,
|
|
data_urls=data_urls,
|
|
prompt=prompt,
|
|
max_tokens=settings.max_tokens,
|
|
temperature=settings.temperature,
|
|
repetition_penalty=settings.repetition_penalty,
|
|
)
|
|
write_json(os.path.join(run_dir, "raw_response.json"), raw_resp)
|
|
|
|
try:
|
|
model_text = raw_resp["choices"][0]["message"]["content"]
|
|
except Exception:
|
|
model_text = ""
|
|
|
|
parsed = parse_verdict_and_explanation(model_text)
|
|
usage = raw_resp.get("usage") or {}
|
|
|
|
indices = meta.get("indices") or []
|
|
timestamps = meta.get("timestamps_s") or []
|
|
evidence = build_frame_evidence(indices, timestamps)
|
|
|
|
result_record = {
|
|
"request_id": request_id,
|
|
"run_dir": run_dir,
|
|
"verdict": parsed["verdict"],
|
|
"explanation": parsed["explanation"],
|
|
"frames_analyzed": int(meta.get("sampled") or len(indices)),
|
|
"evidence": evidence,
|
|
"usage": {
|
|
"prompt_tokens": int(usage.get("prompt_tokens") or 0),
|
|
"completion_tokens": int(usage.get("completion_tokens") or 0),
|
|
"total_tokens": int(usage.get("total_tokens") or 0),
|
|
},
|
|
"latency_s": {
|
|
"sampling_time_s": float(meta.get("sampling_time_s") or 0.0),
|
|
"encode_time_s": encode_time_s,
|
|
"model_inference_time_s": round(infer_time_s, 4),
|
|
},
|
|
"meta": {
|
|
"fps": float(meta.get("fps") or 0.0),
|
|
"total_frames": int(meta.get("total_frames") or 0),
|
|
"duration_s": meta.get("duration_s"),
|
|
"sampled": int(meta.get("sampled") or 0),
|
|
"indices": meta.get("indices") or [],
|
|
"timestamps_s": meta.get("timestamps_s") or [],
|
|
},
|
|
}
|
|
write_json(os.path.join(run_dir, "result.json"), result_record)
|
|
|
|
return JSONResponse(content=result_record)
|
|
|
|
|
|
@app.post("/analyze/video/semantic", response_model=SemanticAnalysisResponse)
|
|
async def analyze_video_semantic(
|
|
file: UploadFile = File(...),
|
|
chunk_duration_s: float = Form(default=None),
|
|
frames_per_chunk: int = Form(default=None),
|
|
enable_aggregation: bool = Form(default=None),
|
|
):
|
|
"""
|
|
Semantic video analysis with dense temporal sampling.
|
|
|
|
This endpoint performs deep content understanding by:
|
|
1. Dividing video into temporal chunks
|
|
2. Densely sampling each chunk (24 frames per 10s default)
|
|
3. Analyzing each chunk with vision LLM
|
|
4. Optionally aggregating into coherent narrative
|
|
|
|
Use this for:
|
|
- Content understanding and description
|
|
- Action recognition
|
|
- Scene analysis
|
|
- Narrative extraction
|
|
|
|
Args:
|
|
file: Video file to analyze
|
|
chunk_duration_s: Duration of each chunk in seconds (default from settings)
|
|
frames_per_chunk: Frames to sample per chunk (default from settings)
|
|
enable_aggregation: Whether to aggregate chunks into summary (default from settings)
|
|
|
|
Returns:
|
|
SemanticAnalysisResponse with chunk descriptions and optional summary
|
|
"""
|
|
# Use settings defaults if not provided
|
|
chunk_duration_s = chunk_duration_s or settings.semantic_chunk_duration_s
|
|
frames_per_chunk = frames_per_chunk or settings.semantic_frames_per_chunk
|
|
enable_aggregation = enable_aggregation if enable_aggregation is not None else settings.semantic_enable_aggregation
|
|
|
|
# Use dedicated semantic vLLM if configured, otherwise fall back to deepfake vLLM
|
|
semantic_vllm_url = settings.semantic_vllm_base_url or settings.vllm_base_url
|
|
semantic_vllm_model = settings.semantic_vllm_model or settings.vllm_model
|
|
|
|
if not semantic_vllm_url:
|
|
raise HTTPException(status_code=500, detail="VIDEO_ANALYSIS_SEMANTIC_VLLM_BASE_URL or VIDEO_ANALYSIS_VLLM_BASE_URL is not set")
|
|
|
|
request_id = str(uuid.uuid4())
|
|
started_at = datetime.now(timezone.utc).isoformat()
|
|
overall_start = time.time()
|
|
|
|
# Create run directory
|
|
run_dir = os.path.join(settings.runs_dir, request_id)
|
|
safe_mkdir(run_dir)
|
|
|
|
# Save uploaded video
|
|
video_path = os.path.join(run_dir, file.filename or "upload.mp4")
|
|
with open(video_path, "wb") as out:
|
|
out.write(file.file.read())
|
|
|
|
video_hash = sha256_file(video_path)
|
|
|
|
# Sample video in chunks
|
|
try:
|
|
chunks, meta = sample_frames_chunked(
|
|
video_path,
|
|
chunk_duration_s=chunk_duration_s,
|
|
frames_per_chunk=frames_per_chunk,
|
|
)
|
|
except Exception as e:
|
|
raise HTTPException(status_code=400, detail=f"Failed to sample video: {str(e)}")
|
|
|
|
# Save request metadata
|
|
request_record = {
|
|
"request_id": request_id,
|
|
"started_at_utc": started_at,
|
|
"analysis_type": "semantic",
|
|
"video": {
|
|
"path": video_path,
|
|
"sha256": video_hash,
|
|
"fps": meta.get("fps"),
|
|
"total_frames": meta.get("total_frames"),
|
|
"duration_s": meta.get("duration_s"),
|
|
"width": meta.get("width"),
|
|
"height": meta.get("height"),
|
|
},
|
|
"sampling_policy": {
|
|
"chunk_duration_s": chunk_duration_s,
|
|
"frames_per_chunk": frames_per_chunk,
|
|
"num_chunks": meta["num_chunks"],
|
|
"total_frames_sampled": meta["total_frames_sampled"],
|
|
"chunks": meta["chunks"],
|
|
},
|
|
"runtime": {
|
|
"vllm_base_url": semantic_vllm_url,
|
|
"model": semantic_vllm_model,
|
|
},
|
|
}
|
|
write_json(os.path.join(run_dir, "semantic_request.json"), request_record)
|
|
|
|
# Process each chunk with vision LLM
|
|
chunk_results = []
|
|
prompt = settings.semantic_prompt
|
|
|
|
for chunk_idx, chunk_frames in enumerate(chunks):
|
|
chunk_meta = meta["chunks"][chunk_idx]
|
|
|
|
# Encode frames for this chunk
|
|
t_enc = time.time()
|
|
data_urls = [
|
|
frame_to_data_url_b64jpeg(f, max_side=settings.max_side, jpeg_quality=settings.jpeg_quality)
|
|
for f in chunk_frames
|
|
]
|
|
encode_time = time.time() - t_enc
|
|
|
|
# Call vLLM for this chunk (uses semantic vLLM - Qwen3-VL)
|
|
try:
|
|
raw_resp, infer_time = call_vllm_chat(
|
|
base_url=semantic_vllm_url,
|
|
model=semantic_vllm_model,
|
|
data_urls=data_urls,
|
|
prompt=prompt,
|
|
max_tokens=settings.max_tokens,
|
|
temperature=settings.temperature,
|
|
repetition_penalty=settings.repetition_penalty,
|
|
)
|
|
except Exception as e:
|
|
raise HTTPException(status_code=500, detail=f"VLM inference failed for chunk {chunk_idx}: {str(e)}")
|
|
|
|
# Extract description
|
|
try:
|
|
chunk_description = raw_resp["choices"][0]["message"]["content"]
|
|
except Exception:
|
|
chunk_description = ""
|
|
|
|
usage_data = raw_resp.get("usage", {})
|
|
|
|
chunk_result = ChunkResult(
|
|
chunk_idx=chunk_idx,
|
|
time_range=f"{chunk_meta['start_time_s']}s - {chunk_meta['end_time_s']}s",
|
|
description=chunk_description,
|
|
frames_analyzed=len(chunk_frames),
|
|
inference_time_s=round(infer_time, 4),
|
|
usage=Usage(
|
|
prompt_tokens=int(usage_data.get("prompt_tokens", 0)),
|
|
completion_tokens=int(usage_data.get("completion_tokens", 0)),
|
|
total_tokens=int(usage_data.get("total_tokens", 0)),
|
|
),
|
|
)
|
|
chunk_results.append(chunk_result)
|
|
|
|
# Save intermediate chunk result
|
|
write_json(
|
|
os.path.join(run_dir, f"chunk_{chunk_idx:03d}_response.json"),
|
|
{
|
|
"chunk_idx": chunk_idx,
|
|
"raw_response": raw_resp,
|
|
"description": chunk_description,
|
|
"encode_time_s": round(encode_time, 4),
|
|
"inference_time_s": round(infer_time, 4),
|
|
},
|
|
)
|
|
|
|
# Aggregate chunks into final summary (if enabled)
|
|
final_summary = None
|
|
aggregation_time = None
|
|
|
|
if enable_aggregation and len(chunk_results) > 1:
|
|
t_agg = time.time()
|
|
|
|
# Build aggregation prompt
|
|
chunks_text = "\n\n".join(
|
|
[f"Segment {r.chunk_idx + 1} ({r.time_range}):\n{r.description}" for r in chunk_results]
|
|
)
|
|
|
|
aggregation_prompt = settings.aggregation_prompt_template.format(
|
|
num_chunks=len(chunk_results),
|
|
duration=meta.get("duration_s", 0),
|
|
chunks_text=chunks_text,
|
|
)
|
|
|
|
# Call text LLM for aggregation
|
|
try:
|
|
headers = {"Content-Type": "application/json"}
|
|
if settings.semantic_llm_api_key:
|
|
headers["Authorization"] = f"Bearer {settings.semantic_llm_api_key}"
|
|
|
|
async with httpx.AsyncClient(timeout=120.0) as client:
|
|
llm_resp = await client.post(
|
|
f"{settings.semantic_llm_base_url}/v1/chat/completions",
|
|
json={
|
|
"model": settings.semantic_aggregation_model,
|
|
"messages": [{"role": "user", "content": aggregation_prompt}],
|
|
"max_tokens": 1500,
|
|
"temperature": 0.3,
|
|
},
|
|
headers=headers,
|
|
)
|
|
llm_resp.raise_for_status()
|
|
llm_data = llm_resp.json()
|
|
final_summary = llm_data["choices"][0]["message"]["content"]
|
|
except Exception as e:
|
|
# Non-fatal: log warning but continue
|
|
final_summary = f"[Aggregation failed: {str(e)}]"
|
|
|
|
aggregation_time = round(time.time() - t_agg, 4)
|
|
|
|
# Save aggregation result
|
|
write_json(
|
|
os.path.join(run_dir, "aggregation.json"),
|
|
{
|
|
"prompt": aggregation_prompt,
|
|
"summary": final_summary,
|
|
"aggregation_time_s": aggregation_time,
|
|
},
|
|
)
|
|
|
|
total_latency = round(time.time() - overall_start, 4)
|
|
|
|
# Build final response
|
|
result = SemanticAnalysisResponse(
|
|
request_id=uuid.UUID(request_id),
|
|
run_dir=run_dir,
|
|
analysis_type="semantic",
|
|
video_duration_s=meta.get("duration_s"),
|
|
num_chunks=len(chunk_results),
|
|
chunk_results=chunk_results,
|
|
final_summary=final_summary,
|
|
aggregation_time_s=aggregation_time,
|
|
total_latency_s=total_latency,
|
|
meta=SemanticMeta(
|
|
fps=float(meta.get("fps", 0)),
|
|
total_frames=int(meta.get("total_frames", 0)),
|
|
duration_s=meta.get("duration_s"),
|
|
chunk_duration_s=chunk_duration_s,
|
|
frames_per_chunk=frames_per_chunk,
|
|
total_frames_sampled=meta["total_frames_sampled"],
|
|
),
|
|
)
|
|
|
|
# Save final result
|
|
write_json(os.path.join(run_dir, "semantic_result.json"), result.model_dump(mode="json"))
|
|
|
|
return result
|
|
|
|
|
|
@app.get("/v1/info")
|
|
def get_component_info():
|
|
"""
|
|
Get component information for service catalog.
|
|
|
|
Returns complete metadata about this service including:
|
|
- Resource information (component metadata)
|
|
- Available models (BusterX for deepfake detection)
|
|
- Available functions (API endpoints)
|
|
|
|
This endpoint is used by the catalog-api to aggregate service information
|
|
and by backend systems to populate the catalog database.
|
|
|
|
Returns:
|
|
dict: Component information matching catalog.resources, catalog.models,
|
|
and catalog.functions schemas.
|
|
"""
|
|
# Build resource information (maps to catalog.resources)
|
|
resource = {
|
|
"name": "Video Analysis Service",
|
|
"slug": "video-analysis",
|
|
"resource_type": "api_service",
|
|
"provider": "internal",
|
|
"base_url": f"http://didiAI-video-api:54600",
|
|
"configuration": {
|
|
"version": "0.1.0",
|
|
"port": 54600,
|
|
"external_url": settings.external_url,
|
|
"vllm_base_url": settings.vllm_base_url,
|
|
"vllm_model": settings.vllm_model,
|
|
"frames": settings.frames,
|
|
"max_side": settings.max_side,
|
|
},
|
|
"authentication": {
|
|
"type": "none",
|
|
"required": False,
|
|
},
|
|
"headers": {
|
|
"Content-Type": "multipart/form-data",
|
|
"Accept": "application/json",
|
|
},
|
|
"rate_limits": {
|
|
"enabled": False,
|
|
},
|
|
"cost_tracking": {
|
|
"enabled": False,
|
|
},
|
|
"tags": ["video", "deepfake", "semantic-analysis", "vision", "computer-vision"],
|
|
"is_active": True,
|
|
"metadata": {
|
|
"category": "computer-vision",
|
|
"gpu_required": True,
|
|
"status": "healthy",
|
|
"runs_dir": settings.runs_dir,
|
|
},
|
|
}
|
|
|
|
# Define models (maps to catalog.models)
|
|
models = [
|
|
{
|
|
"name": "BusterX Deepfake Detector",
|
|
"slug": "busterx",
|
|
"provider": "l8cv",
|
|
"model_type": "vision",
|
|
"capabilities": [
|
|
"deepfake-detection",
|
|
"video-analysis",
|
|
"multimodal",
|
|
],
|
|
"configuration": {
|
|
"backend": "vllm",
|
|
"base_model": "Qwen2.5-VL-7B",
|
|
"max_tokens": settings.max_tokens,
|
|
"temperature": settings.temperature,
|
|
},
|
|
"endpoint": settings.vllm_base_url,
|
|
"api_key_ref": None,
|
|
"tags": ["deepfake", "vision", "busterx", "video", "7b"],
|
|
"is_active": True,
|
|
"metadata": {
|
|
"model_id": settings.vllm_model,
|
|
"vllm_base_url": settings.vllm_base_url,
|
|
"gpu_id": 1,
|
|
"vram_gb": 22,
|
|
"specialized": "deepfake-detection",
|
|
},
|
|
}
|
|
]
|
|
|
|
# Define available functions (maps to catalog.functions)
|
|
functions = [
|
|
{
|
|
"name": "Video Deepfake Analysis",
|
|
"slug": "video-deepfake-analysis",
|
|
"category": "deepfake-detection",
|
|
"description": (
|
|
"Analyze video for deepfake manipulation using BusterX model. "
|
|
"Samples 16 frames uniformly and returns verdict: REAL/FAKE/UNCERTAIN."
|
|
),
|
|
"input_schema": {
|
|
"type": "object",
|
|
"properties": {
|
|
"file": {
|
|
"type": "file",
|
|
"description": "Video file to analyze (mp4, avi, mov, etc.)",
|
|
},
|
|
},
|
|
"required": ["file"],
|
|
},
|
|
"output_schema": {
|
|
"type": "object",
|
|
"properties": {
|
|
"request_id": {"type": "string", "format": "uuid"},
|
|
"run_dir": {"type": "string"},
|
|
"verdict": {
|
|
"type": "string",
|
|
"enum": ["REAL", "FAKE", "UNCERTAIN"],
|
|
},
|
|
"explanation": {"type": "string"},
|
|
"frames_analyzed": {"type": "integer"},
|
|
"evidence": {
|
|
"type": "array",
|
|
"items": {
|
|
"type": "object",
|
|
"properties": {
|
|
"frame_index": {"type": "integer"},
|
|
"timestamp_s": {"type": "number"},
|
|
},
|
|
},
|
|
},
|
|
"usage": {
|
|
"type": "object",
|
|
"properties": {
|
|
"prompt_tokens": {"type": "integer"},
|
|
"completion_tokens": {"type": "integer"},
|
|
"total_tokens": {"type": "integer"},
|
|
},
|
|
},
|
|
"latency_s": {
|
|
"type": "object",
|
|
"properties": {
|
|
"sampling_time_s": {"type": "number"},
|
|
"encode_time_s": {"type": "number"},
|
|
"model_inference_time_s": {"type": "number"},
|
|
},
|
|
},
|
|
"meta": {
|
|
"type": "object",
|
|
"properties": {
|
|
"fps": {"type": "number"},
|
|
"total_frames": {"type": "integer"},
|
|
"duration_s": {"type": "number"},
|
|
"sampled": {"type": "integer"},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
"implementation": {
|
|
"method": "POST",
|
|
"path": "/analyze/video",
|
|
"content_type": "multipart/form-data",
|
|
"timeout": 120,
|
|
},
|
|
"endpoint": "http://localhost:8007/analyze/video",
|
|
"tags": ["video", "deepfake", "detection", "fast"],
|
|
"is_active": True,
|
|
"metadata": {
|
|
"average_latency_s": 13,
|
|
"frames_analyzed": 16,
|
|
"coverage": "~0.89% of frames",
|
|
},
|
|
},
|
|
{
|
|
"name": "Video Semantic Analysis",
|
|
"slug": "video-semantic-analysis",
|
|
"category": "semantic-analysis",
|
|
"description": (
|
|
"Deep semantic analysis of video content with temporal chunking. "
|
|
"Divides video into chunks, analyzes each with vision LLM, and synthesizes narrative."
|
|
),
|
|
"input_schema": {
|
|
"type": "object",
|
|
"properties": {
|
|
"file": {
|
|
"type": "file",
|
|
"description": "Video file to analyze",
|
|
},
|
|
"chunk_duration_s": {
|
|
"type": "number",
|
|
"minimum": 1.0,
|
|
"maximum": 60.0,
|
|
"default": 10.0,
|
|
"description": "Duration of each chunk in seconds",
|
|
},
|
|
"frames_per_chunk": {
|
|
"type": "integer",
|
|
"minimum": 4,
|
|
"maximum": 64,
|
|
"default": 24,
|
|
"description": "Frames to sample per chunk",
|
|
},
|
|
"enable_aggregation": {
|
|
"type": "boolean",
|
|
"default": True,
|
|
"description": "Aggregate chunks into final summary",
|
|
},
|
|
},
|
|
"required": ["file"],
|
|
},
|
|
"output_schema": {
|
|
"type": "object",
|
|
"properties": {
|
|
"request_id": {"type": "string"},
|
|
"num_chunks": {"type": "integer"},
|
|
"chunk_results": {
|
|
"type": "array",
|
|
"items": {
|
|
"type": "object",
|
|
"properties": {
|
|
"chunk_idx": {"type": "integer"},
|
|
"time_range": {"type": "string"},
|
|
"description": {"type": "string"},
|
|
"frames_analyzed": {"type": "integer"},
|
|
},
|
|
},
|
|
},
|
|
"final_summary": {"type": "string"},
|
|
"total_latency_s": {"type": "number"},
|
|
},
|
|
},
|
|
"implementation": {
|
|
"method": "POST",
|
|
"path": "/analyze/video/semantic",
|
|
"content_type": "multipart/form-data",
|
|
"timeout": 300,
|
|
},
|
|
"endpoint": "http://localhost:8007/analyze/video/semantic",
|
|
"tags": ["video", "semantic", "analysis", "detailed"],
|
|
"is_active": True,
|
|
"metadata": {
|
|
"average_latency_s": 77,
|
|
"coverage": "~8% of frames",
|
|
"use_case": "content-description",
|
|
},
|
|
},
|
|
]
|
|
|
|
return {
|
|
"resource": resource,
|
|
"models": models,
|
|
"functions": functions,
|
|
}
|