Livrare LOT 1 - Didi
This commit is contained in:
commit
5380c3fc63
990 changed files with 133308 additions and 0 deletions
772
ai_platform/modules/video-analysis/src/video_analysis/app.py
Normal file
772
ai_platform/modules/video-analysis/src/video_analysis/app.py
Normal file
|
|
@ -0,0 +1,772 @@
|
|||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
import uuid
|
||||
from collections.abc import AsyncGenerator
|
||||
from contextlib import asynccontextmanager
|
||||
from datetime import datetime, timezone
|
||||
|
||||
import httpx
|
||||
from fastapi import FastAPI, File, Form, HTTPException, UploadFile
|
||||
from fastapi.responses import JSONResponse
|
||||
|
||||
from .buster_client import (
|
||||
build_frame_evidence,
|
||||
call_vllm_chat,
|
||||
frame_to_data_url_b64jpeg,
|
||||
parse_verdict_and_explanation,
|
||||
)
|
||||
from .runtime_config import RuntimeConfigClient
|
||||
from .schemas import AnalyzeResponse, ChunkResult, SemanticAnalysisResponse, SemanticMeta, Usage
|
||||
from .settings import settings
|
||||
from .video_sampling import sample_frames_chunked, sample_frames_uniform
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Runtime config (reachable from handlers via app.state.runtime_config)
|
||||
runtime_config = RuntimeConfigClient(
|
||||
dashboard_url=settings.dashboard_url,
|
||||
live_log_logger_name="video_analysis",
|
||||
live_log_key="video.log.level",
|
||||
)
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
"""App lifespan: start runtime config polling."""
|
||||
logger.info("Starting Video Analysis API")
|
||||
await runtime_config.start()
|
||||
app.state.runtime_config = runtime_config
|
||||
yield
|
||||
logger.info("Shutting down Video Analysis API")
|
||||
await runtime_config.stop()
|
||||
|
||||
|
||||
app = FastAPI(
|
||||
title="Video Analysis API",
|
||||
description="Deepfake detection and semantic video analysis",
|
||||
version="0.1.0",
|
||||
servers=[{"url": settings.external_url, "description": "Video Analysis API"}],
|
||||
lifespan=lifespan,
|
||||
)
|
||||
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- observability
|
||||
|
||||
|
||||
# Prometheus /metrics + OTel tracing (no-op if deps missing or OTEL endpoint unset)
|
||||
|
||||
|
||||
try:
|
||||
|
||||
|
||||
from prometheus_fastapi_instrumentator import Instrumentator as _Inst # type: ignore
|
||||
|
||||
|
||||
_Inst(should_group_status_codes=True).instrument(app).expose(app, endpoint="/metrics", include_in_schema=False)
|
||||
|
||||
|
||||
except ImportError:
|
||||
|
||||
|
||||
pass
|
||||
|
||||
|
||||
import os as _os # noqa: E402
|
||||
|
||||
|
||||
_otel_ep = _os.environ.get("OTEL_EXPORTER_OTLP_ENDPOINT")
|
||||
|
||||
|
||||
if _otel_ep:
|
||||
|
||||
|
||||
try:
|
||||
|
||||
|
||||
from opentelemetry import trace as _trace # type: ignore
|
||||
|
||||
|
||||
from opentelemetry.sdk.resources import Resource as _R # type: ignore
|
||||
|
||||
|
||||
from opentelemetry.sdk.trace import TracerProvider as _TP # type: ignore
|
||||
|
||||
|
||||
from opentelemetry.sdk.trace.export import BatchSpanProcessor as _BSP # type: ignore
|
||||
|
||||
|
||||
from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter as _Exp # type: ignore
|
||||
|
||||
|
||||
from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor as _FInst # type: ignore
|
||||
|
||||
|
||||
_provider = _TP(resource=_R.create({"service.name": _os.environ.get("OTEL_SERVICE_NAME", "didiAI-video-api")}))
|
||||
|
||||
|
||||
_provider.add_span_processor(_BSP(_Exp(endpoint=_otel_ep, insecure=True)))
|
||||
|
||||
|
||||
_trace.set_tracer_provider(_provider)
|
||||
|
||||
|
||||
_FInst.instrument_app(app)
|
||||
|
||||
|
||||
print(f"[otel] didiAI-video-api instrumented -> {_otel_ep}")
|
||||
|
||||
|
||||
except ImportError as _e:
|
||||
|
||||
|
||||
print(f"[otel] skip: {_e}")
|
||||
|
||||
|
||||
|
||||
def safe_mkdir(path: str) -> None:
|
||||
os.makedirs(path, exist_ok=True)
|
||||
|
||||
|
||||
def write_json(path: str, obj) -> None:
|
||||
with open(path, "w", encoding="utf-8") as f:
|
||||
json.dump(obj, f, ensure_ascii=False, indent=2)
|
||||
|
||||
|
||||
def sha256_file(path: str, chunk_size: int = 8 * 1024 * 1024) -> str:
|
||||
h = hashlib.sha256()
|
||||
with open(path, "rb") as f:
|
||||
while True:
|
||||
b = f.read(chunk_size)
|
||||
if not b:
|
||||
break
|
||||
h.update(b)
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
@app.get("/health")
|
||||
def health():
|
||||
return {"status": "ok"}
|
||||
|
||||
|
||||
@app.post("/analyze/video", response_model=AnalyzeResponse)
|
||||
def analyze_video(file: UploadFile = File(...)):
|
||||
# Defensive (Settings already requires it)
|
||||
if not settings.vllm_base_url:
|
||||
raise HTTPException(status_code=500, detail="VIDEO_ANALYSIS_VLLM_BASE_URL is not set")
|
||||
|
||||
request_id = str(uuid.uuid4())
|
||||
started_at = datetime.now(timezone.utc).isoformat()
|
||||
|
||||
# Make run_dir absolute under runs_dir (inside container: /app/runs/<uuid>)
|
||||
run_dir = os.path.join(settings.runs_dir, request_id)
|
||||
safe_mkdir(run_dir)
|
||||
|
||||
# Save upload to disk for reproducibility
|
||||
video_path = os.path.join(run_dir, file.filename or "upload.mp4")
|
||||
t0 = time.time()
|
||||
with open(video_path, "wb") as out:
|
||||
out.write(file.file.read())
|
||||
save_time_s = round(time.time() - t0, 4)
|
||||
|
||||
video_hash = sha256_file(video_path)
|
||||
prompt = settings.analysis_prompt
|
||||
|
||||
# Sampling
|
||||
frames, meta = sample_frames_uniform(video_path, num_frames=settings.frames)
|
||||
|
||||
# Encode frames
|
||||
t_enc0 = time.time()
|
||||
data_urls = [
|
||||
frame_to_data_url_b64jpeg(f, max_side=settings.max_side, jpeg_quality=settings.jpeg_quality)
|
||||
for f in frames
|
||||
]
|
||||
encode_time_s = round(time.time() - t_enc0, 4)
|
||||
|
||||
request_record = {
|
||||
"request_id": request_id,
|
||||
"started_at_utc": started_at,
|
||||
"video": {
|
||||
"path": video_path,
|
||||
"sha256": video_hash,
|
||||
"fps": meta.get("fps"),
|
||||
"total_frames": meta.get("total_frames"),
|
||||
"duration_s": meta.get("duration_s"),
|
||||
"width": meta.get("width"),
|
||||
"height": meta.get("height"),
|
||||
},
|
||||
"sampling_policy": {
|
||||
"frames_requested": meta.get("frames_requested"),
|
||||
"sampled": meta.get("sampled"),
|
||||
"indices": meta.get("indices"),
|
||||
"timestamps_s": meta.get("timestamps_s"),
|
||||
"sampling_method": meta.get("sampling_method"),
|
||||
"max_side": settings.max_side,
|
||||
"jpeg_quality": settings.jpeg_quality,
|
||||
},
|
||||
"runtime": {
|
||||
"vllm_base_url": settings.vllm_base_url,
|
||||
"model": settings.vllm_model,
|
||||
},
|
||||
"generation_params": {
|
||||
"max_tokens": settings.max_tokens,
|
||||
"temperature": settings.temperature,
|
||||
"repetition_penalty": settings.repetition_penalty,
|
||||
},
|
||||
"latency_s": {"upload_save_time_s": save_time_s},
|
||||
"prompt": prompt,
|
||||
}
|
||||
write_json(os.path.join(run_dir, "request.json"), request_record)
|
||||
|
||||
# Call vLLM
|
||||
raw_resp, infer_time_s = call_vllm_chat(
|
||||
base_url=settings.vllm_base_url,
|
||||
model=settings.vllm_model,
|
||||
data_urls=data_urls,
|
||||
prompt=prompt,
|
||||
max_tokens=settings.max_tokens,
|
||||
temperature=settings.temperature,
|
||||
repetition_penalty=settings.repetition_penalty,
|
||||
)
|
||||
write_json(os.path.join(run_dir, "raw_response.json"), raw_resp)
|
||||
|
||||
try:
|
||||
model_text = raw_resp["choices"][0]["message"]["content"]
|
||||
except Exception:
|
||||
model_text = ""
|
||||
|
||||
parsed = parse_verdict_and_explanation(model_text)
|
||||
usage = raw_resp.get("usage") or {}
|
||||
|
||||
indices = meta.get("indices") or []
|
||||
timestamps = meta.get("timestamps_s") or []
|
||||
evidence = build_frame_evidence(indices, timestamps)
|
||||
|
||||
result_record = {
|
||||
"request_id": request_id,
|
||||
"run_dir": run_dir,
|
||||
"verdict": parsed["verdict"],
|
||||
"explanation": parsed["explanation"],
|
||||
"frames_analyzed": int(meta.get("sampled") or len(indices)),
|
||||
"evidence": evidence,
|
||||
"usage": {
|
||||
"prompt_tokens": int(usage.get("prompt_tokens") or 0),
|
||||
"completion_tokens": int(usage.get("completion_tokens") or 0),
|
||||
"total_tokens": int(usage.get("total_tokens") or 0),
|
||||
},
|
||||
"latency_s": {
|
||||
"sampling_time_s": float(meta.get("sampling_time_s") or 0.0),
|
||||
"encode_time_s": encode_time_s,
|
||||
"model_inference_time_s": round(infer_time_s, 4),
|
||||
},
|
||||
"meta": {
|
||||
"fps": float(meta.get("fps") or 0.0),
|
||||
"total_frames": int(meta.get("total_frames") or 0),
|
||||
"duration_s": meta.get("duration_s"),
|
||||
"sampled": int(meta.get("sampled") or 0),
|
||||
"indices": meta.get("indices") or [],
|
||||
"timestamps_s": meta.get("timestamps_s") or [],
|
||||
},
|
||||
}
|
||||
write_json(os.path.join(run_dir, "result.json"), result_record)
|
||||
|
||||
return JSONResponse(content=result_record)
|
||||
|
||||
|
||||
@app.post("/analyze/video/semantic", response_model=SemanticAnalysisResponse)
|
||||
async def analyze_video_semantic(
|
||||
file: UploadFile = File(...),
|
||||
chunk_duration_s: float = Form(default=None),
|
||||
frames_per_chunk: int = Form(default=None),
|
||||
enable_aggregation: bool = Form(default=None),
|
||||
):
|
||||
"""
|
||||
Semantic video analysis with dense temporal sampling.
|
||||
|
||||
This endpoint performs deep content understanding by:
|
||||
1. Dividing video into temporal chunks
|
||||
2. Densely sampling each chunk (24 frames per 10s default)
|
||||
3. Analyzing each chunk with vision LLM
|
||||
4. Optionally aggregating into coherent narrative
|
||||
|
||||
Use this for:
|
||||
- Content understanding and description
|
||||
- Action recognition
|
||||
- Scene analysis
|
||||
- Narrative extraction
|
||||
|
||||
Args:
|
||||
file: Video file to analyze
|
||||
chunk_duration_s: Duration of each chunk in seconds (default from settings)
|
||||
frames_per_chunk: Frames to sample per chunk (default from settings)
|
||||
enable_aggregation: Whether to aggregate chunks into summary (default from settings)
|
||||
|
||||
Returns:
|
||||
SemanticAnalysisResponse with chunk descriptions and optional summary
|
||||
"""
|
||||
# Use settings defaults if not provided
|
||||
chunk_duration_s = chunk_duration_s or settings.semantic_chunk_duration_s
|
||||
frames_per_chunk = frames_per_chunk or settings.semantic_frames_per_chunk
|
||||
enable_aggregation = enable_aggregation if enable_aggregation is not None else settings.semantic_enable_aggregation
|
||||
|
||||
# Use dedicated semantic vLLM if configured, otherwise fall back to deepfake vLLM
|
||||
semantic_vllm_url = settings.semantic_vllm_base_url or settings.vllm_base_url
|
||||
semantic_vllm_model = settings.semantic_vllm_model or settings.vllm_model
|
||||
|
||||
if not semantic_vllm_url:
|
||||
raise HTTPException(status_code=500, detail="VIDEO_ANALYSIS_SEMANTIC_VLLM_BASE_URL or VIDEO_ANALYSIS_VLLM_BASE_URL is not set")
|
||||
|
||||
request_id = str(uuid.uuid4())
|
||||
started_at = datetime.now(timezone.utc).isoformat()
|
||||
overall_start = time.time()
|
||||
|
||||
# Create run directory
|
||||
run_dir = os.path.join(settings.runs_dir, request_id)
|
||||
safe_mkdir(run_dir)
|
||||
|
||||
# Save uploaded video
|
||||
video_path = os.path.join(run_dir, file.filename or "upload.mp4")
|
||||
with open(video_path, "wb") as out:
|
||||
out.write(file.file.read())
|
||||
|
||||
video_hash = sha256_file(video_path)
|
||||
|
||||
# Sample video in chunks
|
||||
try:
|
||||
chunks, meta = sample_frames_chunked(
|
||||
video_path,
|
||||
chunk_duration_s=chunk_duration_s,
|
||||
frames_per_chunk=frames_per_chunk,
|
||||
)
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=400, detail=f"Failed to sample video: {str(e)}")
|
||||
|
||||
# Save request metadata
|
||||
request_record = {
|
||||
"request_id": request_id,
|
||||
"started_at_utc": started_at,
|
||||
"analysis_type": "semantic",
|
||||
"video": {
|
||||
"path": video_path,
|
||||
"sha256": video_hash,
|
||||
"fps": meta.get("fps"),
|
||||
"total_frames": meta.get("total_frames"),
|
||||
"duration_s": meta.get("duration_s"),
|
||||
"width": meta.get("width"),
|
||||
"height": meta.get("height"),
|
||||
},
|
||||
"sampling_policy": {
|
||||
"chunk_duration_s": chunk_duration_s,
|
||||
"frames_per_chunk": frames_per_chunk,
|
||||
"num_chunks": meta["num_chunks"],
|
||||
"total_frames_sampled": meta["total_frames_sampled"],
|
||||
"chunks": meta["chunks"],
|
||||
},
|
||||
"runtime": {
|
||||
"vllm_base_url": semantic_vllm_url,
|
||||
"model": semantic_vllm_model,
|
||||
},
|
||||
}
|
||||
write_json(os.path.join(run_dir, "semantic_request.json"), request_record)
|
||||
|
||||
# Process each chunk with vision LLM
|
||||
chunk_results = []
|
||||
prompt = settings.semantic_prompt
|
||||
|
||||
for chunk_idx, chunk_frames in enumerate(chunks):
|
||||
chunk_meta = meta["chunks"][chunk_idx]
|
||||
|
||||
# Encode frames for this chunk
|
||||
t_enc = time.time()
|
||||
data_urls = [
|
||||
frame_to_data_url_b64jpeg(f, max_side=settings.max_side, jpeg_quality=settings.jpeg_quality)
|
||||
for f in chunk_frames
|
||||
]
|
||||
encode_time = time.time() - t_enc
|
||||
|
||||
# Call vLLM for this chunk (uses semantic vLLM - Qwen3-VL)
|
||||
try:
|
||||
raw_resp, infer_time = call_vllm_chat(
|
||||
base_url=semantic_vllm_url,
|
||||
model=semantic_vllm_model,
|
||||
data_urls=data_urls,
|
||||
prompt=prompt,
|
||||
max_tokens=settings.max_tokens,
|
||||
temperature=settings.temperature,
|
||||
repetition_penalty=settings.repetition_penalty,
|
||||
)
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=f"VLM inference failed for chunk {chunk_idx}: {str(e)}")
|
||||
|
||||
# Extract description
|
||||
try:
|
||||
chunk_description = raw_resp["choices"][0]["message"]["content"]
|
||||
except Exception:
|
||||
chunk_description = ""
|
||||
|
||||
usage_data = raw_resp.get("usage", {})
|
||||
|
||||
chunk_result = ChunkResult(
|
||||
chunk_idx=chunk_idx,
|
||||
time_range=f"{chunk_meta['start_time_s']}s - {chunk_meta['end_time_s']}s",
|
||||
description=chunk_description,
|
||||
frames_analyzed=len(chunk_frames),
|
||||
inference_time_s=round(infer_time, 4),
|
||||
usage=Usage(
|
||||
prompt_tokens=int(usage_data.get("prompt_tokens", 0)),
|
||||
completion_tokens=int(usage_data.get("completion_tokens", 0)),
|
||||
total_tokens=int(usage_data.get("total_tokens", 0)),
|
||||
),
|
||||
)
|
||||
chunk_results.append(chunk_result)
|
||||
|
||||
# Save intermediate chunk result
|
||||
write_json(
|
||||
os.path.join(run_dir, f"chunk_{chunk_idx:03d}_response.json"),
|
||||
{
|
||||
"chunk_idx": chunk_idx,
|
||||
"raw_response": raw_resp,
|
||||
"description": chunk_description,
|
||||
"encode_time_s": round(encode_time, 4),
|
||||
"inference_time_s": round(infer_time, 4),
|
||||
},
|
||||
)
|
||||
|
||||
# Aggregate chunks into final summary (if enabled)
|
||||
final_summary = None
|
||||
aggregation_time = None
|
||||
|
||||
if enable_aggregation and len(chunk_results) > 1:
|
||||
t_agg = time.time()
|
||||
|
||||
# Build aggregation prompt
|
||||
chunks_text = "\n\n".join(
|
||||
[f"Segment {r.chunk_idx + 1} ({r.time_range}):\n{r.description}" for r in chunk_results]
|
||||
)
|
||||
|
||||
aggregation_prompt = settings.aggregation_prompt_template.format(
|
||||
num_chunks=len(chunk_results),
|
||||
duration=meta.get("duration_s", 0),
|
||||
chunks_text=chunks_text,
|
||||
)
|
||||
|
||||
# Call text LLM for aggregation
|
||||
try:
|
||||
headers = {"Content-Type": "application/json"}
|
||||
if settings.semantic_llm_api_key:
|
||||
headers["Authorization"] = f"Bearer {settings.semantic_llm_api_key}"
|
||||
|
||||
async with httpx.AsyncClient(timeout=120.0) as client:
|
||||
llm_resp = await client.post(
|
||||
f"{settings.semantic_llm_base_url}/v1/chat/completions",
|
||||
json={
|
||||
"model": settings.semantic_aggregation_model,
|
||||
"messages": [{"role": "user", "content": aggregation_prompt}],
|
||||
"max_tokens": 1500,
|
||||
"temperature": 0.3,
|
||||
},
|
||||
headers=headers,
|
||||
)
|
||||
llm_resp.raise_for_status()
|
||||
llm_data = llm_resp.json()
|
||||
final_summary = llm_data["choices"][0]["message"]["content"]
|
||||
except Exception as e:
|
||||
# Non-fatal: log warning but continue
|
||||
final_summary = f"[Aggregation failed: {str(e)}]"
|
||||
|
||||
aggregation_time = round(time.time() - t_agg, 4)
|
||||
|
||||
# Save aggregation result
|
||||
write_json(
|
||||
os.path.join(run_dir, "aggregation.json"),
|
||||
{
|
||||
"prompt": aggregation_prompt,
|
||||
"summary": final_summary,
|
||||
"aggregation_time_s": aggregation_time,
|
||||
},
|
||||
)
|
||||
|
||||
total_latency = round(time.time() - overall_start, 4)
|
||||
|
||||
# Build final response
|
||||
result = SemanticAnalysisResponse(
|
||||
request_id=uuid.UUID(request_id),
|
||||
run_dir=run_dir,
|
||||
analysis_type="semantic",
|
||||
video_duration_s=meta.get("duration_s"),
|
||||
num_chunks=len(chunk_results),
|
||||
chunk_results=chunk_results,
|
||||
final_summary=final_summary,
|
||||
aggregation_time_s=aggregation_time,
|
||||
total_latency_s=total_latency,
|
||||
meta=SemanticMeta(
|
||||
fps=float(meta.get("fps", 0)),
|
||||
total_frames=int(meta.get("total_frames", 0)),
|
||||
duration_s=meta.get("duration_s"),
|
||||
chunk_duration_s=chunk_duration_s,
|
||||
frames_per_chunk=frames_per_chunk,
|
||||
total_frames_sampled=meta["total_frames_sampled"],
|
||||
),
|
||||
)
|
||||
|
||||
# Save final result
|
||||
write_json(os.path.join(run_dir, "semantic_result.json"), result.model_dump(mode="json"))
|
||||
|
||||
return result
|
||||
|
||||
|
||||
@app.get("/v1/info")
|
||||
def get_component_info():
|
||||
"""
|
||||
Get component information for service catalog.
|
||||
|
||||
Returns complete metadata about this service including:
|
||||
- Resource information (component metadata)
|
||||
- Available models (BusterX for deepfake detection)
|
||||
- Available functions (API endpoints)
|
||||
|
||||
This endpoint is used by the catalog-api to aggregate service information
|
||||
and by backend systems to populate the catalog database.
|
||||
|
||||
Returns:
|
||||
dict: Component information matching catalog.resources, catalog.models,
|
||||
and catalog.functions schemas.
|
||||
"""
|
||||
# Build resource information (maps to catalog.resources)
|
||||
resource = {
|
||||
"name": "Video Analysis Service",
|
||||
"slug": "video-analysis",
|
||||
"resource_type": "api_service",
|
||||
"provider": "internal",
|
||||
"base_url": f"http://didiAI-video-api:54600",
|
||||
"configuration": {
|
||||
"version": "0.1.0",
|
||||
"port": 54600,
|
||||
"external_url": settings.external_url,
|
||||
"vllm_base_url": settings.vllm_base_url,
|
||||
"vllm_model": settings.vllm_model,
|
||||
"frames": settings.frames,
|
||||
"max_side": settings.max_side,
|
||||
},
|
||||
"authentication": {
|
||||
"type": "none",
|
||||
"required": False,
|
||||
},
|
||||
"headers": {
|
||||
"Content-Type": "multipart/form-data",
|
||||
"Accept": "application/json",
|
||||
},
|
||||
"rate_limits": {
|
||||
"enabled": False,
|
||||
},
|
||||
"cost_tracking": {
|
||||
"enabled": False,
|
||||
},
|
||||
"tags": ["video", "deepfake", "semantic-analysis", "vision", "computer-vision"],
|
||||
"is_active": True,
|
||||
"metadata": {
|
||||
"category": "computer-vision",
|
||||
"gpu_required": True,
|
||||
"status": "healthy",
|
||||
"runs_dir": settings.runs_dir,
|
||||
},
|
||||
}
|
||||
|
||||
# Define models (maps to catalog.models)
|
||||
models = [
|
||||
{
|
||||
"name": "BusterX Deepfake Detector",
|
||||
"slug": "busterx",
|
||||
"provider": "l8cv",
|
||||
"model_type": "vision",
|
||||
"capabilities": [
|
||||
"deepfake-detection",
|
||||
"video-analysis",
|
||||
"multimodal",
|
||||
],
|
||||
"configuration": {
|
||||
"backend": "vllm",
|
||||
"base_model": "Qwen2.5-VL-7B",
|
||||
"max_tokens": settings.max_tokens,
|
||||
"temperature": settings.temperature,
|
||||
},
|
||||
"endpoint": settings.vllm_base_url,
|
||||
"api_key_ref": None,
|
||||
"tags": ["deepfake", "vision", "busterx", "video", "7b"],
|
||||
"is_active": True,
|
||||
"metadata": {
|
||||
"model_id": settings.vllm_model,
|
||||
"vllm_base_url": settings.vllm_base_url,
|
||||
"gpu_id": 1,
|
||||
"vram_gb": 22,
|
||||
"specialized": "deepfake-detection",
|
||||
},
|
||||
}
|
||||
]
|
||||
|
||||
# Define available functions (maps to catalog.functions)
|
||||
functions = [
|
||||
{
|
||||
"name": "Video Deepfake Analysis",
|
||||
"slug": "video-deepfake-analysis",
|
||||
"category": "deepfake-detection",
|
||||
"description": (
|
||||
"Analyze video for deepfake manipulation using BusterX model. "
|
||||
"Samples 16 frames uniformly and returns verdict: REAL/FAKE/UNCERTAIN."
|
||||
),
|
||||
"input_schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"file": {
|
||||
"type": "file",
|
||||
"description": "Video file to analyze (mp4, avi, mov, etc.)",
|
||||
},
|
||||
},
|
||||
"required": ["file"],
|
||||
},
|
||||
"output_schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"request_id": {"type": "string", "format": "uuid"},
|
||||
"run_dir": {"type": "string"},
|
||||
"verdict": {
|
||||
"type": "string",
|
||||
"enum": ["REAL", "FAKE", "UNCERTAIN"],
|
||||
},
|
||||
"explanation": {"type": "string"},
|
||||
"frames_analyzed": {"type": "integer"},
|
||||
"evidence": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"frame_index": {"type": "integer"},
|
||||
"timestamp_s": {"type": "number"},
|
||||
},
|
||||
},
|
||||
},
|
||||
"usage": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"prompt_tokens": {"type": "integer"},
|
||||
"completion_tokens": {"type": "integer"},
|
||||
"total_tokens": {"type": "integer"},
|
||||
},
|
||||
},
|
||||
"latency_s": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"sampling_time_s": {"type": "number"},
|
||||
"encode_time_s": {"type": "number"},
|
||||
"model_inference_time_s": {"type": "number"},
|
||||
},
|
||||
},
|
||||
"meta": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"fps": {"type": "number"},
|
||||
"total_frames": {"type": "integer"},
|
||||
"duration_s": {"type": "number"},
|
||||
"sampled": {"type": "integer"},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
"implementation": {
|
||||
"method": "POST",
|
||||
"path": "/analyze/video",
|
||||
"content_type": "multipart/form-data",
|
||||
"timeout": 120,
|
||||
},
|
||||
"endpoint": "http://localhost:8007/analyze/video",
|
||||
"tags": ["video", "deepfake", "detection", "fast"],
|
||||
"is_active": True,
|
||||
"metadata": {
|
||||
"average_latency_s": 13,
|
||||
"frames_analyzed": 16,
|
||||
"coverage": "~0.89% of frames",
|
||||
},
|
||||
},
|
||||
{
|
||||
"name": "Video Semantic Analysis",
|
||||
"slug": "video-semantic-analysis",
|
||||
"category": "semantic-analysis",
|
||||
"description": (
|
||||
"Deep semantic analysis of video content with temporal chunking. "
|
||||
"Divides video into chunks, analyzes each with vision LLM, and synthesizes narrative."
|
||||
),
|
||||
"input_schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"file": {
|
||||
"type": "file",
|
||||
"description": "Video file to analyze",
|
||||
},
|
||||
"chunk_duration_s": {
|
||||
"type": "number",
|
||||
"minimum": 1.0,
|
||||
"maximum": 60.0,
|
||||
"default": 10.0,
|
||||
"description": "Duration of each chunk in seconds",
|
||||
},
|
||||
"frames_per_chunk": {
|
||||
"type": "integer",
|
||||
"minimum": 4,
|
||||
"maximum": 64,
|
||||
"default": 24,
|
||||
"description": "Frames to sample per chunk",
|
||||
},
|
||||
"enable_aggregation": {
|
||||
"type": "boolean",
|
||||
"default": True,
|
||||
"description": "Aggregate chunks into final summary",
|
||||
},
|
||||
},
|
||||
"required": ["file"],
|
||||
},
|
||||
"output_schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"request_id": {"type": "string"},
|
||||
"num_chunks": {"type": "integer"},
|
||||
"chunk_results": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"chunk_idx": {"type": "integer"},
|
||||
"time_range": {"type": "string"},
|
||||
"description": {"type": "string"},
|
||||
"frames_analyzed": {"type": "integer"},
|
||||
},
|
||||
},
|
||||
},
|
||||
"final_summary": {"type": "string"},
|
||||
"total_latency_s": {"type": "number"},
|
||||
},
|
||||
},
|
||||
"implementation": {
|
||||
"method": "POST",
|
||||
"path": "/analyze/video/semantic",
|
||||
"content_type": "multipart/form-data",
|
||||
"timeout": 300,
|
||||
},
|
||||
"endpoint": "http://localhost:8007/analyze/video/semantic",
|
||||
"tags": ["video", "semantic", "analysis", "detailed"],
|
||||
"is_active": True,
|
||||
"metadata": {
|
||||
"average_latency_s": 77,
|
||||
"coverage": "~8% of frames",
|
||||
"use_case": "content-description",
|
||||
},
|
||||
},
|
||||
]
|
||||
|
||||
return {
|
||||
"resource": resource,
|
||||
"models": models,
|
||||
"functions": functions,
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue