Livrare LOT 1 - Didi

This commit is contained in:
Dezvoltari Evotech 2026-06-25 14:13:25 -07:00
commit 5380c3fc63
990 changed files with 133308 additions and 0 deletions

View file

@ -0,0 +1,772 @@
import hashlib
import json
import logging
import os
import time
import uuid
from collections.abc import AsyncGenerator
from contextlib import asynccontextmanager
from datetime import datetime, timezone
import httpx
from fastapi import FastAPI, File, Form, HTTPException, UploadFile
from fastapi.responses import JSONResponse
from .buster_client import (
build_frame_evidence,
call_vllm_chat,
frame_to_data_url_b64jpeg,
parse_verdict_and_explanation,
)
from .runtime_config import RuntimeConfigClient
from .schemas import AnalyzeResponse, ChunkResult, SemanticAnalysisResponse, SemanticMeta, Usage
from .settings import settings
from .video_sampling import sample_frames_chunked, sample_frames_uniform
logger = logging.getLogger(__name__)
# Runtime config (reachable from handlers via app.state.runtime_config)
runtime_config = RuntimeConfigClient(
dashboard_url=settings.dashboard_url,
live_log_logger_name="video_analysis",
live_log_key="video.log.level",
)
@asynccontextmanager
async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
"""App lifespan: start runtime config polling."""
logger.info("Starting Video Analysis API")
await runtime_config.start()
app.state.runtime_config = runtime_config
yield
logger.info("Shutting down Video Analysis API")
await runtime_config.stop()
app = FastAPI(
title="Video Analysis API",
description="Deepfake detection and semantic video analysis",
version="0.1.0",
servers=[{"url": settings.external_url, "description": "Video Analysis API"}],
lifespan=lifespan,
)
# ---------------------------------------------------------------- observability
# Prometheus /metrics + OTel tracing (no-op if deps missing or OTEL endpoint unset)
try:
from prometheus_fastapi_instrumentator import Instrumentator as _Inst # type: ignore
_Inst(should_group_status_codes=True).instrument(app).expose(app, endpoint="/metrics", include_in_schema=False)
except ImportError:
pass
import os as _os # noqa: E402
_otel_ep = _os.environ.get("OTEL_EXPORTER_OTLP_ENDPOINT")
if _otel_ep:
try:
from opentelemetry import trace as _trace # type: ignore
from opentelemetry.sdk.resources import Resource as _R # type: ignore
from opentelemetry.sdk.trace import TracerProvider as _TP # type: ignore
from opentelemetry.sdk.trace.export import BatchSpanProcessor as _BSP # type: ignore
from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter as _Exp # type: ignore
from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor as _FInst # type: ignore
_provider = _TP(resource=_R.create({"service.name": _os.environ.get("OTEL_SERVICE_NAME", "didiAI-video-api")}))
_provider.add_span_processor(_BSP(_Exp(endpoint=_otel_ep, insecure=True)))
_trace.set_tracer_provider(_provider)
_FInst.instrument_app(app)
print(f"[otel] didiAI-video-api instrumented -> {_otel_ep}")
except ImportError as _e:
print(f"[otel] skip: {_e}")
def safe_mkdir(path: str) -> None:
os.makedirs(path, exist_ok=True)
def write_json(path: str, obj) -> None:
with open(path, "w", encoding="utf-8") as f:
json.dump(obj, f, ensure_ascii=False, indent=2)
def sha256_file(path: str, chunk_size: int = 8 * 1024 * 1024) -> str:
h = hashlib.sha256()
with open(path, "rb") as f:
while True:
b = f.read(chunk_size)
if not b:
break
h.update(b)
return h.hexdigest()
@app.get("/health")
def health():
return {"status": "ok"}
@app.post("/analyze/video", response_model=AnalyzeResponse)
def analyze_video(file: UploadFile = File(...)):
# Defensive (Settings already requires it)
if not settings.vllm_base_url:
raise HTTPException(status_code=500, detail="VIDEO_ANALYSIS_VLLM_BASE_URL is not set")
request_id = str(uuid.uuid4())
started_at = datetime.now(timezone.utc).isoformat()
# Make run_dir absolute under runs_dir (inside container: /app/runs/<uuid>)
run_dir = os.path.join(settings.runs_dir, request_id)
safe_mkdir(run_dir)
# Save upload to disk for reproducibility
video_path = os.path.join(run_dir, file.filename or "upload.mp4")
t0 = time.time()
with open(video_path, "wb") as out:
out.write(file.file.read())
save_time_s = round(time.time() - t0, 4)
video_hash = sha256_file(video_path)
prompt = settings.analysis_prompt
# Sampling
frames, meta = sample_frames_uniform(video_path, num_frames=settings.frames)
# Encode frames
t_enc0 = time.time()
data_urls = [
frame_to_data_url_b64jpeg(f, max_side=settings.max_side, jpeg_quality=settings.jpeg_quality)
for f in frames
]
encode_time_s = round(time.time() - t_enc0, 4)
request_record = {
"request_id": request_id,
"started_at_utc": started_at,
"video": {
"path": video_path,
"sha256": video_hash,
"fps": meta.get("fps"),
"total_frames": meta.get("total_frames"),
"duration_s": meta.get("duration_s"),
"width": meta.get("width"),
"height": meta.get("height"),
},
"sampling_policy": {
"frames_requested": meta.get("frames_requested"),
"sampled": meta.get("sampled"),
"indices": meta.get("indices"),
"timestamps_s": meta.get("timestamps_s"),
"sampling_method": meta.get("sampling_method"),
"max_side": settings.max_side,
"jpeg_quality": settings.jpeg_quality,
},
"runtime": {
"vllm_base_url": settings.vllm_base_url,
"model": settings.vllm_model,
},
"generation_params": {
"max_tokens": settings.max_tokens,
"temperature": settings.temperature,
"repetition_penalty": settings.repetition_penalty,
},
"latency_s": {"upload_save_time_s": save_time_s},
"prompt": prompt,
}
write_json(os.path.join(run_dir, "request.json"), request_record)
# Call vLLM
raw_resp, infer_time_s = call_vllm_chat(
base_url=settings.vllm_base_url,
model=settings.vllm_model,
data_urls=data_urls,
prompt=prompt,
max_tokens=settings.max_tokens,
temperature=settings.temperature,
repetition_penalty=settings.repetition_penalty,
)
write_json(os.path.join(run_dir, "raw_response.json"), raw_resp)
try:
model_text = raw_resp["choices"][0]["message"]["content"]
except Exception:
model_text = ""
parsed = parse_verdict_and_explanation(model_text)
usage = raw_resp.get("usage") or {}
indices = meta.get("indices") or []
timestamps = meta.get("timestamps_s") or []
evidence = build_frame_evidence(indices, timestamps)
result_record = {
"request_id": request_id,
"run_dir": run_dir,
"verdict": parsed["verdict"],
"explanation": parsed["explanation"],
"frames_analyzed": int(meta.get("sampled") or len(indices)),
"evidence": evidence,
"usage": {
"prompt_tokens": int(usage.get("prompt_tokens") or 0),
"completion_tokens": int(usage.get("completion_tokens") or 0),
"total_tokens": int(usage.get("total_tokens") or 0),
},
"latency_s": {
"sampling_time_s": float(meta.get("sampling_time_s") or 0.0),
"encode_time_s": encode_time_s,
"model_inference_time_s": round(infer_time_s, 4),
},
"meta": {
"fps": float(meta.get("fps") or 0.0),
"total_frames": int(meta.get("total_frames") or 0),
"duration_s": meta.get("duration_s"),
"sampled": int(meta.get("sampled") or 0),
"indices": meta.get("indices") or [],
"timestamps_s": meta.get("timestamps_s") or [],
},
}
write_json(os.path.join(run_dir, "result.json"), result_record)
return JSONResponse(content=result_record)
@app.post("/analyze/video/semantic", response_model=SemanticAnalysisResponse)
async def analyze_video_semantic(
file: UploadFile = File(...),
chunk_duration_s: float = Form(default=None),
frames_per_chunk: int = Form(default=None),
enable_aggregation: bool = Form(default=None),
):
"""
Semantic video analysis with dense temporal sampling.
This endpoint performs deep content understanding by:
1. Dividing video into temporal chunks
2. Densely sampling each chunk (24 frames per 10s default)
3. Analyzing each chunk with vision LLM
4. Optionally aggregating into coherent narrative
Use this for:
- Content understanding and description
- Action recognition
- Scene analysis
- Narrative extraction
Args:
file: Video file to analyze
chunk_duration_s: Duration of each chunk in seconds (default from settings)
frames_per_chunk: Frames to sample per chunk (default from settings)
enable_aggregation: Whether to aggregate chunks into summary (default from settings)
Returns:
SemanticAnalysisResponse with chunk descriptions and optional summary
"""
# Use settings defaults if not provided
chunk_duration_s = chunk_duration_s or settings.semantic_chunk_duration_s
frames_per_chunk = frames_per_chunk or settings.semantic_frames_per_chunk
enable_aggregation = enable_aggregation if enable_aggregation is not None else settings.semantic_enable_aggregation
# Use dedicated semantic vLLM if configured, otherwise fall back to deepfake vLLM
semantic_vllm_url = settings.semantic_vllm_base_url or settings.vllm_base_url
semantic_vllm_model = settings.semantic_vllm_model or settings.vllm_model
if not semantic_vllm_url:
raise HTTPException(status_code=500, detail="VIDEO_ANALYSIS_SEMANTIC_VLLM_BASE_URL or VIDEO_ANALYSIS_VLLM_BASE_URL is not set")
request_id = str(uuid.uuid4())
started_at = datetime.now(timezone.utc).isoformat()
overall_start = time.time()
# Create run directory
run_dir = os.path.join(settings.runs_dir, request_id)
safe_mkdir(run_dir)
# Save uploaded video
video_path = os.path.join(run_dir, file.filename or "upload.mp4")
with open(video_path, "wb") as out:
out.write(file.file.read())
video_hash = sha256_file(video_path)
# Sample video in chunks
try:
chunks, meta = sample_frames_chunked(
video_path,
chunk_duration_s=chunk_duration_s,
frames_per_chunk=frames_per_chunk,
)
except Exception as e:
raise HTTPException(status_code=400, detail=f"Failed to sample video: {str(e)}")
# Save request metadata
request_record = {
"request_id": request_id,
"started_at_utc": started_at,
"analysis_type": "semantic",
"video": {
"path": video_path,
"sha256": video_hash,
"fps": meta.get("fps"),
"total_frames": meta.get("total_frames"),
"duration_s": meta.get("duration_s"),
"width": meta.get("width"),
"height": meta.get("height"),
},
"sampling_policy": {
"chunk_duration_s": chunk_duration_s,
"frames_per_chunk": frames_per_chunk,
"num_chunks": meta["num_chunks"],
"total_frames_sampled": meta["total_frames_sampled"],
"chunks": meta["chunks"],
},
"runtime": {
"vllm_base_url": semantic_vllm_url,
"model": semantic_vllm_model,
},
}
write_json(os.path.join(run_dir, "semantic_request.json"), request_record)
# Process each chunk with vision LLM
chunk_results = []
prompt = settings.semantic_prompt
for chunk_idx, chunk_frames in enumerate(chunks):
chunk_meta = meta["chunks"][chunk_idx]
# Encode frames for this chunk
t_enc = time.time()
data_urls = [
frame_to_data_url_b64jpeg(f, max_side=settings.max_side, jpeg_quality=settings.jpeg_quality)
for f in chunk_frames
]
encode_time = time.time() - t_enc
# Call vLLM for this chunk (uses semantic vLLM - Qwen3-VL)
try:
raw_resp, infer_time = call_vllm_chat(
base_url=semantic_vllm_url,
model=semantic_vllm_model,
data_urls=data_urls,
prompt=prompt,
max_tokens=settings.max_tokens,
temperature=settings.temperature,
repetition_penalty=settings.repetition_penalty,
)
except Exception as e:
raise HTTPException(status_code=500, detail=f"VLM inference failed for chunk {chunk_idx}: {str(e)}")
# Extract description
try:
chunk_description = raw_resp["choices"][0]["message"]["content"]
except Exception:
chunk_description = ""
usage_data = raw_resp.get("usage", {})
chunk_result = ChunkResult(
chunk_idx=chunk_idx,
time_range=f"{chunk_meta['start_time_s']}s - {chunk_meta['end_time_s']}s",
description=chunk_description,
frames_analyzed=len(chunk_frames),
inference_time_s=round(infer_time, 4),
usage=Usage(
prompt_tokens=int(usage_data.get("prompt_tokens", 0)),
completion_tokens=int(usage_data.get("completion_tokens", 0)),
total_tokens=int(usage_data.get("total_tokens", 0)),
),
)
chunk_results.append(chunk_result)
# Save intermediate chunk result
write_json(
os.path.join(run_dir, f"chunk_{chunk_idx:03d}_response.json"),
{
"chunk_idx": chunk_idx,
"raw_response": raw_resp,
"description": chunk_description,
"encode_time_s": round(encode_time, 4),
"inference_time_s": round(infer_time, 4),
},
)
# Aggregate chunks into final summary (if enabled)
final_summary = None
aggregation_time = None
if enable_aggregation and len(chunk_results) > 1:
t_agg = time.time()
# Build aggregation prompt
chunks_text = "\n\n".join(
[f"Segment {r.chunk_idx + 1} ({r.time_range}):\n{r.description}" for r in chunk_results]
)
aggregation_prompt = settings.aggregation_prompt_template.format(
num_chunks=len(chunk_results),
duration=meta.get("duration_s", 0),
chunks_text=chunks_text,
)
# Call text LLM for aggregation
try:
headers = {"Content-Type": "application/json"}
if settings.semantic_llm_api_key:
headers["Authorization"] = f"Bearer {settings.semantic_llm_api_key}"
async with httpx.AsyncClient(timeout=120.0) as client:
llm_resp = await client.post(
f"{settings.semantic_llm_base_url}/v1/chat/completions",
json={
"model": settings.semantic_aggregation_model,
"messages": [{"role": "user", "content": aggregation_prompt}],
"max_tokens": 1500,
"temperature": 0.3,
},
headers=headers,
)
llm_resp.raise_for_status()
llm_data = llm_resp.json()
final_summary = llm_data["choices"][0]["message"]["content"]
except Exception as e:
# Non-fatal: log warning but continue
final_summary = f"[Aggregation failed: {str(e)}]"
aggregation_time = round(time.time() - t_agg, 4)
# Save aggregation result
write_json(
os.path.join(run_dir, "aggregation.json"),
{
"prompt": aggregation_prompt,
"summary": final_summary,
"aggregation_time_s": aggregation_time,
},
)
total_latency = round(time.time() - overall_start, 4)
# Build final response
result = SemanticAnalysisResponse(
request_id=uuid.UUID(request_id),
run_dir=run_dir,
analysis_type="semantic",
video_duration_s=meta.get("duration_s"),
num_chunks=len(chunk_results),
chunk_results=chunk_results,
final_summary=final_summary,
aggregation_time_s=aggregation_time,
total_latency_s=total_latency,
meta=SemanticMeta(
fps=float(meta.get("fps", 0)),
total_frames=int(meta.get("total_frames", 0)),
duration_s=meta.get("duration_s"),
chunk_duration_s=chunk_duration_s,
frames_per_chunk=frames_per_chunk,
total_frames_sampled=meta["total_frames_sampled"],
),
)
# Save final result
write_json(os.path.join(run_dir, "semantic_result.json"), result.model_dump(mode="json"))
return result
@app.get("/v1/info")
def get_component_info():
"""
Get component information for service catalog.
Returns complete metadata about this service including:
- Resource information (component metadata)
- Available models (BusterX for deepfake detection)
- Available functions (API endpoints)
This endpoint is used by the catalog-api to aggregate service information
and by backend systems to populate the catalog database.
Returns:
dict: Component information matching catalog.resources, catalog.models,
and catalog.functions schemas.
"""
# Build resource information (maps to catalog.resources)
resource = {
"name": "Video Analysis Service",
"slug": "video-analysis",
"resource_type": "api_service",
"provider": "internal",
"base_url": f"http://didiAI-video-api:54600",
"configuration": {
"version": "0.1.0",
"port": 54600,
"external_url": settings.external_url,
"vllm_base_url": settings.vllm_base_url,
"vllm_model": settings.vllm_model,
"frames": settings.frames,
"max_side": settings.max_side,
},
"authentication": {
"type": "none",
"required": False,
},
"headers": {
"Content-Type": "multipart/form-data",
"Accept": "application/json",
},
"rate_limits": {
"enabled": False,
},
"cost_tracking": {
"enabled": False,
},
"tags": ["video", "deepfake", "semantic-analysis", "vision", "computer-vision"],
"is_active": True,
"metadata": {
"category": "computer-vision",
"gpu_required": True,
"status": "healthy",
"runs_dir": settings.runs_dir,
},
}
# Define models (maps to catalog.models)
models = [
{
"name": "BusterX Deepfake Detector",
"slug": "busterx",
"provider": "l8cv",
"model_type": "vision",
"capabilities": [
"deepfake-detection",
"video-analysis",
"multimodal",
],
"configuration": {
"backend": "vllm",
"base_model": "Qwen2.5-VL-7B",
"max_tokens": settings.max_tokens,
"temperature": settings.temperature,
},
"endpoint": settings.vllm_base_url,
"api_key_ref": None,
"tags": ["deepfake", "vision", "busterx", "video", "7b"],
"is_active": True,
"metadata": {
"model_id": settings.vllm_model,
"vllm_base_url": settings.vllm_base_url,
"gpu_id": 1,
"vram_gb": 22,
"specialized": "deepfake-detection",
},
}
]
# Define available functions (maps to catalog.functions)
functions = [
{
"name": "Video Deepfake Analysis",
"slug": "video-deepfake-analysis",
"category": "deepfake-detection",
"description": (
"Analyze video for deepfake manipulation using BusterX model. "
"Samples 16 frames uniformly and returns verdict: REAL/FAKE/UNCERTAIN."
),
"input_schema": {
"type": "object",
"properties": {
"file": {
"type": "file",
"description": "Video file to analyze (mp4, avi, mov, etc.)",
},
},
"required": ["file"],
},
"output_schema": {
"type": "object",
"properties": {
"request_id": {"type": "string", "format": "uuid"},
"run_dir": {"type": "string"},
"verdict": {
"type": "string",
"enum": ["REAL", "FAKE", "UNCERTAIN"],
},
"explanation": {"type": "string"},
"frames_analyzed": {"type": "integer"},
"evidence": {
"type": "array",
"items": {
"type": "object",
"properties": {
"frame_index": {"type": "integer"},
"timestamp_s": {"type": "number"},
},
},
},
"usage": {
"type": "object",
"properties": {
"prompt_tokens": {"type": "integer"},
"completion_tokens": {"type": "integer"},
"total_tokens": {"type": "integer"},
},
},
"latency_s": {
"type": "object",
"properties": {
"sampling_time_s": {"type": "number"},
"encode_time_s": {"type": "number"},
"model_inference_time_s": {"type": "number"},
},
},
"meta": {
"type": "object",
"properties": {
"fps": {"type": "number"},
"total_frames": {"type": "integer"},
"duration_s": {"type": "number"},
"sampled": {"type": "integer"},
},
},
},
},
"implementation": {
"method": "POST",
"path": "/analyze/video",
"content_type": "multipart/form-data",
"timeout": 120,
},
"endpoint": "http://localhost:8007/analyze/video",
"tags": ["video", "deepfake", "detection", "fast"],
"is_active": True,
"metadata": {
"average_latency_s": 13,
"frames_analyzed": 16,
"coverage": "~0.89% of frames",
},
},
{
"name": "Video Semantic Analysis",
"slug": "video-semantic-analysis",
"category": "semantic-analysis",
"description": (
"Deep semantic analysis of video content with temporal chunking. "
"Divides video into chunks, analyzes each with vision LLM, and synthesizes narrative."
),
"input_schema": {
"type": "object",
"properties": {
"file": {
"type": "file",
"description": "Video file to analyze",
},
"chunk_duration_s": {
"type": "number",
"minimum": 1.0,
"maximum": 60.0,
"default": 10.0,
"description": "Duration of each chunk in seconds",
},
"frames_per_chunk": {
"type": "integer",
"minimum": 4,
"maximum": 64,
"default": 24,
"description": "Frames to sample per chunk",
},
"enable_aggregation": {
"type": "boolean",
"default": True,
"description": "Aggregate chunks into final summary",
},
},
"required": ["file"],
},
"output_schema": {
"type": "object",
"properties": {
"request_id": {"type": "string"},
"num_chunks": {"type": "integer"},
"chunk_results": {
"type": "array",
"items": {
"type": "object",
"properties": {
"chunk_idx": {"type": "integer"},
"time_range": {"type": "string"},
"description": {"type": "string"},
"frames_analyzed": {"type": "integer"},
},
},
},
"final_summary": {"type": "string"},
"total_latency_s": {"type": "number"},
},
},
"implementation": {
"method": "POST",
"path": "/analyze/video/semantic",
"content_type": "multipart/form-data",
"timeout": 300,
},
"endpoint": "http://localhost:8007/analyze/video/semantic",
"tags": ["video", "semantic", "analysis", "detailed"],
"is_active": True,
"metadata": {
"average_latency_s": 77,
"coverage": "~8% of frames",
"use_case": "content-description",
},
},
]
return {
"resource": resource,
"models": models,
"functions": functions,
}