import hashlib import json import logging import os import time import uuid from collections.abc import AsyncGenerator from contextlib import asynccontextmanager from datetime import datetime, timezone import httpx from fastapi import FastAPI, File, Form, HTTPException, UploadFile from fastapi.responses import JSONResponse from .buster_client import ( build_frame_evidence, call_vllm_chat, frame_to_data_url_b64jpeg, parse_verdict_and_explanation, ) from .runtime_config import RuntimeConfigClient from .schemas import AnalyzeResponse, ChunkResult, SemanticAnalysisResponse, SemanticMeta, Usage from .settings import settings from .video_sampling import sample_frames_chunked, sample_frames_uniform logger = logging.getLogger(__name__) # Runtime config (reachable from handlers via app.state.runtime_config) runtime_config = RuntimeConfigClient( dashboard_url=settings.dashboard_url, live_log_logger_name="video_analysis", live_log_key="video.log.level", ) @asynccontextmanager async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]: """App lifespan: start runtime config polling.""" logger.info("Starting Video Analysis API") await runtime_config.start() app.state.runtime_config = runtime_config yield logger.info("Shutting down Video Analysis API") await runtime_config.stop() app = FastAPI( title="Video Analysis API", description="Deepfake detection and semantic video analysis", version="0.1.0", servers=[{"url": settings.external_url, "description": "Video Analysis API"}], lifespan=lifespan, ) # ---------------------------------------------------------------- observability # Prometheus /metrics + OTel tracing (no-op if deps missing or OTEL endpoint unset) try: from prometheus_fastapi_instrumentator import Instrumentator as _Inst # type: ignore _Inst(should_group_status_codes=True).instrument(app).expose(app, endpoint="/metrics", include_in_schema=False) except ImportError: pass import os as _os # noqa: E402 _otel_ep = _os.environ.get("OTEL_EXPORTER_OTLP_ENDPOINT") if _otel_ep: try: from opentelemetry import trace as _trace # type: ignore from opentelemetry.sdk.resources import Resource as _R # type: ignore from opentelemetry.sdk.trace import TracerProvider as _TP # type: ignore from opentelemetry.sdk.trace.export import BatchSpanProcessor as _BSP # type: ignore from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter as _Exp # type: ignore from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor as _FInst # type: ignore _provider = _TP(resource=_R.create({"service.name": _os.environ.get("OTEL_SERVICE_NAME", "didiAI-video-api")})) _provider.add_span_processor(_BSP(_Exp(endpoint=_otel_ep, insecure=True))) _trace.set_tracer_provider(_provider) _FInst.instrument_app(app) print(f"[otel] didiAI-video-api instrumented -> {_otel_ep}") except ImportError as _e: print(f"[otel] skip: {_e}") def safe_mkdir(path: str) -> None: os.makedirs(path, exist_ok=True) def write_json(path: str, obj) -> None: with open(path, "w", encoding="utf-8") as f: json.dump(obj, f, ensure_ascii=False, indent=2) def sha256_file(path: str, chunk_size: int = 8 * 1024 * 1024) -> str: h = hashlib.sha256() with open(path, "rb") as f: while True: b = f.read(chunk_size) if not b: break h.update(b) return h.hexdigest() @app.get("/health") def health(): return {"status": "ok"} @app.post("/analyze/video", response_model=AnalyzeResponse) def analyze_video(file: UploadFile = File(...)): # Defensive (Settings already requires it) if not settings.vllm_base_url: raise HTTPException(status_code=500, detail="VIDEO_ANALYSIS_VLLM_BASE_URL is not set") request_id = str(uuid.uuid4()) started_at = datetime.now(timezone.utc).isoformat() # Make run_dir absolute under runs_dir (inside container: /app/runs/) run_dir = os.path.join(settings.runs_dir, request_id) safe_mkdir(run_dir) # Save upload to disk for reproducibility video_path = os.path.join(run_dir, file.filename or "upload.mp4") t0 = time.time() with open(video_path, "wb") as out: out.write(file.file.read()) save_time_s = round(time.time() - t0, 4) video_hash = sha256_file(video_path) prompt = settings.analysis_prompt # Sampling frames, meta = sample_frames_uniform(video_path, num_frames=settings.frames) # Encode frames t_enc0 = time.time() data_urls = [ frame_to_data_url_b64jpeg(f, max_side=settings.max_side, jpeg_quality=settings.jpeg_quality) for f in frames ] encode_time_s = round(time.time() - t_enc0, 4) request_record = { "request_id": request_id, "started_at_utc": started_at, "video": { "path": video_path, "sha256": video_hash, "fps": meta.get("fps"), "total_frames": meta.get("total_frames"), "duration_s": meta.get("duration_s"), "width": meta.get("width"), "height": meta.get("height"), }, "sampling_policy": { "frames_requested": meta.get("frames_requested"), "sampled": meta.get("sampled"), "indices": meta.get("indices"), "timestamps_s": meta.get("timestamps_s"), "sampling_method": meta.get("sampling_method"), "max_side": settings.max_side, "jpeg_quality": settings.jpeg_quality, }, "runtime": { "vllm_base_url": settings.vllm_base_url, "model": settings.vllm_model, }, "generation_params": { "max_tokens": settings.max_tokens, "temperature": settings.temperature, "repetition_penalty": settings.repetition_penalty, }, "latency_s": {"upload_save_time_s": save_time_s}, "prompt": prompt, } write_json(os.path.join(run_dir, "request.json"), request_record) # Call vLLM raw_resp, infer_time_s = call_vllm_chat( base_url=settings.vllm_base_url, model=settings.vllm_model, data_urls=data_urls, prompt=prompt, max_tokens=settings.max_tokens, temperature=settings.temperature, repetition_penalty=settings.repetition_penalty, ) write_json(os.path.join(run_dir, "raw_response.json"), raw_resp) try: model_text = raw_resp["choices"][0]["message"]["content"] except Exception: model_text = "" parsed = parse_verdict_and_explanation(model_text) usage = raw_resp.get("usage") or {} indices = meta.get("indices") or [] timestamps = meta.get("timestamps_s") or [] evidence = build_frame_evidence(indices, timestamps) result_record = { "request_id": request_id, "run_dir": run_dir, "verdict": parsed["verdict"], "explanation": parsed["explanation"], "frames_analyzed": int(meta.get("sampled") or len(indices)), "evidence": evidence, "usage": { "prompt_tokens": int(usage.get("prompt_tokens") or 0), "completion_tokens": int(usage.get("completion_tokens") or 0), "total_tokens": int(usage.get("total_tokens") or 0), }, "latency_s": { "sampling_time_s": float(meta.get("sampling_time_s") or 0.0), "encode_time_s": encode_time_s, "model_inference_time_s": round(infer_time_s, 4), }, "meta": { "fps": float(meta.get("fps") or 0.0), "total_frames": int(meta.get("total_frames") or 0), "duration_s": meta.get("duration_s"), "sampled": int(meta.get("sampled") or 0), "indices": meta.get("indices") or [], "timestamps_s": meta.get("timestamps_s") or [], }, } write_json(os.path.join(run_dir, "result.json"), result_record) return JSONResponse(content=result_record) @app.post("/analyze/video/semantic", response_model=SemanticAnalysisResponse) async def analyze_video_semantic( file: UploadFile = File(...), chunk_duration_s: float = Form(default=None), frames_per_chunk: int = Form(default=None), enable_aggregation: bool = Form(default=None), ): """ Semantic video analysis with dense temporal sampling. This endpoint performs deep content understanding by: 1. Dividing video into temporal chunks 2. Densely sampling each chunk (24 frames per 10s default) 3. Analyzing each chunk with vision LLM 4. Optionally aggregating into coherent narrative Use this for: - Content understanding and description - Action recognition - Scene analysis - Narrative extraction Args: file: Video file to analyze chunk_duration_s: Duration of each chunk in seconds (default from settings) frames_per_chunk: Frames to sample per chunk (default from settings) enable_aggregation: Whether to aggregate chunks into summary (default from settings) Returns: SemanticAnalysisResponse with chunk descriptions and optional summary """ # Use settings defaults if not provided chunk_duration_s = chunk_duration_s or settings.semantic_chunk_duration_s frames_per_chunk = frames_per_chunk or settings.semantic_frames_per_chunk enable_aggregation = enable_aggregation if enable_aggregation is not None else settings.semantic_enable_aggregation # Use dedicated semantic vLLM if configured, otherwise fall back to deepfake vLLM semantic_vllm_url = settings.semantic_vllm_base_url or settings.vllm_base_url semantic_vllm_model = settings.semantic_vllm_model or settings.vllm_model if not semantic_vllm_url: raise HTTPException(status_code=500, detail="VIDEO_ANALYSIS_SEMANTIC_VLLM_BASE_URL or VIDEO_ANALYSIS_VLLM_BASE_URL is not set") request_id = str(uuid.uuid4()) started_at = datetime.now(timezone.utc).isoformat() overall_start = time.time() # Create run directory run_dir = os.path.join(settings.runs_dir, request_id) safe_mkdir(run_dir) # Save uploaded video video_path = os.path.join(run_dir, file.filename or "upload.mp4") with open(video_path, "wb") as out: out.write(file.file.read()) video_hash = sha256_file(video_path) # Sample video in chunks try: chunks, meta = sample_frames_chunked( video_path, chunk_duration_s=chunk_duration_s, frames_per_chunk=frames_per_chunk, ) except Exception as e: raise HTTPException(status_code=400, detail=f"Failed to sample video: {str(e)}") # Save request metadata request_record = { "request_id": request_id, "started_at_utc": started_at, "analysis_type": "semantic", "video": { "path": video_path, "sha256": video_hash, "fps": meta.get("fps"), "total_frames": meta.get("total_frames"), "duration_s": meta.get("duration_s"), "width": meta.get("width"), "height": meta.get("height"), }, "sampling_policy": { "chunk_duration_s": chunk_duration_s, "frames_per_chunk": frames_per_chunk, "num_chunks": meta["num_chunks"], "total_frames_sampled": meta["total_frames_sampled"], "chunks": meta["chunks"], }, "runtime": { "vllm_base_url": semantic_vllm_url, "model": semantic_vllm_model, }, } write_json(os.path.join(run_dir, "semantic_request.json"), request_record) # Process each chunk with vision LLM chunk_results = [] prompt = settings.semantic_prompt for chunk_idx, chunk_frames in enumerate(chunks): chunk_meta = meta["chunks"][chunk_idx] # Encode frames for this chunk t_enc = time.time() data_urls = [ frame_to_data_url_b64jpeg(f, max_side=settings.max_side, jpeg_quality=settings.jpeg_quality) for f in chunk_frames ] encode_time = time.time() - t_enc # Call vLLM for this chunk (uses semantic vLLM - Qwen3-VL) try: raw_resp, infer_time = call_vllm_chat( base_url=semantic_vllm_url, model=semantic_vllm_model, data_urls=data_urls, prompt=prompt, max_tokens=settings.max_tokens, temperature=settings.temperature, repetition_penalty=settings.repetition_penalty, ) except Exception as e: raise HTTPException(status_code=500, detail=f"VLM inference failed for chunk {chunk_idx}: {str(e)}") # Extract description try: chunk_description = raw_resp["choices"][0]["message"]["content"] except Exception: chunk_description = "" usage_data = raw_resp.get("usage", {}) chunk_result = ChunkResult( chunk_idx=chunk_idx, time_range=f"{chunk_meta['start_time_s']}s - {chunk_meta['end_time_s']}s", description=chunk_description, frames_analyzed=len(chunk_frames), inference_time_s=round(infer_time, 4), usage=Usage( prompt_tokens=int(usage_data.get("prompt_tokens", 0)), completion_tokens=int(usage_data.get("completion_tokens", 0)), total_tokens=int(usage_data.get("total_tokens", 0)), ), ) chunk_results.append(chunk_result) # Save intermediate chunk result write_json( os.path.join(run_dir, f"chunk_{chunk_idx:03d}_response.json"), { "chunk_idx": chunk_idx, "raw_response": raw_resp, "description": chunk_description, "encode_time_s": round(encode_time, 4), "inference_time_s": round(infer_time, 4), }, ) # Aggregate chunks into final summary (if enabled) final_summary = None aggregation_time = None if enable_aggregation and len(chunk_results) > 1: t_agg = time.time() # Build aggregation prompt chunks_text = "\n\n".join( [f"Segment {r.chunk_idx + 1} ({r.time_range}):\n{r.description}" for r in chunk_results] ) aggregation_prompt = settings.aggregation_prompt_template.format( num_chunks=len(chunk_results), duration=meta.get("duration_s", 0), chunks_text=chunks_text, ) # Call text LLM for aggregation try: headers = {"Content-Type": "application/json"} if settings.semantic_llm_api_key: headers["Authorization"] = f"Bearer {settings.semantic_llm_api_key}" async with httpx.AsyncClient(timeout=120.0) as client: llm_resp = await client.post( f"{settings.semantic_llm_base_url}/v1/chat/completions", json={ "model": settings.semantic_aggregation_model, "messages": [{"role": "user", "content": aggregation_prompt}], "max_tokens": 1500, "temperature": 0.3, }, headers=headers, ) llm_resp.raise_for_status() llm_data = llm_resp.json() final_summary = llm_data["choices"][0]["message"]["content"] except Exception as e: # Non-fatal: log warning but continue final_summary = f"[Aggregation failed: {str(e)}]" aggregation_time = round(time.time() - t_agg, 4) # Save aggregation result write_json( os.path.join(run_dir, "aggregation.json"), { "prompt": aggregation_prompt, "summary": final_summary, "aggregation_time_s": aggregation_time, }, ) total_latency = round(time.time() - overall_start, 4) # Build final response result = SemanticAnalysisResponse( request_id=uuid.UUID(request_id), run_dir=run_dir, analysis_type="semantic", video_duration_s=meta.get("duration_s"), num_chunks=len(chunk_results), chunk_results=chunk_results, final_summary=final_summary, aggregation_time_s=aggregation_time, total_latency_s=total_latency, meta=SemanticMeta( fps=float(meta.get("fps", 0)), total_frames=int(meta.get("total_frames", 0)), duration_s=meta.get("duration_s"), chunk_duration_s=chunk_duration_s, frames_per_chunk=frames_per_chunk, total_frames_sampled=meta["total_frames_sampled"], ), ) # Save final result write_json(os.path.join(run_dir, "semantic_result.json"), result.model_dump(mode="json")) return result @app.get("/v1/info") def get_component_info(): """ Get component information for service catalog. Returns complete metadata about this service including: - Resource information (component metadata) - Available models (BusterX for deepfake detection) - Available functions (API endpoints) This endpoint is used by the catalog-api to aggregate service information and by backend systems to populate the catalog database. Returns: dict: Component information matching catalog.resources, catalog.models, and catalog.functions schemas. """ # Build resource information (maps to catalog.resources) resource = { "name": "Video Analysis Service", "slug": "video-analysis", "resource_type": "api_service", "provider": "internal", "base_url": f"http://didiAI-video-api:54600", "configuration": { "version": "0.1.0", "port": 54600, "external_url": settings.external_url, "vllm_base_url": settings.vllm_base_url, "vllm_model": settings.vllm_model, "frames": settings.frames, "max_side": settings.max_side, }, "authentication": { "type": "none", "required": False, }, "headers": { "Content-Type": "multipart/form-data", "Accept": "application/json", }, "rate_limits": { "enabled": False, }, "cost_tracking": { "enabled": False, }, "tags": ["video", "deepfake", "semantic-analysis", "vision", "computer-vision"], "is_active": True, "metadata": { "category": "computer-vision", "gpu_required": True, "status": "healthy", "runs_dir": settings.runs_dir, }, } # Define models (maps to catalog.models) models = [ { "name": "BusterX Deepfake Detector", "slug": "busterx", "provider": "l8cv", "model_type": "vision", "capabilities": [ "deepfake-detection", "video-analysis", "multimodal", ], "configuration": { "backend": "vllm", "base_model": "Qwen2.5-VL-7B", "max_tokens": settings.max_tokens, "temperature": settings.temperature, }, "endpoint": settings.vllm_base_url, "api_key_ref": None, "tags": ["deepfake", "vision", "busterx", "video", "7b"], "is_active": True, "metadata": { "model_id": settings.vllm_model, "vllm_base_url": settings.vllm_base_url, "gpu_id": 1, "vram_gb": 22, "specialized": "deepfake-detection", }, } ] # Define available functions (maps to catalog.functions) functions = [ { "name": "Video Deepfake Analysis", "slug": "video-deepfake-analysis", "category": "deepfake-detection", "description": ( "Analyze video for deepfake manipulation using BusterX model. " "Samples 16 frames uniformly and returns verdict: REAL/FAKE/UNCERTAIN." ), "input_schema": { "type": "object", "properties": { "file": { "type": "file", "description": "Video file to analyze (mp4, avi, mov, etc.)", }, }, "required": ["file"], }, "output_schema": { "type": "object", "properties": { "request_id": {"type": "string", "format": "uuid"}, "run_dir": {"type": "string"}, "verdict": { "type": "string", "enum": ["REAL", "FAKE", "UNCERTAIN"], }, "explanation": {"type": "string"}, "frames_analyzed": {"type": "integer"}, "evidence": { "type": "array", "items": { "type": "object", "properties": { "frame_index": {"type": "integer"}, "timestamp_s": {"type": "number"}, }, }, }, "usage": { "type": "object", "properties": { "prompt_tokens": {"type": "integer"}, "completion_tokens": {"type": "integer"}, "total_tokens": {"type": "integer"}, }, }, "latency_s": { "type": "object", "properties": { "sampling_time_s": {"type": "number"}, "encode_time_s": {"type": "number"}, "model_inference_time_s": {"type": "number"}, }, }, "meta": { "type": "object", "properties": { "fps": {"type": "number"}, "total_frames": {"type": "integer"}, "duration_s": {"type": "number"}, "sampled": {"type": "integer"}, }, }, }, }, "implementation": { "method": "POST", "path": "/analyze/video", "content_type": "multipart/form-data", "timeout": 120, }, "endpoint": "http://localhost:8007/analyze/video", "tags": ["video", "deepfake", "detection", "fast"], "is_active": True, "metadata": { "average_latency_s": 13, "frames_analyzed": 16, "coverage": "~0.89% of frames", }, }, { "name": "Video Semantic Analysis", "slug": "video-semantic-analysis", "category": "semantic-analysis", "description": ( "Deep semantic analysis of video content with temporal chunking. " "Divides video into chunks, analyzes each with vision LLM, and synthesizes narrative." ), "input_schema": { "type": "object", "properties": { "file": { "type": "file", "description": "Video file to analyze", }, "chunk_duration_s": { "type": "number", "minimum": 1.0, "maximum": 60.0, "default": 10.0, "description": "Duration of each chunk in seconds", }, "frames_per_chunk": { "type": "integer", "minimum": 4, "maximum": 64, "default": 24, "description": "Frames to sample per chunk", }, "enable_aggregation": { "type": "boolean", "default": True, "description": "Aggregate chunks into final summary", }, }, "required": ["file"], }, "output_schema": { "type": "object", "properties": { "request_id": {"type": "string"}, "num_chunks": {"type": "integer"}, "chunk_results": { "type": "array", "items": { "type": "object", "properties": { "chunk_idx": {"type": "integer"}, "time_range": {"type": "string"}, "description": {"type": "string"}, "frames_analyzed": {"type": "integer"}, }, }, }, "final_summary": {"type": "string"}, "total_latency_s": {"type": "number"}, }, }, "implementation": { "method": "POST", "path": "/analyze/video/semantic", "content_type": "multipart/form-data", "timeout": 300, }, "endpoint": "http://localhost:8007/analyze/video/semantic", "tags": ["video", "semantic", "analysis", "detailed"], "is_active": True, "metadata": { "average_latency_s": 77, "coverage": "~8% of frames", "use_case": "content-description", }, }, ] return { "resource": resource, "models": models, "functions": functions, }