Livrare LOT 1 - Didi

ai_platform/shared/observability.py

@@ -0,0 +1,193 @@
+"""
+Prometheus metrics + OpenTelemetry tracing setup for DiDi AI Platform modules.
+
+Usage in FastAPI app.py:
+
+    from shared.observability import init_observability, mount_metrics
+
+    app = FastAPI(lifespan=...)
+    init_observability(app, service_name="llm-inference", version="0.1.0")
+    mount_metrics(app)
+
+This:
+  - Mounts /metrics endpoint (Prometheus scrape target)
+  - Instruments FastAPI with OpenTelemetry (auto-traces requests)
+  - Sets up resource attributes (service.name, service.version, cluster)
+  - Configures OTLP exporter pointing to OTEL_EXPORTER_OTLP_ENDPOINT
+
+Optional: import custom metrics:
+
+    from shared.observability import (
+        llm_calls_total, llm_tokens_total, inference_duration_seconds
+    )
+    llm_calls_total.labels(provider="vllm", model="qwen3.5", status="ok").inc()
+
+Required pip deps (add to each module's pyproject.toml):
+    prometheus-client>=0.20.0
+    opentelemetry-api>=1.25.0
+    opentelemetry-sdk>=1.25.0
+    opentelemetry-exporter-otlp-proto-grpc>=1.25.0
+    opentelemetry-instrumentation-fastapi>=0.46b0
+    opentelemetry-instrumentation-httpx>=0.46b0
+"""
+from __future__ import annotations
+
+import os
+import logging
+from typing import Optional
+
+from prometheus_client import (
+    Counter,
+    Histogram,
+    Gauge,
+    CollectorRegistry,
+    generate_latest,
+    CONTENT_TYPE_LATEST,
+    REGISTRY,
+)
+
+logger = logging.getLogger(__name__)
+
+_initialized = False
+
+
+def init_observability(
+    app,
+    service_name: str,
+    version: str = "0.1.0",
+    enable_tracing: bool = True,
+) -> None:
+    """Initialize OpenTelemetry tracing + auto-instrument FastAPI.
+
+    Safe to call multiple times — no-op on subsequent calls.
+    Fail-open: if OTel deps missing, logs warning and continues without tracing.
+    """
+    global _initialized
+    if _initialized:
+        return
+    _initialized = True
+
+    otel_endpoint = os.environ.get("OTEL_EXPORTER_OTLP_ENDPOINT")
+    if not enable_tracing or not otel_endpoint:
+        logger.info(
+            "[observability] tracing disabled (OTEL_EXPORTER_OTLP_ENDPOINT not set or disabled)"
+        )
+        return
+
+    try:
+        from opentelemetry import trace
+        from opentelemetry.sdk.trace import TracerProvider
+        from opentelemetry.sdk.trace.export import BatchSpanProcessor
+        from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
+        from opentelemetry.sdk.resources import Resource
+        from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
+
+        resource = Resource(
+            attributes={
+                "service.name": service_name,
+                "service.version": version,
+                "cluster": os.environ.get("OTEL_CLUSTER", "didi-prod"),
+                "environment": os.environ.get("OTEL_ENVIRONMENT", "production"),
+            }
+        )
+        provider = TracerProvider(resource=resource)
+        provider.add_span_processor(
+            BatchSpanProcessor(OTLPSpanExporter(endpoint=otel_endpoint, insecure=True))
+        )
+        trace.set_tracer_provider(provider)
+
+        FastAPIInstrumentor.instrument_app(app)
+
+        # Optional: instrument httpx clients automatically
+        try:
+            from opentelemetry.instrumentation.httpx import HTTPXClientInstrumentor
+
+            HTTPXClientInstrumentor().instrument()
+        except ImportError:
+            pass
+
+        logger.info(f"[observability] OTel tracing initialized for {service_name}")
+    except ImportError as e:
+        logger.warning(
+            f"[observability] OTel deps missing — tracing disabled. {e}. "
+            f"Install: pip install opentelemetry-api opentelemetry-sdk "
+            f"opentelemetry-exporter-otlp-proto-grpc opentelemetry-instrumentation-fastapi"
+        )
+
+
+def mount_metrics(app, registry: Optional[CollectorRegistry] = None) -> None:
+    """Mount /metrics endpoint on FastAPI app for Prometheus scraping."""
+    from fastapi import Response
+
+    reg = registry or REGISTRY
+
+    @app.get("/metrics", include_in_schema=False)
+    async def _metrics():
+        return Response(content=generate_latest(reg), media_type=CONTENT_TYPE_LATEST)
+
+
+# ============================================================================
+# Standard DiDi AI Platform metrics
+# ============================================================================
+
+llm_calls_total = Counter(
+    "didi_llm_calls_total",
+    "Total LLM calls made (per provider+model+status)",
+    labelnames=["provider", "model", "status"],
+)
+
+llm_tokens_total = Counter(
+    "didi_llm_tokens_total",
+    "Total LLM tokens consumed",
+    labelnames=["provider", "model", "type"],  # type=prompt|completion
+)
+
+inference_duration_seconds = Histogram(
+    "didi_inference_duration_seconds",
+    "Inference duration in seconds (per backend+model)",
+    labelnames=["backend", "model"],
+    buckets=(0.1, 0.5, 1, 2, 5, 10, 30, 60, 120),
+)
+
+http_requests_total = Counter(
+    "didi_ai_http_requests_total",
+    "HTTP requests received by AI platform service",
+    labelnames=["method", "route", "status"],
+)
+
+active_backends = Gauge(
+    "didi_active_backends",
+    "Number of active/healthy backends per type",
+    labelnames=["backend_type"],
+)
+
+cache_hits_total = Counter(
+    "didi_ai_cache_hits_total",
+    "Cache hits per cache type",
+    labelnames=["cache"],
+)
+
+cache_misses_total = Counter(
+    "didi_ai_cache_misses_total",
+    "Cache misses per cache type",
+    labelnames=["cache"],
+)
+
+# Brain-specific (only used by didi_brain module)
+brain_atoms_total = Gauge(
+    "didi_brain_atoms_total",
+    "Total brain analysis atoms by tier",
+    labelnames=["tier"],
+)
+
+brain_facts_total = Gauge(
+    "didi_brain_facts_total",
+    "Total brain facts by current_truth",
+    labelnames=["truth"],
+)
+
+brain_judge_decisions_total = Counter(
+    "didi_brain_judge_decisions_total",
+    "NLI judge decisions (KEEP/INVALIDATE/RECHECK)",
+    labelnames=["decision"],
+)