Livrare LOT 1 - Didi
This commit is contained in:
commit
5380c3fc63
990 changed files with 133308 additions and 0 deletions
|
|
@ -0,0 +1,5 @@
|
|||
"""FastAPI application for LLM inference."""
|
||||
|
||||
from llm_inference.api.app import create_app
|
||||
|
||||
__all__ = ["create_app"]
|
||||
165
ai_platform/modules/llm-inference/src/llm_inference/api/app.py
Normal file
165
ai_platform/modules/llm-inference/src/llm_inference/api/app.py
Normal file
|
|
@ -0,0 +1,165 @@
|
|||
"""FastAPI application factory."""
|
||||
|
||||
from collections.abc import AsyncGenerator
|
||||
from contextlib import asynccontextmanager
|
||||
|
||||
from fastapi import FastAPI
|
||||
|
||||
from llm_inference.api.dependencies import init_concurrency_limiter
|
||||
from llm_inference.api.middleware import RateLimitMiddleware, RequestIdMiddleware
|
||||
from llm_inference.api.routes import completions, health, info, models
|
||||
from llm_inference.backends.llamacpp_backend import LlamaCppBackend
|
||||
from llm_inference.client import LLMClient
|
||||
from llm_inference.config import SettingsCache
|
||||
from llm_inference.image_processing import close_http_client
|
||||
from llm_inference.logging import configure_logging, get_logger
|
||||
from llm_inference.runtime_config import RuntimeConfigClient
|
||||
from llm_inference.types import BackendType
|
||||
|
||||
logger = get_logger("app")
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
"""Application lifespan manager.
|
||||
|
||||
Initializes the LLM client, settings, logging, and concurrency limiter
|
||||
on startup, and performs cleanup on shutdown.
|
||||
|
||||
Args:
|
||||
app: FastAPI application instance.
|
||||
|
||||
Yields:
|
||||
None: Control to the application.
|
||||
"""
|
||||
settings = SettingsCache.get()
|
||||
configure_logging(settings.log_level, settings.log_json)
|
||||
|
||||
logger.info("Starting LLM Inference API")
|
||||
logger.info("Default backend: %s", settings.default_backend)
|
||||
logger.info("vLLM enabled: %s", settings.enable_vllm)
|
||||
logger.info("llama.cpp enabled: %s", settings.enable_llamacpp)
|
||||
|
||||
init_concurrency_limiter(settings.max_concurrent_completions)
|
||||
|
||||
app.state.settings = settings
|
||||
app.state.client = LLMClient(settings)
|
||||
|
||||
# Start the runtime config client created in create_app()
|
||||
if hasattr(app.state, "runtime_config"):
|
||||
await app.state.runtime_config.start()
|
||||
|
||||
if settings.enable_llamacpp:
|
||||
try:
|
||||
backend = app.state.client.registry.get(BackendType.LLAMACPP)
|
||||
if isinstance(backend, LlamaCppBackend):
|
||||
backend.start_health_checks()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
logger.info("LLM Inference API started successfully")
|
||||
|
||||
yield
|
||||
|
||||
logger.info("Shutting down LLM Inference API")
|
||||
if hasattr(app.state, "runtime_config"):
|
||||
await app.state.runtime_config.stop()
|
||||
await close_http_client()
|
||||
|
||||
|
||||
def create_app() -> FastAPI:
|
||||
"""Create and configure the FastAPI application.
|
||||
|
||||
Returns:
|
||||
FastAPI: Configured FastAPI application.
|
||||
"""
|
||||
settings = SettingsCache.get()
|
||||
|
||||
app = FastAPI(
|
||||
title="LLM Inference API",
|
||||
description=(
|
||||
"Unified LLM inference with multiple backends (LiteLLM, vLLM, llama.cpp)"
|
||||
),
|
||||
version="0.1.0",
|
||||
lifespan=lifespan,
|
||||
servers=[{"url": settings.external_url, "description": "LLM Inference API"}],
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- observability
|
||||
|
||||
# Prometheus /metrics + OTel tracing (no-op if deps missing or OTEL endpoint unset)
|
||||
|
||||
try:
|
||||
|
||||
from prometheus_fastapi_instrumentator import Instrumentator as _Inst # type: ignore
|
||||
|
||||
_Inst(should_group_status_codes=True).instrument(app).expose(app, endpoint="/metrics", include_in_schema=False)
|
||||
|
||||
except ImportError:
|
||||
|
||||
pass
|
||||
|
||||
import os as _os # noqa: E402
|
||||
|
||||
_otel_ep = _os.environ.get("OTEL_EXPORTER_OTLP_ENDPOINT")
|
||||
|
||||
if _otel_ep:
|
||||
|
||||
try:
|
||||
|
||||
from opentelemetry import trace as _trace # type: ignore
|
||||
|
||||
from opentelemetry.sdk.resources import Resource as _R # type: ignore
|
||||
|
||||
from opentelemetry.sdk.trace import TracerProvider as _TP # type: ignore
|
||||
|
||||
from opentelemetry.sdk.trace.export import BatchSpanProcessor as _BSP # type: ignore
|
||||
|
||||
from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter as _Exp # type: ignore
|
||||
|
||||
from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor as _FInst # type: ignore
|
||||
|
||||
_provider = _TP(resource=_R.create({"service.name": _os.environ.get("OTEL_SERVICE_NAME", "didiAI-llm-api")}))
|
||||
|
||||
_provider.add_span_processor(_BSP(_Exp(endpoint=_otel_ep, insecure=True)))
|
||||
|
||||
_trace.set_tracer_provider(_provider)
|
||||
|
||||
_FInst.instrument_app(app)
|
||||
|
||||
print(f"[otel] didiAI-llm-api instrumented -> {_otel_ep}")
|
||||
|
||||
except ImportError as _e:
|
||||
|
||||
print(f"[otel] skip: {_e}")
|
||||
|
||||
|
||||
# Runtime config client — instantiated here so middleware can capture it.
|
||||
# Started/stopped inside lifespan().
|
||||
runtime_config = RuntimeConfigClient(
|
||||
dashboard_url=settings.dashboard_url,
|
||||
live_log_logger_name="llm_inference",
|
||||
live_log_key="llm.log.level",
|
||||
)
|
||||
app.state.runtime_config = runtime_config
|
||||
|
||||
# Add middleware (order matters - first added is outermost)
|
||||
app.add_middleware(
|
||||
RateLimitMiddleware,
|
||||
rate=settings.rate_limit_rps,
|
||||
burst=settings.rate_limit_burst,
|
||||
exclude_paths=["/health", "/ready"],
|
||||
runtime_config=runtime_config,
|
||||
rate_key="llm.rate_limit.rps",
|
||||
burst_key="llm.rate_limit.burst",
|
||||
)
|
||||
|
||||
app.add_middleware(RequestIdMiddleware)
|
||||
|
||||
app.include_router(health.router, tags=["Health"])
|
||||
app.include_router(completions.router, prefix="/v1", tags=["Completions"])
|
||||
app.include_router(models.router, prefix="/v1", tags=["Models"])
|
||||
app.include_router(info.router, tags=["Catalog"])
|
||||
|
||||
return app
|
||||
|
|
@ -0,0 +1,232 @@
|
|||
"""FastAPI dependencies for LLM inference."""
|
||||
|
||||
import asyncio
|
||||
import hmac
|
||||
from collections.abc import AsyncIterator
|
||||
from contextlib import asynccontextmanager
|
||||
|
||||
from fastapi import Header, HTTPException, Request
|
||||
|
||||
from llm_inference.client import LLMClient
|
||||
from llm_inference.config import LLMSettings
|
||||
from llm_inference.logging import get_logger
|
||||
|
||||
logger = get_logger("dependencies")
|
||||
|
||||
|
||||
def get_client(request: Request) -> LLMClient:
|
||||
"""Get the LLM client from application state.
|
||||
|
||||
Args:
|
||||
request: FastAPI request object.
|
||||
|
||||
Returns:
|
||||
LLMClient: The LLM client instance.
|
||||
"""
|
||||
return request.app.state.client
|
||||
|
||||
|
||||
def get_settings(request: Request) -> LLMSettings:
|
||||
"""Get settings from application state.
|
||||
|
||||
Args:
|
||||
request: FastAPI request object.
|
||||
|
||||
Returns:
|
||||
LLMSettings: Application settings.
|
||||
"""
|
||||
return request.app.state.settings
|
||||
|
||||
|
||||
def verify_bearer_token(
|
||||
request: Request,
|
||||
authorization: str | None = Header(default=None, alias="Authorization"),
|
||||
) -> str | None:
|
||||
"""Verify Bearer token authentication.
|
||||
|
||||
This dependency checks the Authorization header for a valid Bearer token.
|
||||
If authentication is disabled (no tokens configured), returns None.
|
||||
If authentication is enabled, validates the token and returns it.
|
||||
|
||||
Args:
|
||||
request: FastAPI request object.
|
||||
authorization: Authorization header value.
|
||||
|
||||
Returns:
|
||||
The validated token if auth enabled, None if auth disabled.
|
||||
|
||||
Raises:
|
||||
HTTPException: 401 if auth enabled and token is invalid/missing.
|
||||
"""
|
||||
settings = request.app.state.settings
|
||||
|
||||
# Auth disabled - allow all requests
|
||||
if not settings.auth_enabled:
|
||||
return None
|
||||
|
||||
# Auth enabled - validate token
|
||||
if not authorization:
|
||||
raise HTTPException(
|
||||
status_code=401,
|
||||
detail={
|
||||
"error": "Authentication required",
|
||||
"message": "Missing Authorization header",
|
||||
},
|
||||
headers={"WWW-Authenticate": "Bearer"},
|
||||
)
|
||||
|
||||
# Parse Bearer token
|
||||
parts = authorization.split()
|
||||
if len(parts) != 2 or parts[0].lower() != "bearer":
|
||||
raise HTTPException(
|
||||
status_code=401,
|
||||
detail={
|
||||
"error": "Authentication required",
|
||||
"message": "Invalid Authorization header format. Use: Bearer <token>",
|
||||
},
|
||||
headers={"WWW-Authenticate": "Bearer"},
|
||||
)
|
||||
|
||||
token = parts[1]
|
||||
|
||||
# Constant-time comparison against all valid tokens
|
||||
is_valid = any(
|
||||
hmac.compare_digest(token.encode(), valid_token.encode())
|
||||
for valid_token in settings.api_tokens
|
||||
)
|
||||
|
||||
if not is_valid:
|
||||
raise HTTPException(
|
||||
status_code=401,
|
||||
detail={
|
||||
"error": "Authentication failed",
|
||||
"message": "Invalid API token",
|
||||
},
|
||||
headers={"WWW-Authenticate": "Bearer"},
|
||||
)
|
||||
|
||||
return token
|
||||
|
||||
|
||||
class ConcurrencyLimiter:
|
||||
"""Limits concurrent completions to prevent resource exhaustion.
|
||||
|
||||
Uses a semaphore to limit the number of concurrent completion requests.
|
||||
Returns 503 Service Unavailable when limit is exceeded.
|
||||
"""
|
||||
|
||||
def __init__(self, max_concurrent: int) -> None:
|
||||
"""Initialize concurrency limiter.
|
||||
|
||||
Args:
|
||||
max_concurrent: Maximum number of concurrent completions.
|
||||
"""
|
||||
self.max_concurrent = max_concurrent
|
||||
self._semaphore = asyncio.Semaphore(max_concurrent)
|
||||
self._counter_lock = asyncio.Lock()
|
||||
self._current = 0
|
||||
|
||||
@property
|
||||
def current_count(self) -> int:
|
||||
"""Get current number of active requests."""
|
||||
return self._current
|
||||
|
||||
@property
|
||||
def available(self) -> int:
|
||||
"""Get number of available slots."""
|
||||
return self.max_concurrent - self._current
|
||||
|
||||
@asynccontextmanager
|
||||
async def acquire(self, blocking: bool = False) -> AsyncIterator[None]:
|
||||
"""Acquire a completion slot.
|
||||
|
||||
Args:
|
||||
blocking: If True, wait for a slot. If False (default), return 503 immediately.
|
||||
|
||||
Yields:
|
||||
None when slot is acquired.
|
||||
|
||||
Raises:
|
||||
HTTPException: 503 if no slots available and blocking=False.
|
||||
"""
|
||||
if not blocking and self._semaphore.locked():
|
||||
# Non-blocking mode and no slots available - reject immediately
|
||||
# Note: small race window exists but semaphore still enforces limit
|
||||
logger.warning(
|
||||
"Concurrency limit reached: %d/%d active",
|
||||
self._current,
|
||||
self.max_concurrent,
|
||||
)
|
||||
raise HTTPException(
|
||||
status_code=503,
|
||||
detail={
|
||||
"error": "Service temporarily unavailable",
|
||||
"reason": "Too many concurrent requests",
|
||||
"max_concurrent": self.max_concurrent,
|
||||
"retry_after": 5,
|
||||
},
|
||||
headers={"Retry-After": "5"},
|
||||
)
|
||||
|
||||
# Acquire semaphore (blocks if blocking=True and no slots, or immediate if available)
|
||||
await self._semaphore.acquire()
|
||||
|
||||
# Update counter with lock for thread safety
|
||||
async with self._counter_lock:
|
||||
self._current += 1
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
async with self._counter_lock:
|
||||
self._current -= 1
|
||||
self._semaphore.release()
|
||||
|
||||
|
||||
# Global concurrency limiter instance (initialized in app startup)
|
||||
_concurrency_limiter: ConcurrencyLimiter | None = None
|
||||
|
||||
|
||||
def init_concurrency_limiter(max_concurrent: int) -> ConcurrencyLimiter:
|
||||
"""Initialize the global concurrency limiter.
|
||||
|
||||
Args:
|
||||
max_concurrent: Maximum concurrent completions.
|
||||
|
||||
Returns:
|
||||
ConcurrencyLimiter: The initialized limiter.
|
||||
"""
|
||||
global _concurrency_limiter
|
||||
_concurrency_limiter = ConcurrencyLimiter(max_concurrent)
|
||||
logger.info("Concurrency limiter initialized: max_concurrent=%d", max_concurrent)
|
||||
return _concurrency_limiter
|
||||
|
||||
|
||||
def get_concurrency_limiter() -> ConcurrencyLimiter:
|
||||
"""Get the global concurrency limiter.
|
||||
|
||||
Returns:
|
||||
ConcurrencyLimiter: The global limiter instance.
|
||||
|
||||
Raises:
|
||||
RuntimeError: If limiter not initialized.
|
||||
"""
|
||||
if _concurrency_limiter is None:
|
||||
raise RuntimeError("Concurrency limiter not initialized")
|
||||
return _concurrency_limiter
|
||||
|
||||
|
||||
async def require_completion_slot(request: Request) -> AsyncIterator[None]:
|
||||
"""FastAPI dependency that acquires a completion slot.
|
||||
|
||||
Args:
|
||||
request: FastAPI request.
|
||||
|
||||
Yields:
|
||||
None when slot is acquired.
|
||||
|
||||
Raises:
|
||||
HTTPException: 503 if no slots available.
|
||||
"""
|
||||
limiter = get_concurrency_limiter()
|
||||
async with limiter.acquire():
|
||||
yield
|
||||
|
|
@ -0,0 +1,230 @@
|
|||
"""FastAPI middleware for request handling and rate limiting.
|
||||
|
||||
Provides:
|
||||
- RequestIdMiddleware: Extracts/generates request IDs and propagates via context
|
||||
- RateLimitMiddleware: Token bucket rate limiting with 429 responses
|
||||
"""
|
||||
|
||||
import time
|
||||
import uuid
|
||||
from collections.abc import Awaitable, Callable
|
||||
|
||||
from fastapi import Request, Response
|
||||
from starlette.middleware.base import BaseHTTPMiddleware
|
||||
from starlette.responses import JSONResponse
|
||||
|
||||
from llm_inference.logging import get_logger, set_request_id
|
||||
|
||||
logger = get_logger("middleware")
|
||||
|
||||
|
||||
class RequestIdMiddleware(BaseHTTPMiddleware):
|
||||
"""Middleware to extract or generate request IDs.
|
||||
|
||||
Extracts X-Request-ID from incoming headers or generates a new UUID.
|
||||
Sets the request ID in context for logging and returns it in response headers.
|
||||
"""
|
||||
|
||||
async def dispatch(
|
||||
self,
|
||||
request: Request,
|
||||
call_next: Callable[[Request], Awaitable[Response]],
|
||||
) -> Response:
|
||||
"""Process request with request ID tracking.
|
||||
|
||||
Args:
|
||||
request: Incoming HTTP request.
|
||||
call_next: Next middleware/handler in chain.
|
||||
|
||||
Returns:
|
||||
Response: HTTP response with X-Request-ID header.
|
||||
"""
|
||||
# Extract from header or generate new
|
||||
request_id = request.headers.get("X-Request-ID")
|
||||
if not request_id:
|
||||
request_id = str(uuid.uuid4())
|
||||
|
||||
# Set in context for logging
|
||||
set_request_id(request_id)
|
||||
|
||||
# Store in request state for handlers
|
||||
request.state.request_id = request_id
|
||||
|
||||
try:
|
||||
response = await call_next(request)
|
||||
response.headers["X-Request-ID"] = request_id
|
||||
return response
|
||||
finally:
|
||||
# Clear context after request
|
||||
set_request_id(None)
|
||||
|
||||
|
||||
class TokenBucket:
|
||||
"""Token bucket for rate limiting with optional live tuning.
|
||||
|
||||
Implements a simple token bucket algorithm where tokens are added
|
||||
at a fixed rate up to a maximum burst size. If a runtime_config and
|
||||
rate/burst keys are supplied, the bucket reads the latest override
|
||||
on every acquire() — making the limit live-tunable from dashboard
|
||||
without restart.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
rate: float,
|
||||
burst: int,
|
||||
runtime_config: object | None = None,
|
||||
rate_key: str | None = None,
|
||||
burst_key: str | None = None,
|
||||
) -> None:
|
||||
"""Initialize token bucket.
|
||||
|
||||
Args:
|
||||
rate: Tokens added per second (fallback / initial).
|
||||
burst: Maximum tokens (bucket capacity, fallback / initial).
|
||||
runtime_config: Optional RuntimeConfigClient. When set together
|
||||
with rate_key / burst_key, the bucket pulls live values on
|
||||
each acquire() and falls back to the constructor values.
|
||||
rate_key: Dashboard config key for live rate (e.g., "llm.rate_limit.rps").
|
||||
burst_key: Dashboard config key for live burst.
|
||||
"""
|
||||
self._fallback_rate = rate
|
||||
self._fallback_burst = burst
|
||||
self.rate = rate
|
||||
self.burst = burst
|
||||
self.tokens = float(burst)
|
||||
self.last_update = time.monotonic()
|
||||
self._runtime_config = runtime_config
|
||||
self._rate_key = rate_key
|
||||
self._burst_key = burst_key
|
||||
|
||||
def _refresh_from_config(self) -> None:
|
||||
"""Pull latest rate/burst from runtime_config if wired."""
|
||||
if self._runtime_config is None:
|
||||
return
|
||||
try:
|
||||
new_rate = self._runtime_config.get_float(self._rate_key, self._fallback_rate) if self._rate_key else self._fallback_rate
|
||||
new_burst = self._runtime_config.get_int(self._burst_key, self._fallback_burst) if self._burst_key else self._fallback_burst
|
||||
except Exception:
|
||||
return
|
||||
if new_rate != self.rate or new_burst != self.burst:
|
||||
# On burst increase, top up; on decrease, clamp tokens to new burst
|
||||
self.rate = new_rate
|
||||
self.burst = new_burst
|
||||
self.tokens = min(self.tokens, float(new_burst))
|
||||
|
||||
def acquire(self) -> bool:
|
||||
"""Try to acquire a token.
|
||||
|
||||
Returns:
|
||||
bool: True if token acquired, False if rate limited.
|
||||
"""
|
||||
self._refresh_from_config()
|
||||
now = time.monotonic()
|
||||
elapsed = now - self.last_update
|
||||
self.last_update = now
|
||||
|
||||
# Add tokens based on elapsed time
|
||||
self.tokens = min(self.burst, self.tokens + elapsed * self.rate)
|
||||
|
||||
if self.tokens >= 1:
|
||||
self.tokens -= 1
|
||||
return True
|
||||
return False
|
||||
|
||||
def retry_after(self) -> float:
|
||||
"""Calculate seconds until a token is available.
|
||||
|
||||
Returns:
|
||||
float: Seconds to wait before retrying.
|
||||
"""
|
||||
if self.tokens >= 1:
|
||||
return 0.0
|
||||
tokens_needed = 1 - self.tokens
|
||||
return tokens_needed / self.rate
|
||||
|
||||
|
||||
class RateLimitMiddleware(BaseHTTPMiddleware):
|
||||
"""Middleware for global rate limiting using token bucket algorithm.
|
||||
|
||||
Returns 429 Too Many Requests with Retry-After header when exceeded.
|
||||
|
||||
WARNING: This is per-process rate limiting. In multi-replica deployments
|
||||
(e.g., Kubernetes with multiple pods), each replica maintains its own
|
||||
independent rate limit. A configured limit of 10 RPS with 5 replicas
|
||||
effectively allows 50 RPS total.
|
||||
|
||||
For distributed rate limiting in production, use an external solution:
|
||||
- Redis-based rate limiting (e.g., redis-rate-limiter)
|
||||
- API gateway rate limiting (e.g., Kong, nginx)
|
||||
- Cloud provider rate limiting (e.g., AWS API Gateway)
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
app: object,
|
||||
rate: float = 10.0,
|
||||
burst: int = 20,
|
||||
exclude_paths: list[str] | None = None,
|
||||
runtime_config: object | None = None,
|
||||
rate_key: str | None = None,
|
||||
burst_key: str | None = None,
|
||||
) -> None:
|
||||
"""Initialize rate limiter.
|
||||
|
||||
Args:
|
||||
app: FastAPI application.
|
||||
rate: Requests per second limit (fallback when no runtime_config).
|
||||
burst: Maximum burst size (fallback).
|
||||
exclude_paths: Paths to exclude from rate limiting.
|
||||
runtime_config: Optional RuntimeConfigClient to enable live rate tuning.
|
||||
rate_key: Dashboard config key (e.g., "llm.rate_limit.rps").
|
||||
burst_key: Dashboard config key (e.g., "llm.rate_limit.burst").
|
||||
"""
|
||||
super().__init__(app)
|
||||
self.bucket = TokenBucket(
|
||||
rate=rate,
|
||||
burst=burst,
|
||||
runtime_config=runtime_config,
|
||||
rate_key=rate_key,
|
||||
burst_key=burst_key,
|
||||
)
|
||||
self.exclude_paths = exclude_paths or ["/health", "/ready"]
|
||||
|
||||
async def dispatch(
|
||||
self,
|
||||
request: Request,
|
||||
call_next: Callable[[Request], Awaitable[Response]],
|
||||
) -> Response:
|
||||
"""Process request with rate limiting.
|
||||
|
||||
Args:
|
||||
request: Incoming HTTP request.
|
||||
call_next: Next middleware/handler in chain.
|
||||
|
||||
Returns:
|
||||
Response: HTTP response or 429 if rate limited.
|
||||
"""
|
||||
# Skip rate limiting for excluded paths
|
||||
if request.url.path in self.exclude_paths:
|
||||
return await call_next(request)
|
||||
|
||||
# Try to acquire token
|
||||
if not self.bucket.acquire():
|
||||
retry_after = self.bucket.retry_after()
|
||||
logger.warning(
|
||||
"Rate limit exceeded for %s %s, retry_after=%.2f",
|
||||
request.method,
|
||||
request.url.path,
|
||||
retry_after,
|
||||
)
|
||||
return JSONResponse(
|
||||
status_code=429,
|
||||
content={
|
||||
"detail": "Too many requests",
|
||||
"retry_after": retry_after,
|
||||
},
|
||||
headers={"Retry-After": str(int(retry_after) + 1)},
|
||||
)
|
||||
|
||||
return await call_next(request)
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
"""API routes for LLM inference."""
|
||||
|
||||
from llm_inference.api.routes import completions, health, info, models
|
||||
|
||||
__all__ = ["completions", "health", "info", "models"]
|
||||
|
|
@ -0,0 +1,221 @@
|
|||
"""Chat completion routes with streaming support."""
|
||||
|
||||
import json
|
||||
from collections.abc import AsyncGenerator
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from sse_starlette.sse import EventSourceResponse
|
||||
|
||||
from llm_inference.api.dependencies import (
|
||||
ConcurrencyLimiter,
|
||||
get_client,
|
||||
get_concurrency_limiter,
|
||||
verify_bearer_token,
|
||||
)
|
||||
from llm_inference.client import LLMClient
|
||||
from llm_inference.exceptions import (
|
||||
BackendNotAvailableError,
|
||||
BackendNotEnabledError,
|
||||
CompletionError,
|
||||
)
|
||||
from llm_inference.schemas import (
|
||||
CompletionRequest,
|
||||
CompletionResponse,
|
||||
TextCompletionRequest,
|
||||
TextCompletionResponse,
|
||||
)
|
||||
|
||||
router = APIRouter(dependencies=[Depends(verify_bearer_token)])
|
||||
|
||||
|
||||
@router.post(
|
||||
"/chat/completions",
|
||||
response_model=None, # Disable auto-generation due to Union with EventSourceResponse
|
||||
)
|
||||
async def chat_completions(
|
||||
request: CompletionRequest,
|
||||
client: LLMClient = Depends(get_client),
|
||||
) -> CompletionResponse | EventSourceResponse:
|
||||
"""OpenAI-compatible chat completions endpoint.
|
||||
|
||||
Supports both streaming (SSE) and non-streaming responses.
|
||||
Backend can be overridden per-request via the `backend` field.
|
||||
|
||||
Concurrency is limited per-process. For streaming requests, the slot is
|
||||
held for the entire duration of the stream.
|
||||
|
||||
Args:
|
||||
request: Completion request with messages, model, and options.
|
||||
client: LLM client instance.
|
||||
|
||||
Returns:
|
||||
CompletionResponse or EventSourceResponse for streaming.
|
||||
|
||||
Raises:
|
||||
HTTPException: If the request fails or concurrency limit exceeded (503).
|
||||
"""
|
||||
limiter = get_concurrency_limiter()
|
||||
|
||||
if request.stream:
|
||||
# For streaming: wrap generator to hold slot throughout entire stream
|
||||
# This is critical - the slot must be held until streaming completes
|
||||
return EventSourceResponse(
|
||||
_stream_with_slot(_stream_generator(client, request), limiter),
|
||||
media_type="text/event-stream",
|
||||
ping=15, # Send ping every 15s to detect dead connections
|
||||
)
|
||||
|
||||
# For non-streaming: acquire slot, run completion, release slot
|
||||
async with limiter.acquire():
|
||||
try:
|
||||
kwargs = _build_completion_kwargs(request)
|
||||
|
||||
response = await client.complete(
|
||||
messages=request.messages,
|
||||
model=request.model,
|
||||
backend=request.backend,
|
||||
**kwargs,
|
||||
)
|
||||
return response
|
||||
|
||||
except (BackendNotAvailableError, BackendNotEnabledError) as e:
|
||||
raise HTTPException(status_code=400, detail=str(e)) from None
|
||||
except CompletionError as e:
|
||||
raise HTTPException(status_code=500, detail=str(e)) from None
|
||||
|
||||
|
||||
@router.post("/completions", response_model=TextCompletionResponse)
|
||||
async def text_completions(
|
||||
request: TextCompletionRequest,
|
||||
client: LLMClient = Depends(get_client),
|
||||
) -> TextCompletionResponse:
|
||||
"""OpenAI-compatible legacy text completions endpoint (/v1/completions).
|
||||
|
||||
Non-streaming. Routes to the model's backend (vLLM primary, cloud via
|
||||
LiteLLM). Backends that do not support text completion return 501.
|
||||
"""
|
||||
limiter = get_concurrency_limiter()
|
||||
async with limiter.acquire():
|
||||
try:
|
||||
kwargs: dict[str, object] = {"temperature": request.temperature}
|
||||
if request.max_tokens is not None:
|
||||
kwargs["max_tokens"] = request.max_tokens
|
||||
if request.top_p is not None:
|
||||
kwargs["top_p"] = request.top_p
|
||||
if request.frequency_penalty is not None:
|
||||
kwargs["frequency_penalty"] = request.frequency_penalty
|
||||
if request.presence_penalty is not None:
|
||||
kwargs["presence_penalty"] = request.presence_penalty
|
||||
if request.stop is not None:
|
||||
stop_seq = (
|
||||
[request.stop]
|
||||
if isinstance(request.stop, str)
|
||||
else list(request.stop)
|
||||
)
|
||||
stop_seq = [s for s in stop_seq if s]
|
||||
if stop_seq:
|
||||
kwargs["stop"] = stop_seq
|
||||
|
||||
return await client.text_complete(
|
||||
prompt=request.prompt,
|
||||
model=request.model,
|
||||
backend=request.backend,
|
||||
**kwargs,
|
||||
)
|
||||
except NotImplementedError as e:
|
||||
raise HTTPException(status_code=501, detail=str(e)) from None
|
||||
except (BackendNotAvailableError, BackendNotEnabledError) as e:
|
||||
raise HTTPException(status_code=400, detail=str(e)) from None
|
||||
except CompletionError as e:
|
||||
raise HTTPException(status_code=500, detail=str(e)) from None
|
||||
|
||||
|
||||
async def _stream_generator(
|
||||
client: LLMClient,
|
||||
request: CompletionRequest,
|
||||
) -> AsyncGenerator[dict[str, str], None]:
|
||||
"""Generate SSE events for streaming response.
|
||||
|
||||
Args:
|
||||
client: LLM client instance.
|
||||
request: Completion request.
|
||||
|
||||
Yields:
|
||||
SSE event dictionaries with data field.
|
||||
"""
|
||||
try:
|
||||
kwargs = _build_completion_kwargs(request)
|
||||
|
||||
async for chunk in client.stream(
|
||||
messages=request.messages,
|
||||
model=request.model,
|
||||
backend=request.backend,
|
||||
**kwargs,
|
||||
):
|
||||
yield {"data": json.dumps(chunk.model_dump())}
|
||||
|
||||
# Send [DONE] marker to signal end of stream
|
||||
yield {"data": "[DONE]"}
|
||||
|
||||
except (BackendNotAvailableError, BackendNotEnabledError, CompletionError) as e:
|
||||
# Send error event followed by [DONE] to signal stream termination
|
||||
yield {
|
||||
"event": "error",
|
||||
"data": json.dumps({"error": str(e)}),
|
||||
}
|
||||
yield {"data": "[DONE]"}
|
||||
|
||||
|
||||
async def _stream_with_slot(
|
||||
generator: AsyncGenerator[dict[str, str], None],
|
||||
limiter: ConcurrencyLimiter,
|
||||
) -> AsyncGenerator[dict[str, str], None]:
|
||||
"""Wrap a stream generator to hold a concurrency slot throughout streaming.
|
||||
|
||||
This ensures the concurrency limiter slot is held for the entire duration
|
||||
of the SSE stream, not just until the EventSourceResponse is returned.
|
||||
|
||||
Args:
|
||||
generator: The underlying stream generator.
|
||||
limiter: Concurrency limiter to acquire slot from.
|
||||
|
||||
Yields:
|
||||
SSE event dictionaries from the wrapped generator.
|
||||
"""
|
||||
async with limiter.acquire():
|
||||
async for item in generator:
|
||||
yield item
|
||||
|
||||
|
||||
def _build_completion_kwargs(request: CompletionRequest) -> dict[str, object]:
|
||||
"""Build kwargs dict from completion request.
|
||||
|
||||
Args:
|
||||
request: Completion request.
|
||||
|
||||
Returns:
|
||||
dict: Keyword arguments for completion call.
|
||||
"""
|
||||
kwargs: dict[str, object] = {
|
||||
"temperature": request.temperature,
|
||||
}
|
||||
|
||||
if request.max_tokens is not None:
|
||||
kwargs["max_tokens"] = request.max_tokens
|
||||
if request.top_p is not None:
|
||||
kwargs["top_p"] = request.top_p
|
||||
if request.frequency_penalty is not None:
|
||||
kwargs["frequency_penalty"] = request.frequency_penalty
|
||||
if request.presence_penalty is not None:
|
||||
kwargs["presence_penalty"] = request.presence_penalty
|
||||
if request.stop is not None:
|
||||
# Normalize to list and filter out empty strings
|
||||
# Some backends choke on empty stop sequences
|
||||
stop_seq = (
|
||||
[request.stop] if isinstance(request.stop, str) else list(request.stop)
|
||||
)
|
||||
stop_seq = [s for s in stop_seq if s] # Filter empty strings
|
||||
if stop_seq:
|
||||
kwargs["stop"] = stop_seq
|
||||
|
||||
return kwargs
|
||||
|
|
@ -0,0 +1,78 @@
|
|||
"""Health check routes."""
|
||||
|
||||
from fastapi import APIRouter, Depends
|
||||
|
||||
from llm_inference.api.dependencies import get_client
|
||||
from llm_inference.client import LLMClient
|
||||
from llm_inference.logging import get_logger
|
||||
from llm_inference.schemas import BackendHealth, HealthResponse, ReadinessResponse
|
||||
|
||||
router = APIRouter()
|
||||
logger = get_logger("routes.health")
|
||||
|
||||
|
||||
@router.get("/health", response_model=HealthResponse)
|
||||
async def health_check(
|
||||
client: LLMClient = Depends(get_client),
|
||||
) -> HealthResponse:
|
||||
"""Health check endpoint.
|
||||
|
||||
Returns the overall health status and per-backend health status.
|
||||
|
||||
Returns:
|
||||
HealthResponse: Overall and per-backend health status.
|
||||
"""
|
||||
backend_health: list[BackendHealth] = []
|
||||
|
||||
for backend_type in client.list_backends():
|
||||
backend = client.registry.get(backend_type)
|
||||
is_healthy = await backend.health_check()
|
||||
backend_health.append(
|
||||
BackendHealth(
|
||||
name=backend_type.value,
|
||||
healthy=is_healthy,
|
||||
)
|
||||
)
|
||||
|
||||
# Determine overall status
|
||||
all_healthy = all(b.healthy for b in backend_health)
|
||||
any_healthy = any(b.healthy for b in backend_health)
|
||||
|
||||
if all_healthy:
|
||||
status = "healthy"
|
||||
elif any_healthy:
|
||||
status = "degraded"
|
||||
else:
|
||||
status = "unhealthy"
|
||||
|
||||
return HealthResponse(status=status, backends=backend_health)
|
||||
|
||||
|
||||
@router.get("/ready", response_model=ReadinessResponse)
|
||||
async def readiness_check(
|
||||
client: LLMClient = Depends(get_client),
|
||||
) -> ReadinessResponse:
|
||||
"""Readiness probe for Kubernetes.
|
||||
|
||||
Verifies that the default backend is healthy and able to serve requests.
|
||||
This is the critical check for load balancer routing.
|
||||
|
||||
Returns:
|
||||
ReadinessResponse: Readiness status (ready if default backend is healthy).
|
||||
"""
|
||||
try:
|
||||
# Get the default backend and check its health
|
||||
default_backend = client.registry.get()
|
||||
is_healthy = await default_backend.health_check()
|
||||
|
||||
if not is_healthy:
|
||||
logger.warning(
|
||||
"Readiness check failed: default backend '%s' is unhealthy",
|
||||
default_backend.name,
|
||||
)
|
||||
|
||||
return ReadinessResponse(ready=is_healthy)
|
||||
|
||||
except Exception as e:
|
||||
logger.error("Readiness check failed with error: %s", str(e))
|
||||
return ReadinessResponse(ready=False)
|
||||
|
|
@ -0,0 +1,468 @@
|
|||
"""Component information endpoint for service catalog."""
|
||||
|
||||
from fastapi import APIRouter, Depends
|
||||
|
||||
from llm_inference.api.dependencies import get_client
|
||||
from llm_inference.client import LLMClient
|
||||
from llm_inference.config import SettingsCache
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
@router.get("/v1/info")
|
||||
async def get_component_info(
|
||||
client: LLMClient = Depends(get_client),
|
||||
) -> dict:
|
||||
"""Get component information for service catalog.
|
||||
|
||||
Returns complete metadata about this service including:
|
||||
- Resource information (component metadata)
|
||||
- Available models (from all enabled backends)
|
||||
- Available functions (API endpoints)
|
||||
|
||||
This endpoint is used by the catalog-api to aggregate service information
|
||||
and by backend systems to populate the catalog database.
|
||||
|
||||
Returns:
|
||||
dict: Component information matching catalog.resources, catalog.models,
|
||||
and catalog.functions schemas.
|
||||
"""
|
||||
settings = SettingsCache.get()
|
||||
|
||||
# Determine which backends are available
|
||||
backends = ["litellm"] # Always available
|
||||
if settings.enable_vllm:
|
||||
backends.append("vllm")
|
||||
if settings.enable_llamacpp:
|
||||
backends.append("llamacpp")
|
||||
|
||||
# Build resource information (maps to catalog.resources)
|
||||
resource = {
|
||||
"name": "LLM Inference Gateway",
|
||||
"slug": "llm-inference",
|
||||
"resource_type": "api_service",
|
||||
"provider": "internal",
|
||||
"base_url": f"http://didiAI-llm-api:{settings.port}",
|
||||
"configuration": {
|
||||
"version": "1.0.0",
|
||||
"port": settings.port,
|
||||
"external_url": settings.external_url,
|
||||
"default_backend": settings.default_backend,
|
||||
"backends": backends,
|
||||
"default_model": settings.default_model,
|
||||
},
|
||||
"authentication": {
|
||||
"type": "bearer",
|
||||
"required": bool(settings.api_tokens),
|
||||
"env_var": "LLM_API_TOKENS",
|
||||
},
|
||||
"headers": {
|
||||
"Content-Type": "application/json",
|
||||
"Accept": "application/json",
|
||||
},
|
||||
"rate_limits": {
|
||||
"requests_per_second": settings.rate_limit_rps,
|
||||
"burst": settings.rate_limit_burst,
|
||||
"concurrent": settings.max_concurrent_completions,
|
||||
},
|
||||
"cost_tracking": {
|
||||
"enabled": False,
|
||||
},
|
||||
"tags": ["llm", "inference", "openai-compatible", "gateway", "nlp"],
|
||||
"is_active": True,
|
||||
"metadata": {
|
||||
"category": "nlp",
|
||||
"gpu_required": settings.enable_vllm,
|
||||
"status": "healthy",
|
||||
"vllm_base_url": settings.vllm_base_url if settings.enable_vllm else None,
|
||||
"llamacpp_base_url": (
|
||||
settings.llamacpp_base_url if settings.enable_llamacpp else None
|
||||
),
|
||||
},
|
||||
}
|
||||
|
||||
# Collect models from all backends
|
||||
models = []
|
||||
|
||||
# Get models from enabled backends
|
||||
try:
|
||||
all_models = await client.list_models()
|
||||
|
||||
for model_info in all_models:
|
||||
model_entry = {
|
||||
"name": model_info.id,
|
||||
"slug": model_info.id.lower().replace("/", "-").replace("_", "-"),
|
||||
"provider": _get_provider_from_model_id(model_info.id),
|
||||
"model_type": _get_model_type(model_info),
|
||||
"capabilities": model_info.capabilities or [],
|
||||
"configuration": {
|
||||
"backend": model_info.backend,
|
||||
"context_length": model_info.context_length,
|
||||
"loaded": model_info.loaded,
|
||||
},
|
||||
"endpoint": _get_model_endpoint(model_info, settings),
|
||||
"api_key_ref": None,
|
||||
"tags": _get_model_tags(model_info),
|
||||
"is_active": model_info.loaded,
|
||||
"metadata": {
|
||||
"backend": model_info.backend,
|
||||
"model_id": model_info.id,
|
||||
},
|
||||
}
|
||||
models.append(model_entry)
|
||||
except Exception:
|
||||
# If model listing fails, continue with empty models list
|
||||
pass
|
||||
|
||||
# Define available functions (maps to catalog.functions)
|
||||
functions = [
|
||||
{
|
||||
"name": "Chat Completions",
|
||||
"slug": "llm-chat-completions",
|
||||
"category": "completion",
|
||||
"description": (
|
||||
"Create a chat completion with optional streaming support. "
|
||||
"OpenAI-compatible endpoint supporting multiple backends."
|
||||
),
|
||||
"input_schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"messages": {
|
||||
"type": "array",
|
||||
"description": "List of messages (1-1000)",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"role": {
|
||||
"type": "string",
|
||||
"enum": ["system", "user", "assistant", "function", "tool"],
|
||||
},
|
||||
"content": {"type": "string"},
|
||||
},
|
||||
"required": ["role"],
|
||||
},
|
||||
},
|
||||
"model": {"type": "string", "description": "Model identifier"},
|
||||
"temperature": {
|
||||
"type": "number",
|
||||
"minimum": 0.0,
|
||||
"maximum": 2.0,
|
||||
"default": 0.7,
|
||||
},
|
||||
"max_tokens": {"type": "integer", "minimum": 1, "maximum": 1000000},
|
||||
"stream": {"type": "boolean", "default": False},
|
||||
"backend": {
|
||||
"type": "string",
|
||||
"enum": backends,
|
||||
"description": "Backend override",
|
||||
},
|
||||
"top_p": {"type": "number", "minimum": 0.0, "maximum": 1.0},
|
||||
"frequency_penalty": {"type": "number", "minimum": -2.0, "maximum": 2.0},
|
||||
"presence_penalty": {"type": "number", "minimum": -2.0, "maximum": 2.0},
|
||||
"stop": {
|
||||
"oneOf": [
|
||||
{"type": "string"},
|
||||
{"type": "array", "items": {"type": "string"}},
|
||||
]
|
||||
},
|
||||
},
|
||||
"required": ["messages", "model"],
|
||||
},
|
||||
"output_schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"id": {"type": "string"},
|
||||
"object": {"type": "string", "const": "chat.completion"},
|
||||
"created": {"type": "integer"},
|
||||
"model": {"type": "string"},
|
||||
"choices": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"index": {"type": "integer"},
|
||||
"message": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"role": {"type": "string"},
|
||||
"content": {"type": "string"},
|
||||
},
|
||||
},
|
||||
"finish_reason": {"type": "string"},
|
||||
},
|
||||
},
|
||||
},
|
||||
"usage": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"prompt_tokens": {"type": "integer"},
|
||||
"completion_tokens": {"type": "integer"},
|
||||
"total_tokens": {"type": "integer"},
|
||||
},
|
||||
},
|
||||
"backend": {"type": "string"},
|
||||
},
|
||||
},
|
||||
"implementation": {
|
||||
"method": "POST",
|
||||
"path": "/v1/chat/completions",
|
||||
"content_type": "application/json",
|
||||
"timeout": 120,
|
||||
"supports_streaming": True,
|
||||
},
|
||||
"endpoint": f"http://localhost:{settings.port}/v1/chat/completions",
|
||||
"tags": ["llm", "chat", "openai", "streaming"],
|
||||
"is_active": True,
|
||||
"metadata": {
|
||||
"rate_limited": True,
|
||||
"auth_required": bool(settings.api_tokens),
|
||||
},
|
||||
},
|
||||
{
|
||||
"name": "List Models",
|
||||
"slug": "llm-list-models",
|
||||
"category": "discovery",
|
||||
"description": "List all available models across all enabled backends",
|
||||
"input_schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"backend": {
|
||||
"type": "string",
|
||||
"enum": backends,
|
||||
"description": "Optional backend filter",
|
||||
}
|
||||
},
|
||||
},
|
||||
"output_schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"object": {"type": "string", "const": "list"},
|
||||
"data": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"id": {"type": "string"},
|
||||
"backend": {"type": "string"},
|
||||
"loaded": {"type": "boolean"},
|
||||
"context_length": {"type": "integer"},
|
||||
"capabilities": {
|
||||
"type": "array",
|
||||
"items": {"type": "string"},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
"implementation": {
|
||||
"method": "GET",
|
||||
"path": "/v1/models",
|
||||
"timeout": 10,
|
||||
},
|
||||
"endpoint": f"http://localhost:{settings.port}/v1/models",
|
||||
"tags": ["discovery", "models"],
|
||||
"is_active": True,
|
||||
"metadata": {},
|
||||
},
|
||||
{
|
||||
"name": "List Backends",
|
||||
"slug": "llm-list-backends",
|
||||
"category": "discovery",
|
||||
"description": "List all available backends",
|
||||
"input_schema": {"type": "object", "properties": {}},
|
||||
"output_schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"backends": {
|
||||
"type": "array",
|
||||
"items": {"type": "string"},
|
||||
}
|
||||
},
|
||||
},
|
||||
"implementation": {
|
||||
"method": "GET",
|
||||
"path": "/v1/backends",
|
||||
"timeout": 5,
|
||||
},
|
||||
"endpoint": f"http://localhost:{settings.port}/v1/backends",
|
||||
"tags": ["discovery"],
|
||||
"is_active": True,
|
||||
"metadata": {},
|
||||
},
|
||||
{
|
||||
"name": "Load Model",
|
||||
"slug": "llm-load-model",
|
||||
"category": "management",
|
||||
"description": "Load a model on a local backend (vLLM or llama.cpp only)",
|
||||
"input_schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"model": {"type": "string"},
|
||||
"backend": {"type": "string", "enum": ["vllm", "llamacpp"]},
|
||||
},
|
||||
"required": ["model", "backend"],
|
||||
},
|
||||
"output_schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"success": {"type": "boolean"},
|
||||
"model": {"type": "string"},
|
||||
"backend": {"type": "string"},
|
||||
"message": {"type": "string"},
|
||||
},
|
||||
},
|
||||
"implementation": {
|
||||
"method": "POST",
|
||||
"path": "/v1/models/load",
|
||||
"content_type": "application/json",
|
||||
"timeout": 60,
|
||||
},
|
||||
"endpoint": f"http://localhost:{settings.port}/v1/models/load",
|
||||
"tags": ["management", "models"],
|
||||
"is_active": settings.enable_vllm or settings.enable_llamacpp,
|
||||
"metadata": {},
|
||||
},
|
||||
{
|
||||
"name": "Unload Model",
|
||||
"slug": "llm-unload-model",
|
||||
"category": "management",
|
||||
"description": "Unload a model from a local backend (vLLM or llama.cpp only)",
|
||||
"input_schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"model": {"type": "string"},
|
||||
"backend": {"type": "string", "enum": ["vllm", "llamacpp"]},
|
||||
},
|
||||
"required": ["model", "backend"],
|
||||
},
|
||||
"output_schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"success": {"type": "boolean"},
|
||||
"model": {"type": "string"},
|
||||
"backend": {"type": "string"},
|
||||
"message": {"type": "string"},
|
||||
},
|
||||
},
|
||||
"implementation": {
|
||||
"method": "POST",
|
||||
"path": "/v1/models/unload",
|
||||
"content_type": "application/json",
|
||||
"timeout": 30,
|
||||
},
|
||||
"endpoint": f"http://localhost:{settings.port}/v1/models/unload",
|
||||
"tags": ["management", "models"],
|
||||
"is_active": settings.enable_vllm or settings.enable_llamacpp,
|
||||
"metadata": {},
|
||||
},
|
||||
]
|
||||
|
||||
return {
|
||||
"resource": resource,
|
||||
"models": models,
|
||||
"functions": functions,
|
||||
}
|
||||
|
||||
|
||||
def _get_provider_from_model_id(model_id: str) -> str:
|
||||
"""Extract provider name from model ID.
|
||||
|
||||
Args:
|
||||
model_id: Model identifier (e.g., "openai/gpt-4", "meta-llama/Llama-2-7b")
|
||||
|
||||
Returns:
|
||||
str: Provider name (e.g., "openai", "meta-llama", "unknown")
|
||||
"""
|
||||
if "/" in model_id:
|
||||
return model_id.split("/")[0]
|
||||
if "gpt" in model_id.lower():
|
||||
return "openai"
|
||||
if "claude" in model_id.lower():
|
||||
return "anthropic"
|
||||
if "llama" in model_id.lower():
|
||||
return "meta"
|
||||
if "qwen" in model_id.lower():
|
||||
return "qwen"
|
||||
if "mistral" in model_id.lower():
|
||||
return "mistralai"
|
||||
return "unknown"
|
||||
|
||||
|
||||
def _get_model_type(model_info) -> str:
|
||||
"""Determine model type from capabilities.
|
||||
|
||||
Args:
|
||||
model_info: Model information object
|
||||
|
||||
Returns:
|
||||
str: Model type (llm, vision, audio, etc.)
|
||||
"""
|
||||
capabilities = model_info.capabilities or []
|
||||
if "vision" in capabilities or "multimodal" in capabilities:
|
||||
return "vision"
|
||||
if "audio" in capabilities:
|
||||
return "audio"
|
||||
if "embedding" in capabilities:
|
||||
return "embedding"
|
||||
return "llm"
|
||||
|
||||
|
||||
def _get_model_tags(model_info) -> list[str]:
|
||||
"""Generate tags for a model.
|
||||
|
||||
Args:
|
||||
model_info: Model information object
|
||||
|
||||
Returns:
|
||||
list[str]: List of tags
|
||||
"""
|
||||
tags = []
|
||||
|
||||
# Add model type tags
|
||||
if "gpt" in model_info.id.lower():
|
||||
tags.extend(["gpt", "openai"])
|
||||
if "claude" in model_info.id.lower():
|
||||
tags.extend(["claude", "anthropic"])
|
||||
if "llama" in model_info.id.lower():
|
||||
tags.extend(["llama", "meta"])
|
||||
if "qwen" in model_info.id.lower():
|
||||
tags.extend(["qwen"])
|
||||
if "mistral" in model_info.id.lower():
|
||||
tags.extend(["mistral"])
|
||||
|
||||
# Add capability tags
|
||||
capabilities = model_info.capabilities or []
|
||||
tags.extend(capabilities)
|
||||
|
||||
# Add backend tag
|
||||
tags.append(model_info.backend)
|
||||
|
||||
# Add size tags if detectable
|
||||
model_id_lower = model_info.id.lower()
|
||||
if "7b" in model_id_lower:
|
||||
tags.append("7b")
|
||||
elif "13b" in model_id_lower:
|
||||
tags.append("13b")
|
||||
elif "30b" in model_id_lower:
|
||||
tags.append("30b")
|
||||
elif "70b" in model_id_lower:
|
||||
tags.append("70b")
|
||||
elif "120b" in model_id_lower:
|
||||
tags.append("120b")
|
||||
|
||||
return list(set(tags)) # Remove duplicates
|
||||
|
||||
|
||||
def _get_model_endpoint(model_info, settings) -> str:
|
||||
"""Get the endpoint URL for a model.
|
||||
|
||||
Args:
|
||||
model_info: Model information object
|
||||
settings: Application settings
|
||||
|
||||
Returns:
|
||||
str: Endpoint URL
|
||||
"""
|
||||
# Default to gateway endpoint
|
||||
return f"http://localhost:{settings.port}/v1/chat/completions"
|
||||
|
|
@ -0,0 +1,134 @@
|
|||
"""Model management routes."""
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
|
||||
from llm_inference.api.dependencies import get_client, verify_bearer_token
|
||||
from llm_inference.client import LLMClient
|
||||
from llm_inference.exceptions import (
|
||||
BackendNotAvailableError,
|
||||
BackendNotEnabledError,
|
||||
ModelLoadError,
|
||||
)
|
||||
from llm_inference.schemas import (
|
||||
BackendListResponse,
|
||||
ModelListResponse,
|
||||
ModelLoadRequest,
|
||||
ModelLoadResponse,
|
||||
)
|
||||
from llm_inference.types import BackendType
|
||||
|
||||
router = APIRouter(dependencies=[Depends(verify_bearer_token)])
|
||||
|
||||
|
||||
@router.get("/models", response_model=ModelListResponse)
|
||||
async def list_models(
|
||||
backend: str | None = None,
|
||||
client: LLMClient = Depends(get_client),
|
||||
) -> ModelListResponse:
|
||||
"""List available models.
|
||||
|
||||
Args:
|
||||
backend: Optional backend filter. If not specified, returns models
|
||||
from all available backends.
|
||||
client: LLM client instance.
|
||||
|
||||
Returns:
|
||||
ModelListResponse: List of available models.
|
||||
|
||||
Raises:
|
||||
HTTPException: If the specified backend is not available.
|
||||
"""
|
||||
try:
|
||||
backend_type = BackendType(backend) if backend else None
|
||||
models = await client.list_models(backend=backend_type)
|
||||
return ModelListResponse(data=models)
|
||||
except ValueError:
|
||||
raise HTTPException(
|
||||
status_code=400, detail=f"Invalid backend: {backend}"
|
||||
) from None
|
||||
except (BackendNotAvailableError, BackendNotEnabledError) as e:
|
||||
raise HTTPException(status_code=400, detail=str(e)) from None
|
||||
|
||||
|
||||
@router.post("/models/load", response_model=ModelLoadResponse)
|
||||
async def load_model(
|
||||
request: ModelLoadRequest,
|
||||
client: LLMClient = Depends(get_client),
|
||||
) -> ModelLoadResponse:
|
||||
"""Load a model on the specified backend.
|
||||
|
||||
This is only supported by local backends (vLLM, llama.cpp).
|
||||
|
||||
Args:
|
||||
request: Model load request.
|
||||
client: LLM client instance.
|
||||
|
||||
Returns:
|
||||
ModelLoadResponse: Result of the load operation.
|
||||
|
||||
Raises:
|
||||
HTTPException: If loading fails or backend doesn't support it.
|
||||
"""
|
||||
try:
|
||||
success = await client.load_model(request.model, request.backend)
|
||||
return ModelLoadResponse(
|
||||
success=success,
|
||||
model=request.model,
|
||||
backend=request.backend.value,
|
||||
message="Model loaded successfully" if success else "Model load failed",
|
||||
)
|
||||
except NotImplementedError as e:
|
||||
raise HTTPException(status_code=400, detail=str(e)) from None
|
||||
except (BackendNotAvailableError, BackendNotEnabledError) as e:
|
||||
raise HTTPException(status_code=400, detail=str(e)) from None
|
||||
except ModelLoadError as e:
|
||||
raise HTTPException(status_code=500, detail=str(e)) from None
|
||||
|
||||
|
||||
@router.post("/models/unload", response_model=ModelLoadResponse)
|
||||
async def unload_model(
|
||||
request: ModelLoadRequest,
|
||||
client: LLMClient = Depends(get_client),
|
||||
) -> ModelLoadResponse:
|
||||
"""Unload a model from the specified backend.
|
||||
|
||||
This is only supported by local backends (vLLM, llama.cpp).
|
||||
|
||||
Args:
|
||||
request: Model unload request.
|
||||
client: LLM client instance.
|
||||
|
||||
Returns:
|
||||
ModelLoadResponse: Result of the unload operation.
|
||||
|
||||
Raises:
|
||||
HTTPException: If unloading fails or backend doesn't support it.
|
||||
"""
|
||||
try:
|
||||
success = await client.unload_model(request.model, request.backend)
|
||||
return ModelLoadResponse(
|
||||
success=success,
|
||||
model=request.model,
|
||||
backend=request.backend.value,
|
||||
message="Model unloaded successfully" if success else "Model unload failed",
|
||||
)
|
||||
except NotImplementedError as e:
|
||||
raise HTTPException(status_code=400, detail=str(e)) from None
|
||||
except (BackendNotAvailableError, BackendNotEnabledError) as e:
|
||||
raise HTTPException(status_code=400, detail=str(e)) from None
|
||||
|
||||
|
||||
@router.get("/backends", response_model=BackendListResponse)
|
||||
async def list_backends(
|
||||
client: LLMClient = Depends(get_client),
|
||||
) -> BackendListResponse:
|
||||
"""List available backends.
|
||||
|
||||
Args:
|
||||
client: LLM client instance.
|
||||
|
||||
Returns:
|
||||
BackendListResponse: List of available backend names.
|
||||
"""
|
||||
backends = [b.value for b in client.list_backends()]
|
||||
return BackendListResponse(backends=backends)
|
||||
Loading…
Add table
Add a link
Reference in a new issue