Livrare LOT 1 - Didi

ai_platform/modules/llm-inference/deploy/docker-compose.yml

@@ -0,0 +1,142 @@
+# LLM Inference Module - Docker Compose Configuration
+#
+# Port Allocation:
+#   14001 - vLLM Qwen3.5-35B-A3B (text + vision MoE, native multimodal)
+#   14011 - LLM Inference API (unified router)
+#
+# Profiles:
+#   api      - API server only (uses external LLM services)
+#   vllm     - API + vLLM servers (GPU required)
+#
+# Model:
+#   - Qwen3.5-35B-A3B (~57GB weights BF16, gpu-util 0.45)
+#   - Native vision support (replaces separate Qwen3-VL)
+#
+# Hardware: 1x NVIDIA H200 NVL (~143GB VRAM)
+#   - GPU 0: Qwen3.5-35B-A3B + Whisper (~2GB)
+#
+# Naming Convention: didiAI-{module}-{service}
+#
+# Network:
+#   Uses deploy_default network (shared with other modules)
+
+networks:
+  deploy_default:
+    external: true
+
+services:
+  # ==========================================================================
+  # LLM Inference API Server
+  # ==========================================================================
+  llm-api:
+    container_name: didiAI-llm-api
+    image: didiai-llm-api
+    build:
+      context: ..
+      dockerfile: deploy/Dockerfile
+    ports:
+      - "14011:14011"
+    networks:
+      - deploy_default
+    environment:
+      - LLM_PORT=14011
+      - LLM_EXTERNAL_URL=${LLM_EXTERNAL_URL}
+      - LLM_DEFAULT_BACKEND=${LLM_DEFAULT_BACKEND}
+      - LLM_ENABLE_VLLM=${LLM_ENABLE_VLLM}
+      - LLM_ENABLE_LLAMACPP=${LLM_ENABLE_LLAMACPP}
+      - LLM_VLLM_BASE_URL=http://didiAI-vllm-qwen3.5:14001
+      - LLM_LLAMACPP_BASE_URLS=${LLM_LLAMACPP_BASE_URLS:-}
+      - LLM_API_TOKENS=${LLM_API_TOKENS:-}
+      - LLM_DASHBOARD_URL=${LLM_DASHBOARD_URL:-http://didiAI-dashboard:51300}
+      - OPENROUTER_API_KEY=${OPENROUTER_API_KEY:-}
+      - OPENAI_API_KEY=${OPENAI_API_KEY:-}
+      - ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY:-}
+    healthcheck:
+      test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:14011/health')"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+      start_period: 30s
+    restart: unless-stopped
+    profiles:
+      - api
+      - vllm
+
+  # ==========================================================================
+  # vLLM Server - Qwen3.5-35B-A3B (Unified Text + Vision MoE Model)
+  # ==========================================================================
+  # Native multimodal - replaces separate Qwen3 text + Qwen3-VL vision
+  vllm-qwen3.5:
+    container_name: didiAI-vllm-qwen3.5
+    image: vllm/vllm-openai:qwen3_5
+    ports:
+      - "14001:14001"
+    networks:
+      - deploy_default
+    volumes:
+      - ${HF_CACHE_DIR}:/root/.cache/huggingface
+    environment:
+      - HF_HOME=/root/.cache/huggingface
+      - HUGGING_FACE_HUB_TOKEN=${HF_TOKEN:-}
+      - CUDA_VISIBLE_DEVICES=0
+      - VLLM_ALLOW_LONG_MAX_MODEL_LEN=1
+    command: >
+      --model Qwen/Qwen3.5-35B-A3B
+      --host 0.0.0.0
+      --port 14001
+      --served-model-name qwen3.5
+      --tensor-parallel-size 1
+      --max-model-len 32000
+      --gpu-memory-utilization 0.65
+      --trust-remote-code
+      --enable-prefix-caching
+      --disable-log-requests
+      --enable-auto-tool-choice
+      --tool-call-parser hermes
+    deploy:
+      resources:
+        reservations:
+          devices:
+            - driver: nvidia
+              device_ids: ['0']
+              capabilities: [gpu]
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:14001/health"]
+      interval: 30s
+      timeout: 10s
+      retries: 10
+      start_period: 600s
+    restart: unless-stopped
+    profiles:
+      - vllm
+
+  # ==========================================================================
+  # llama.cpp Server (CPU/Metal Inference) - Optional
+  # ==========================================================================
+  llamacpp:
+    image: ghcr.io/ggml-org/llama.cpp:server-b4769
+    expose:
+      - "14011"
+    volumes:
+      - ${MODELS_DIR:-/cai2_ds_storage/models}:/models:ro
+    command: >
+      --model /models/${LLAMACPP_MODEL:-model.gguf}
+      --host 0.0.0.0
+      --port 14011
+      --ctx-size 4096
+      --threads ${LLAMACPP_THREADS:-4}
+      --parallel ${LLAMACPP_PARALLEL:-1}
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:14011/health"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+      start_period: 60s
+    restart: unless-stopped
+    profiles:
+      - llamacpp
+      - full
+
+volumes:
+  vllm-cache:
+    name: llm-inference-vllm-cache