didi-lot1-ai/ai_platform/modules/llm-inference/deploy/docker-compose.yml

# LLM Inference Module - Docker Compose Configuration
#
# Port Allocation:
#   14001 - vLLM Qwen3.5-35B-A3B (text + vision MoE, native multimodal)
#   14011 - LLM Inference API (unified router)
#
# Profiles:
#   api      - API server only (uses external LLM services)
#   vllm     - API + vLLM servers (GPU required)
#
# Model:
#   - Qwen3.5-35B-A3B (~57GB weights BF16, gpu-util 0.45)
#   - Native vision support (replaces separate Qwen3-VL)
#
# Hardware: 1x NVIDIA H200 NVL (~143GB VRAM)
#   - GPU 0: Qwen3.5-35B-A3B + Whisper (~2GB)
#
# Naming Convention: didiAI-{module}-{service}
#
# Network:
#   Uses deploy_default network (shared with other modules)

networks:
  deploy_default:
    external: true

services:
  # ==========================================================================
  # LLM Inference API Server
  # ==========================================================================
  llm-api:
    container_name: didiAI-llm-api
    image: didiai-llm-api
    build:
      context: ..
      dockerfile: deploy/Dockerfile
    ports:
      - "14011:14011"
    networks:
      - deploy_default
    environment:
      - LLM_PORT=14011
      - LLM_EXTERNAL_URL=${LLM_EXTERNAL_URL}
      - LLM_DEFAULT_BACKEND=${LLM_DEFAULT_BACKEND}
      - LLM_ENABLE_VLLM=${LLM_ENABLE_VLLM}
      - LLM_ENABLE_LLAMACPP=${LLM_ENABLE_LLAMACPP}
      - LLM_VLLM_BASE_URL=http://didiAI-vllm-qwen3.5:14001
      - LLM_LLAMACPP_BASE_URLS=${LLM_LLAMACPP_BASE_URLS:-}
      - LLM_API_TOKENS=${LLM_API_TOKENS:-}
      - LLM_DASHBOARD_URL=${LLM_DASHBOARD_URL:-http://didiAI-dashboard:51300}
      - OPENROUTER_API_KEY=${OPENROUTER_API_KEY:-}
      - OPENAI_API_KEY=${OPENAI_API_KEY:-}
      - ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY:-}
    healthcheck:
      test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:14011/health')"]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 30s
    restart: unless-stopped
    profiles:
      - api
      - vllm

  # ==========================================================================
  # vLLM Server - Qwen3.5-35B-A3B (Unified Text + Vision MoE Model)
  # ==========================================================================
  # Native multimodal - replaces separate Qwen3 text + Qwen3-VL vision
  vllm-qwen3.5:
    container_name: didiAI-vllm-qwen3.5
    image: vllm/vllm-openai:qwen3_5
    ports:
      - "14001:14001"
    networks:
      - deploy_default
    volumes:
      - ${HF_CACHE_DIR}:/root/.cache/huggingface
    environment:
      - HF_HOME=/root/.cache/huggingface
      - HUGGING_FACE_HUB_TOKEN=${HF_TOKEN:-}
      - CUDA_VISIBLE_DEVICES=0
      - VLLM_ALLOW_LONG_MAX_MODEL_LEN=1
    command: >
      --model Qwen/Qwen3.5-35B-A3B
      --host 0.0.0.0
      --port 14001
      --served-model-name qwen3.5
      --tensor-parallel-size 1
      --max-model-len 32000
      --gpu-memory-utilization 0.65
      --trust-remote-code
      --enable-prefix-caching
      --disable-log-requests
      --enable-auto-tool-choice
      --tool-call-parser hermes
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              device_ids: ['0']
              capabilities: [gpu]
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:14001/health"]
      interval: 30s
      timeout: 10s
      retries: 10
      start_period: 600s
    restart: unless-stopped
    profiles:
      - vllm

  # ==========================================================================
  # llama.cpp Server (CPU/Metal Inference) - Optional
  # ==========================================================================
  llamacpp:
    image: ghcr.io/ggml-org/llama.cpp:server-b4769
    expose:
      - "14011"
    volumes:
      - ${MODELS_DIR:-/cai2_ds_storage/models}:/models:ro
    command: >
      --model /models/${LLAMACPP_MODEL:-model.gguf}
      --host 0.0.0.0
      --port 14011
      --ctx-size 4096
      --threads ${LLAMACPP_THREADS:-4}
      --parallel ${LLAMACPP_PARALLEL:-1}
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:14011/health"]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 60s
    restart: unless-stopped
    profiles:
      - llamacpp
      - full

volumes:
  vllm-cache:
    name: llm-inference-vllm-cache