131 lines
4.2 KiB
YAML
131 lines
4.2 KiB
YAML
# Rerank Module - Docker Compose Configuration
|
|
#
|
|
# Port Allocation (x42xx = Reranking):
|
|
# 14200 - Rerank API (Prod)
|
|
# 54200 - Rerank API (Dev)
|
|
# 14201/54201 - vLLM Rerank Server
|
|
# 14210/54210 - llama.cpp Rerank Server
|
|
#
|
|
# Profiles:
|
|
# api - API server only (uses external rerank servers)
|
|
# vllm - API + vLLM server (GPU required)
|
|
# llamacpp - API + llama.cpp server
|
|
#
|
|
# Naming Convention: didiAI-{module}-{service}
|
|
#
|
|
# Network:
|
|
# Uses deploy_default network (shared with other modules)
|
|
|
|
networks:
|
|
deploy_default:
|
|
external: true
|
|
|
|
services:
|
|
# ==========================================================================
|
|
# Rerank API Server
|
|
# ==========================================================================
|
|
rerank-api:
|
|
container_name: didiAI-rerank-api
|
|
image: didiai-rerank-api
|
|
build:
|
|
context: ..
|
|
dockerfile: deploy/Dockerfile
|
|
ports:
|
|
- "${RERANK_PORT:-14200}:${RERANK_PORT:-14200}"
|
|
networks:
|
|
- deploy_default
|
|
environment:
|
|
- RERANK_PORT=${RERANK_PORT:-14200}
|
|
- RERANK_EXTERNAL_URL=${RERANK_EXTERNAL_URL}
|
|
- RERANK_DEFAULT_BACKEND=${RERANK_DEFAULT_BACKEND}
|
|
- RERANK_ENABLE_VLLM=${RERANK_ENABLE_VLLM}
|
|
- RERANK_ENABLE_LLAMACPP=${RERANK_ENABLE_LLAMACPP}
|
|
- RERANK_VLLM_BASE_URL=http://didiAI-rerank-vllm:14201
|
|
- RERANK_LLAMACPP_BASE_URL=http://didiAI-rerank-llamacpp:8080
|
|
- RERANK_API_TOKENS=${RERANK_API_TOKENS:-}
|
|
- RERANK_DASHBOARD_URL=${RERANK_DASHBOARD_URL:-http://didiAI-dashboard:51300}
|
|
healthcheck:
|
|
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:${RERANK_PORT:-14200}/health')"]
|
|
interval: 30s
|
|
timeout: 10s
|
|
retries: 3
|
|
start_period: 30s
|
|
restart: unless-stopped
|
|
profiles:
|
|
- api
|
|
- vllm
|
|
- llamacpp
|
|
|
|
# ==========================================================================
|
|
# vLLM Rerank Server
|
|
# ==========================================================================
|
|
# Cross-encoder model using vLLM with --task score (v0.8.x syntax)
|
|
vllm-rerank:
|
|
container_name: didiAI-rerank-vllm
|
|
image: vllm/vllm-openai:v0.8.5
|
|
ports:
|
|
- "${RERANK_VLLM_PORT:-14201}:14201"
|
|
networks:
|
|
- deploy_default
|
|
volumes:
|
|
- ${HF_CACHE_DIR:-/cai2_ds_storage/hf_cache}:/root/.cache/huggingface
|
|
environment:
|
|
- HF_HOME=/root/.cache/huggingface
|
|
- HUGGING_FACE_HUB_TOKEN=${HF_TOKEN:-}
|
|
- CUDA_VISIBLE_DEVICES=${RERANK_VLLM_GPU:-0}
|
|
command: >
|
|
--model ${RERANK_VLLM_MODEL:-BAAI/bge-reranker-v2-m3}
|
|
--host 0.0.0.0
|
|
--port 14201
|
|
--task score
|
|
--trust-remote-code
|
|
--max-model-len ${RERANK_VLLM_MAX_LEN:-8192}
|
|
--gpu-memory-utilization ${RERANK_VLLM_GPU_UTIL:-0.50}
|
|
--disable-log-requests
|
|
deploy:
|
|
resources:
|
|
reservations:
|
|
devices:
|
|
- driver: nvidia
|
|
device_ids: ['${RERANK_VLLM_GPU:-0}']
|
|
capabilities: [gpu]
|
|
healthcheck:
|
|
test: ["CMD", "curl", "-f", "http://localhost:14201/health"]
|
|
interval: 30s
|
|
timeout: 10s
|
|
retries: 10
|
|
start_period: 300s
|
|
restart: unless-stopped
|
|
profiles:
|
|
- vllm
|
|
|
|
# ==========================================================================
|
|
# llama.cpp Rerank Server
|
|
# ==========================================================================
|
|
# Cross-encoder model using llama.cpp with --reranking
|
|
llamacpp-rerank:
|
|
container_name: didiAI-rerank-llamacpp
|
|
image: ghcr.io/ggml-org/llama.cpp:server-b4769
|
|
ports:
|
|
- "${RERANK_LLAMACPP_PORT:-14210}:8080"
|
|
networks:
|
|
- deploy_default
|
|
volumes:
|
|
- ${MODELS_DIR:-/cai2_ds_storage/models}:/models:ro
|
|
command: >
|
|
--model /models/${RERANK_LLAMACPP_MODEL:-bge-reranker-v2-m3-q4_k_m.gguf}
|
|
--host 0.0.0.0
|
|
--port 8080
|
|
--reranking
|
|
--ctx-size ${RERANK_LLAMACPP_CTX:-8192}
|
|
--threads ${RERANK_LLAMACPP_THREADS:-4}
|
|
--parallel ${RERANK_LLAMACPP_PARALLEL:-4}
|
|
healthcheck:
|
|
test: ["CMD", "curl", "-f", "http://localhost:8080/health"]
|
|
interval: 30s
|
|
timeout: 10s
|
|
retries: 3
|
|
start_period: 60s
|
|
restart: unless-stopped
|
|
profiles:
|
|
- llamacpp
|