Livrare LOT 1 - Didi
This commit is contained in:
commit
5380c3fc63
990 changed files with 133308 additions and 0 deletions
43
ai_platform/modules/rerank/deploy/Dockerfile
Normal file
43
ai_platform/modules/rerank/deploy/Dockerfile
Normal file
|
|
@ -0,0 +1,43 @@
|
|||
# Rerank API Server
|
||||
# Multi-stage build for smaller final image
|
||||
|
||||
FROM python:3.11.12-slim AS builder
|
||||
|
||||
# Install uv
|
||||
COPY --from=ghcr.io/astral-sh/uv:0.10 /uv /usr/local/bin/uv
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# Copy project files
|
||||
COPY pyproject.toml README.md ./
|
||||
COPY src/ ./src/
|
||||
|
||||
# Install dependencies (with optional extras for backends)
|
||||
RUN uv sync --frozen --no-dev --all-extras || uv sync --no-dev --all-extras
|
||||
|
||||
# Production image
|
||||
FROM python:3.11.12-slim
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# Copy virtual environment from builder
|
||||
COPY --from=builder /app/.venv /app/.venv
|
||||
|
||||
# Copy source code
|
||||
COPY src/ ./src/
|
||||
|
||||
# Set environment variables
|
||||
ENV PATH="/app/.venv/bin:$PATH"
|
||||
ENV PYTHONUNBUFFERED=1
|
||||
ENV RERANK_HOST=0.0.0.0
|
||||
ENV RERANK_PORT=14200
|
||||
|
||||
# Health check (using Python since curl not available in slim image)
|
||||
HEALTHCHECK --interval=30s --timeout=10s --retries=3 \
|
||||
CMD python -c "import urllib.request; urllib.request.urlopen('http://localhost:14200/health')" || exit 1
|
||||
|
||||
# Expose port
|
||||
EXPOSE 14200
|
||||
|
||||
# Run the server
|
||||
CMD ["python", "-m", "rerank.cli"]
|
||||
142
ai_platform/modules/rerank/deploy/deploy.sh
Normal file
142
ai_platform/modules/rerank/deploy/deploy.sh
Normal file
|
|
@ -0,0 +1,142 @@
|
|||
#!/usr/bin/env bash
|
||||
#
|
||||
# Docker Compose Startup Script for Rerank
|
||||
#
|
||||
# Usage: ./deploy/deploy.sh [OPTIONS]
|
||||
#
|
||||
# Options:
|
||||
# --profile <api|vllm|llamacpp> Docker compose profile
|
||||
# --detach Run in detached mode
|
||||
# --down Stop and remove containers
|
||||
# --logs Show logs
|
||||
# --help Show this help message
|
||||
#
|
||||
# Required: Set environment variables in .env file or export them before running.
|
||||
# See .env.example for the full list of required variables.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# Script directory
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
# Load .env file if it exists (check deploy/ first, then module root)
|
||||
ENV_FILE=""
|
||||
if [[ -f "$SCRIPT_DIR/.env" ]]; then
|
||||
ENV_FILE="$SCRIPT_DIR/.env"
|
||||
elif [[ -f "$SCRIPT_DIR/../.env" ]]; then
|
||||
ENV_FILE="$SCRIPT_DIR/../.env"
|
||||
fi
|
||||
|
||||
if [[ -n "$ENV_FILE" ]]; then
|
||||
echo "Loading environment from: $ENV_FILE"
|
||||
set -a
|
||||
source "$ENV_FILE"
|
||||
set +a
|
||||
fi
|
||||
|
||||
PROFILE=""
|
||||
DETACH=""
|
||||
ACTION="up"
|
||||
|
||||
show_help() {
|
||||
sed -n '2,14p' "$0" | sed 's/^# //' | sed 's/^#//'
|
||||
exit 0
|
||||
}
|
||||
|
||||
check_required_var() {
|
||||
local var_name="$1"
|
||||
if [[ -z "${!var_name:-}" ]]; then
|
||||
echo "ERROR: Required environment variable $var_name is not set"
|
||||
echo "Set it in .env file or export it before running this script"
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Parse arguments
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case $1 in
|
||||
--profile)
|
||||
PROFILE="$2"
|
||||
shift 2
|
||||
;;
|
||||
--detach|-d)
|
||||
DETACH="-d"
|
||||
shift
|
||||
;;
|
||||
--down)
|
||||
ACTION="down"
|
||||
shift
|
||||
;;
|
||||
--logs)
|
||||
ACTION="logs"
|
||||
shift
|
||||
;;
|
||||
--help|-h)
|
||||
show_help
|
||||
;;
|
||||
*)
|
||||
echo "Unknown option: $1"
|
||||
echo "Use --help for usage information"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
# Require profile to be specified
|
||||
if [[ -z "$PROFILE" && "$ACTION" == "up" ]]; then
|
||||
echo "ERROR: --profile is required"
|
||||
echo "Options: api, vllm, llamacpp"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check required variables
|
||||
check_required_var "RERANK_DEFAULT_BACKEND"
|
||||
check_required_var "RERANK_ENABLE_VLLM"
|
||||
check_required_var "RERANK_ENABLE_LLAMACPP"
|
||||
check_required_var "RERANK_EXTERNAL_URL"
|
||||
|
||||
# Check profile-specific variables
|
||||
case $PROFILE in
|
||||
vllm)
|
||||
check_required_var "RERANK_VLLM_MODEL"
|
||||
;;
|
||||
llamacpp)
|
||||
check_required_var "MODELS_DIR"
|
||||
check_required_var "RERANK_LLAMACPP_MODEL"
|
||||
;;
|
||||
esac
|
||||
|
||||
cd "$SCRIPT_DIR"
|
||||
|
||||
case $ACTION in
|
||||
up)
|
||||
echo "Starting Rerank with profile: $PROFILE"
|
||||
echo " Default backend: $RERANK_DEFAULT_BACKEND"
|
||||
echo " vLLM enabled: $RERANK_ENABLE_VLLM"
|
||||
echo " llama.cpp enabled: $RERANK_ENABLE_LLAMACPP"
|
||||
if [[ "$PROFILE" == "vllm" ]]; then
|
||||
echo " vLLM model: ${RERANK_VLLM_MODEL:-BAAI/bge-reranker-v2-m3}"
|
||||
fi
|
||||
if [[ "$PROFILE" == "llamacpp" ]]; then
|
||||
echo " llama.cpp model: ${RERANK_LLAMACPP_MODEL:-bge-reranker-v2-m3-q4_k_m.gguf}"
|
||||
fi
|
||||
echo ""
|
||||
# shellcheck disable=SC2086
|
||||
exec docker compose --profile "$PROFILE" up $DETACH
|
||||
;;
|
||||
down)
|
||||
if [[ -z "$PROFILE" ]]; then
|
||||
echo "ERROR: --profile is required with --down"
|
||||
exit 1
|
||||
fi
|
||||
echo "Stopping Rerank containers..."
|
||||
exec docker compose --profile "$PROFILE" down
|
||||
;;
|
||||
logs)
|
||||
if [[ -z "$PROFILE" ]]; then
|
||||
echo "ERROR: --profile is required with --logs"
|
||||
exit 1
|
||||
fi
|
||||
exec docker compose --profile "$PROFILE" logs -f
|
||||
;;
|
||||
esac
|
||||
131
ai_platform/modules/rerank/deploy/docker-compose.yml
Normal file
131
ai_platform/modules/rerank/deploy/docker-compose.yml
Normal file
|
|
@ -0,0 +1,131 @@
|
|||
# Rerank Module - Docker Compose Configuration
|
||||
#
|
||||
# Port Allocation (x42xx = Reranking):
|
||||
# 14200 - Rerank API (Prod)
|
||||
# 54200 - Rerank API (Dev)
|
||||
# 14201/54201 - vLLM Rerank Server
|
||||
# 14210/54210 - llama.cpp Rerank Server
|
||||
#
|
||||
# Profiles:
|
||||
# api - API server only (uses external rerank servers)
|
||||
# vllm - API + vLLM server (GPU required)
|
||||
# llamacpp - API + llama.cpp server
|
||||
#
|
||||
# Naming Convention: didiAI-{module}-{service}
|
||||
#
|
||||
# Network:
|
||||
# Uses deploy_default network (shared with other modules)
|
||||
|
||||
networks:
|
||||
deploy_default:
|
||||
external: true
|
||||
|
||||
services:
|
||||
# ==========================================================================
|
||||
# Rerank API Server
|
||||
# ==========================================================================
|
||||
rerank-api:
|
||||
container_name: didiAI-rerank-api
|
||||
image: didiai-rerank-api
|
||||
build:
|
||||
context: ..
|
||||
dockerfile: deploy/Dockerfile
|
||||
ports:
|
||||
- "${RERANK_PORT:-14200}:${RERANK_PORT:-14200}"
|
||||
networks:
|
||||
- deploy_default
|
||||
environment:
|
||||
- RERANK_PORT=${RERANK_PORT:-14200}
|
||||
- RERANK_EXTERNAL_URL=${RERANK_EXTERNAL_URL}
|
||||
- RERANK_DEFAULT_BACKEND=${RERANK_DEFAULT_BACKEND}
|
||||
- RERANK_ENABLE_VLLM=${RERANK_ENABLE_VLLM}
|
||||
- RERANK_ENABLE_LLAMACPP=${RERANK_ENABLE_LLAMACPP}
|
||||
- RERANK_VLLM_BASE_URL=http://didiAI-rerank-vllm:14201
|
||||
- RERANK_LLAMACPP_BASE_URL=http://didiAI-rerank-llamacpp:8080
|
||||
- RERANK_API_TOKENS=${RERANK_API_TOKENS:-}
|
||||
- RERANK_DASHBOARD_URL=${RERANK_DASHBOARD_URL:-http://didiAI-dashboard:51300}
|
||||
healthcheck:
|
||||
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:${RERANK_PORT:-14200}/health')"]
|
||||
interval: 30s
|
||||
timeout: 10s
|
||||
retries: 3
|
||||
start_period: 30s
|
||||
restart: unless-stopped
|
||||
profiles:
|
||||
- api
|
||||
- vllm
|
||||
- llamacpp
|
||||
|
||||
# ==========================================================================
|
||||
# vLLM Rerank Server
|
||||
# ==========================================================================
|
||||
# Cross-encoder model using vLLM with --task score (v0.8.x syntax)
|
||||
vllm-rerank:
|
||||
container_name: didiAI-rerank-vllm
|
||||
image: vllm/vllm-openai:v0.8.5
|
||||
ports:
|
||||
- "${RERANK_VLLM_PORT:-14201}:14201"
|
||||
networks:
|
||||
- deploy_default
|
||||
volumes:
|
||||
- ${HF_CACHE_DIR:-/cai2_ds_storage/hf_cache}:/root/.cache/huggingface
|
||||
environment:
|
||||
- HF_HOME=/root/.cache/huggingface
|
||||
- HUGGING_FACE_HUB_TOKEN=${HF_TOKEN:-}
|
||||
- CUDA_VISIBLE_DEVICES=${RERANK_VLLM_GPU:-0}
|
||||
command: >
|
||||
--model ${RERANK_VLLM_MODEL:-BAAI/bge-reranker-v2-m3}
|
||||
--host 0.0.0.0
|
||||
--port 14201
|
||||
--task score
|
||||
--trust-remote-code
|
||||
--max-model-len ${RERANK_VLLM_MAX_LEN:-8192}
|
||||
--gpu-memory-utilization ${RERANK_VLLM_GPU_UTIL:-0.50}
|
||||
--disable-log-requests
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
device_ids: ['${RERANK_VLLM_GPU:-0}']
|
||||
capabilities: [gpu]
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "-f", "http://localhost:14201/health"]
|
||||
interval: 30s
|
||||
timeout: 10s
|
||||
retries: 10
|
||||
start_period: 300s
|
||||
restart: unless-stopped
|
||||
profiles:
|
||||
- vllm
|
||||
|
||||
# ==========================================================================
|
||||
# llama.cpp Rerank Server
|
||||
# ==========================================================================
|
||||
# Cross-encoder model using llama.cpp with --reranking
|
||||
llamacpp-rerank:
|
||||
container_name: didiAI-rerank-llamacpp
|
||||
image: ghcr.io/ggml-org/llama.cpp:server-b4769
|
||||
ports:
|
||||
- "${RERANK_LLAMACPP_PORT:-14210}:8080"
|
||||
networks:
|
||||
- deploy_default
|
||||
volumes:
|
||||
- ${MODELS_DIR:-/cai2_ds_storage/models}:/models:ro
|
||||
command: >
|
||||
--model /models/${RERANK_LLAMACPP_MODEL:-bge-reranker-v2-m3-q4_k_m.gguf}
|
||||
--host 0.0.0.0
|
||||
--port 8080
|
||||
--reranking
|
||||
--ctx-size ${RERANK_LLAMACPP_CTX:-8192}
|
||||
--threads ${RERANK_LLAMACPP_THREADS:-4}
|
||||
--parallel ${RERANK_LLAMACPP_PARALLEL:-4}
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "-f", "http://localhost:8080/health"]
|
||||
interval: 30s
|
||||
timeout: 10s
|
||||
retries: 3
|
||||
start_period: 60s
|
||||
restart: unless-stopped
|
||||
profiles:
|
||||
- llamacpp
|
||||
Loading…
Add table
Add a link
Reference in a new issue