#!/usr/bin/env bash # ============================================================================= # DiDi LOT 1 — deploy progresiv „cărămidă cu cărămidă". # Pentru fiecare componentă: build/pull + download → seed (dacă e cazul) → # TEST funcțional → doar dacă testul TRECE corect → trece la următoarea. # Idempotent. STOP la primul test picat. # # ./deploy.sh # tot, de la preflight la dashboard # ./deploy.sh --from web # sare peste cărămizile dinainte de 'web' # ./deploy.sh --only llm # rulează o singură cărămidă (presupune deps up) # ============================================================================= set -uo pipefail cd "$(dirname "$0")" HERE=$(pwd) export COMPOSE_PROJECT_NAME=local_gpu_stack COMPOSE="${HERE}/docker-compose.yml" ENV_FILE="${HERE}/.env" MODULES="${HERE}/../modules" DASH_DIR="${MODULES}/dashboard/deploy" GW_DIR="${MODULES}/gateway/deploy" BRAIN_DIR="${MODULES}/didi_brain" ASSETS="${HERE}/test_assets" source "${HERE}/lib.sh" source "${HERE}/seed/seed_catalog.sh" source "${HERE}/seed/seed_brain.sh" # ---- .env ------------------------------------------------------------------- [ -f "$ENV_FILE" ] || { cp "${HERE}/.env.example" "$ENV_FILE"; info ".env creat din .env.example"; } set -a; source "$ENV_FILE"; set +a # auto-detect HOST_IP daca e gol (si scrie-l inapoi in .env pt interpolarea compose) if [ -z "${HOST_IP:-}" ]; then HOST_IP=$(hostname -I 2>/dev/null | awk '{print $1}') [ -z "$HOST_IP" ] && HOST_IP=$(ip -4 route get 1.1.1.1 2>/dev/null | awk '{for(i=1;i<=NF;i++)if($i=="src")print $(i+1)}') [ -n "$HOST_IP" ] && { sed -i "s#^HOST_IP=.*#HOST_IP=${HOST_IP}#" "$ENV_FILE"; info "HOST_IP detectat automat: ${HOST_IP}"; } fi export HOST_IP H="${HOST_IP:-127.0.0.1}" # MODELS_DIR: zona locala unica pentru TOATE modelele (default: ai_platform/models) if [ -z "${MODELS_DIR:-}" ]; then MODELS_DIR="$(cd "${HERE}/.." && pwd)/models" sed -i "s#^MODELS_DIR=.*#MODELS_DIR=${MODELS_DIR}#" "$ENV_FILE" info "MODELS_DIR: ${MODELS_DIR}" fi mkdir -p "$MODELS_DIR"; export MODELS_DIR # ---- arg parsing ------------------------------------------------------------ FROM=""; ONLY=""; INSTALL_DEPS=0 while [ $# -gt 0 ]; do case "$1" in --from) FROM=$2; shift 2;; --only) ONLY=$2; shift 2;; --install-deps) INSTALL_DEPS=1; shift;; *) die "argument necunoscut: $1";; esac; done SKIP=0; [ -n "$FROM" ] && SKIP=1 should_run(){ # $1 = brick name [ -n "$ONLY" ] && { [ "$ONLY" = "$1" ]; return; } [ "$SKIP" = 1 ] && { [ "$FROM" = "$1" ] && SKIP=0; } [ "$SKIP" = 0 ] } # ============================================================================= # PREFLIGHT (verifică + instruiește; nu instalează) # ============================================================================= # ============================================================================= # INSTALL DEPS (optional, --install-deps; necesita sudo; Ubuntu/Debian) # ============================================================================= install_deps() { brick "INSTALL DEPS — Docker + NVIDIA Container Toolkit (sudo)" command -v sudo >/dev/null || die "sudo necesar pentru --install-deps" if ! command -v docker >/dev/null; then info "instalez Docker Engine..."; curl -fsSL https://get.docker.com | sudo sh || die "instalare Docker a picat" sudo usermod -aG docker "$USER" || true warn "Adăugat în grupul docker — poate fi nevoie de re-login pentru ca 'docker' să meargă fără sudo." else ok "Docker prezent"; fi if ! docker info 2>/dev/null | grep -qiE 'nvidia|cdi' && ! sudo docker info 2>/dev/null | grep -qiE 'nvidia|cdi'; then info "instalez nvidia-container-toolkit..." curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey | sudo gpg --dearmor -o /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg curl -s -L https://nvidia.github.io/libnvidia-container/stable/deb/nvidia-container-toolkit.list \ | sed 's#deb https://#deb [signed-by=/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg] https://#g' \ | sudo tee /etc/apt/sources.list.d/nvidia-container-toolkit.list >/dev/null sudo apt-get update -y && sudo apt-get install -y nvidia-container-toolkit || die "instalare toolkit a picat" sudo nvidia-ctk runtime configure --runtime=docker && sudo systemctl restart docker ok "nvidia-container-toolkit instalat & Docker reconfigurat" else ok "NVIDIA runtime prezent"; fi command -v nvidia-smi >/dev/null || warn "Driver NVIDIA lipsește — instalează manual (ex: sudo ubuntu-drivers install) și REBOOT, apoi re-rulează deploy.sh." } preflight() { brick "PREFLIGHT — unelte & GPU" command -v docker >/dev/null || die "Docker lipsește. Instalează: https://docs.docker.com/engine/install/ubuntu/" if ! docker ps >/dev/null 2>&1; then [ "$USE_SG" = 1 ] && warn "rulez docker prin 'sg docker' (userul nu e în grupul docker activ)" \ || die "Nu pot accesa Docker. Adaugă userul în grup: sudo usermod -aG docker \$USER (apoi re-login)" fi docker compose version >/dev/null 2>&1 || die "Docker Compose v2 lipsește (plugin 'docker compose')." command -v nvidia-smi >/dev/null || die "nvidia-smi lipsește — instalează driverul NVIDIA (>=535) + CUDA." local ngpu; ngpu=$(nvidia-smi --query-gpu=name --format=csv,noheader 2>/dev/null | wc -l) [ "$ngpu" -ge 1 ] || die "Niciun GPU detectat de nvidia-smi." if ! docker info 2>/dev/null | grep -qiE 'nvidia|cdi'; then die "Runtime-ul NVIDIA pt Docker lipsește. Instalează nvidia-container-toolkit: https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html apoi: sudo nvidia-ctk runtime configure --runtime=docker && sudo systemctl restart docker" fi command -v curl >/dev/null || die "curl lipsește (apt-get install -y curl)." command -v python3 >/dev/null || die "python3 lipsește." local freeg; freeg=$(df -BG --output=avail / | tail -1 | tr -dc '0-9') [ "${freeg:-0}" -ge 80 ] || warn "Doar ${freeg}G liberi pe / — Qwen are ~72GB. Recomandat >100G." ok "Docker $(docker --version | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -1) · ${ngpu} GPU · ${freeg}G liberi" } # ============================================================================= # NETWORK # ============================================================================= ensure_network() { brick "REȚEA — didi-network (una singură, partajată)" if docker network inspect didi-network >/dev/null 2>&1; then ok "didi-network există" else docker network create didi-network >/dev/null && ok "didi-network creată"; fi } # ============================================================================= # BUILD images (app) + PULL (vLLM/searxng) # ============================================================================= build_images() { brick "BUILD — imagini aplicație" local m declare -A MAP=( [llm-inference]=didiai-llm-inference [embeddings]=didiai-embeddings [rerank]=didiai-rerank [audio]=didiai-audio [video-analysis]=didiai-video-analysis [extractors]=didiai-extractors [web]=didiai-web [catalog-api]=didiai-catalog-api ) for m in "${!MAP[@]}"; do printf ' build %-16s ' "$m" if docker build -q -t "${MAP[$m]}:audit" -f "${MODULES}/$m/deploy/Dockerfile" "${MODULES}/$m" >/tmp/build_$m.log 2>&1; then ok "ok" else err "FAIL"; tail -15 /tmp/build_$m.log | sed 's/^/ /'; die "build $m a picat"; fi done # forensic (layout non-standard: Dockerfile la root) printf ' build %-16s ' "forensic" if docker build -q -t didiai-forensic:audit -f "${MODULES}/forensic_features/Dockerfile" "${MODULES}/forensic_features" >/tmp/build_forensic.log 2>&1; then ok "ok" else err "FAIL"; tail -15 /tmp/build_forensic.log | sed 's/^/ /'; die "build forensic a picat"; fi gate "imagini prezente" bash -c 'docker images --format "{{.Repository}}" | grep -q didiai-llm-inference' } pull_images() { brick "PULL — runtime-uri vLLM + SearXNG" local img for img in vllm/vllm-openai:v0.8.5 vllm/vllm-openai:qwen3_5 docker.io/searxng/searxng:latest; do printf ' pull %-34s ' "$img" if docker image inspect "$img" >/dev/null 2>&1; then ok "deja prezent" elif docker pull -q "$img" >/dev/null 2>&1; then ok "ok" else die "pull $img a picat (verifică rețeaua / Docker Hub)"; fi done } # ============================================================================= # PRE-DOWNLOAD modele (cu progres) # ============================================================================= predownload_models() { brick "DOWNLOAD — modele în MODELS_DIR (offline-first, un folder per model)" mkdir -p "${MODELS_DIR}" # mini-script de download (evită iadul de ghilimele prin sg docker) local dl="${HERE}/.dl_model.py" printf 'import os\nfrom huggingface_hub import snapshot_download\nsnapshot_download(os.environ["REPO"])\nprint("OK", os.environ["REPO"])\n' > "$dl" # repo HF -> subfolder cache (models----) local repo cache for repo in "$BUSTER_MODEL" "$LLM_MODEL" "$EMB_MODEL" "$RERANK_MODEL" "$WHISPER_REPO" "$NER_MODEL" "$NER_BACKBONE"; do cache="models--$(printf '%s' "$repo" | sed 's#/#--#')" if [ -d "${MODELS_DIR}/${cache}/snapshots" ]; then ok "prezent local: ${repo}" else info "download: ${repo}" docker run --rm -e HF_HOME=/c -e "REPO=${repo}" \ -v "${MODELS_DIR}:/c/hub" -v "${dl}:/dl.py" \ --entrypoint python3 vllm/vllm-openai:qwen3_5 /dl.py \ || die "download ${repo} a picat." ok "${repo} descărcat" fi done rm -f "$dl" ok "Toate modelele sunt în ${MODELS_DIR}" } # ============================================================================= # TEST helpers (output corect, nu doar 200) # ============================================================================= t_embeddings(){ post_json "http://$H:14100/v1/embeddings" '{"model":"bge-m3","input":"test"}' 20 | python3 -c 'import sys,json;v=json.load(sys.stdin)["data"][0]["embedding"];assert len(v)==1024;print(f"dim={len(v)}")'; } t_rerank(){ post_json "http://$H:14200/v1/rerank" '{"model":"bge-reranker-v2-m3","query":"deepfake detection","documents":["o pisica","sistem detectie deepfake video"]}' 20 | python3 -c 'import sys,json;r=json.load(sys.stdin)["results"];assert r[0]["index"]==1;print("top_idx=%d score=%.3f"%(r[0]["index"],r[0]["relevance_score"]))'; } t_llm(){ post_json "http://$H:14011/v1/chat/completions" '{"model":"qwen3.5","messages":[{"role":"user","content":"Spune un singur cuvant: salut"}],"max_tokens":20}' 90 | python3 -c 'import sys,json;c=json.load(sys.stdin)["choices"][0]["message"]["content"].strip();assert c and not c.lower().startswith("thinking");print(repr(c[:40]))'; } t_audio(){ curl -s -m120 "http://$H:54300/v1/audio/transcriptions" -F "file=@${ASSETS}/audio.wav" -F "model=${WHISPER_MODEL}" | python3 -c 'import sys,json;t=json.load(sys.stdin)["text"].strip();assert len(t)>0;print(f"text[:40]={t[:40]!r}")'; } t_video(){ curl -s -m180 "http://$H:54600/analyze/video" -F "file=@${ASSETS}/sample.mp4" | python3 -c 'import sys,json;d=json.load(sys.stdin);assert d["verdict"] in ("REAL","FAKE","UNCERTAIN");print("verdict=%s frames=%s"%(d["verdict"],d.get("frames_analyzed")))'; } t_ner(){ post_json "http://$H:54400/v1/ner" '{"text":"Klaus Iohannis s-a intalnit cu Emmanuel Macron la Bucuresti."}' 60 | python3 -c 'import sys,json;e=json.load(sys.stdin)["results"]["entities"];assert len(e)>=2;print(f"entities={len(e)}")'; } t_detect(){ curl -s -m90 "http://$H:54400/v1/detect" -F "file=@${ASSETS}/frame.jpg" | python3 -c 'import sys,json;d=json.load(sys.stdin)["results"]["detections"];assert len(d)>=1;print(f"objects={len(d)}")'; } t_metadata(){ curl -s -m40 "http://$H:54400/v1/metadata" -F "file=@${ASSETS}/sample.mp4" | python3 -c 'import sys,json;d=json.load(sys.stdin);assert d["analyses"]["integrity"]["ok"];print("integrity ok")'; } t_sentiment(){ post_json "http://$H:54400/v1/sentiment" '{"text":"Produsul este excelent, sunt foarte multumit!"}' 60 | python3 -c 'import sys,json;d=json.load(sys.stdin);assert d["ok"];print("label=%s"%d["results"].get("label"))'; } t_forensic(){ curl -s -m15 "http://$H:8085/api/forensic-modules" | python3 -c 'import sys,json;m=json.load(sys.stdin)["available_modules"];assert len(m)>=3;print(f"modules={len(m)}")'; } t_web(){ post_json "http://$H:51100/v1/search" '{"queries":["deepfake detection"],"max_results":3}' 40 | python3 -c 'import sys,json;r=json.load(sys.stdin)["results"];assert len(r)>0;print(f"results={len(r)}")'; } t_catalog(){ curl -s -m15 "http://$H:11000/catalog/v1/status" -H "Authorization: Bearer ${GATEWAY_API_TOKEN}" | python3 -c 'import sys,json;d=json.load(sys.stdin);assert d["status"]=="healthy";print("catalog healthy")'; } t_gateway(){ local a b; a=$(curl -s -m10 -o /dev/null -w '%{http_code}' "http://$H:11000/health"); b=$(curl -s -m10 -o /dev/null -w '%{http_code}' "http://$H:11000/web/v1/info"); [ "$a" = 200 ] && [ "$b" = 401 ] && echo "health=200 noauth=401" || { echo "health=$a noauth=$b (astept 200/401)"; return 1; }; } t_brain(){ curl -s -m15 "http://$H:8090/v1/analysis_atom/stats/extended" | python3 -c 'import sys,json;json.load(sys.stdin);print("stats ok")' && post_json "http://$H:8090/v1/search" '{"queries":["deepfake"],"max_results":2}' 30 | python3 -c 'import sys,json;assert "results" in json.load(sys.stdin);print("search ok")'; } t_dashboard(){ curl -s -m15 "http://$H:51300/api/monitoring/services" | python3 -c 'import sys,json;d=json.load(sys.stdin);s=d if isinstance(d,list) else d.get("services",[]);h=sum(x.get("status")=="healthy" for x in s);assert h==len(s) and h>=11,f"{h}/{len(s)}";print(f"{h}/{len(s)} healthy")'; } prepare_assets() { mkdir -p "$ASSETS" [ -f "${ASSETS}/sample.mp4" ] || cp "$(find ${HERE}/../../artefacte_lot1/runs_deepfake -iname '*.mp4' -printf '%s\t%p\n' | sort -n | head -1 | cut -f2)" "${ASSETS}/sample.mp4" if [ ! -f "${ASSETS}/frame.jpg" ] || [ ! -f "${ASSETS}/audio.wav" ]; then docker cp "${ASSETS}/sample.mp4" didiAI-extractors:/tmp/s.mp4 >/dev/null docker exec didiAI-extractors sh -c 'ffmpeg -y -i /tmp/s.mp4 -frames:v 1 /tmp/frame.jpg -vn -ar 16000 -ac 1 -t 8 /tmp/audio.wav' >/dev/null 2>&1 docker cp didiAI-extractors:/tmp/frame.jpg "${ASSETS}/frame.jpg" >/dev/null docker cp didiAI-extractors:/tmp/audio.wav "${ASSETS}/audio.wav" >/dev/null fi } # ============================================================================= # CĂRĂMIZI (fiecare: up dependențe → health → test → gate) # ============================================================================= brick_embeddings(){ brick "EMBEDDINGS (bge-m3)"; dc up -d vllm-embeddings embeddings-api wait_health "http://$H:14100/health" 180 "embeddings-api" || die "embeddings-api nu pornește"; gate "embeddings dim 1024" t_embeddings; } brick_rerank(){ brick "RERANK (bge-reranker-v2-m3)"; dc up -d vllm-rerank rerank-api wait_health "http://$H:14200/health" 180 "rerank-api" || die "rerank-api nu pornește"; gate "rerank ranking" t_rerank; } brick_llm(){ brick "LLM (Qwen3.5-35B-A3B)"; dc up -d vllm-qwen llm-api wait_health "http://$H:14001/v1/models" 900 "vLLM Qwen (load+compile)" || die "vLLM Qwen nu pornește" wait_health "http://$H:14011/health" 60 "llm-api" || die "llm-api nu pornește"; gate "LLM output curat" t_llm; } brick_extractors(){ brick "EXTRACTORS (NER/YOLO/metadata/OCR/sentiment)"; dc up -d extractors wait_health "http://$H:54400/health" 120 "extractors" || die "extractors nu pornește" prepare_assets gate "metadata" t_metadata; gate "NER (GLiNER)" t_ner; gate "detect (YOLO)" t_detect; gate "sentiment (LLM)" t_sentiment; } brick_audio(){ brick "AUDIO (Whisper large-v3-turbo)"; dc up -d audio-api wait_health "http://$H:54300/health" 240 "audio (download model)" || die "audio nu pornește"; gate "transcriere" t_audio; } brick_video(){ brick "VIDEO / BusterX++ (deepfake)"; dc up -d vllm-buster video-api wait_health "http://$H:54500/health" 300 "vLLM BusterX" || die "vLLM BusterX nu pornește" wait_health "http://$H:54600/health" 60 "video-api" || die "video-api nu pornește"; gate "verdict deepfake" t_video; } brick_forensic(){ brick "FORENSIC (rPPG/lip-sync/forgery/lighting)"; dc up -d forensic wait_health "http://$H:8085/health" 120 "forensic" || die "forensic nu pornește"; gate "module forensice" t_forensic; } brick_web(){ brick "WEB + SearXNG (căutare)" # curăță o instanță searxng pornită manual (docker run) ca s-o preia compose if docker inspect didiAI-web-searxng >/dev/null 2>&1 && \ [ -z "$(docker inspect didiAI-web-searxng --format '{{ index .Config.Labels "com.docker.compose.project" }}' 2>/dev/null)" ]; then docker rm -f didiAI-web-searxng >/dev/null 2>&1 || true fi dc up -d searxng web-api wait_health "http://$H:51100/health" 120 "web-api" || die "web-api nu pornește" sleep 8; gate "căutare web" t_web; } brick_catalog(){ brick "CATALOG (service discovery)"; dc up -d catalog-api sleep 6; } # health-ul agregat depinde de gateway; testat după gateway brick_gateway(){ brick "GATEWAY (nginx + auth)" ( cd "$GW_DIR"; grep -q . .env 2>/dev/null || echo "GATEWAY_API_TOKEN=${GATEWAY_API_TOKEN}" > .env; docker compose up -d ) wait_health "http://$H:11000/health" 60 "gateway" || die "gateway nu pornește" gate "gateway auth+rutare" t_gateway; gate "catalog agregat" t_catalog; } brick_brain(){ brick "BRAIN (RAG + fact-check + cache)" # render env din master, up postgres+atomic, aliniază parola, token, taxonomie, up api+scheduler brain_write_env "$POSTGRES_PASSWORD" "${ATOMIC_TOKEN:-PLACEHOLDER}" ( cd "$BRAIN_DIR"; docker compose -f infra/docker-compose.yml up -d postgres atomic-server ) wait_health "http://$H:8088/health" 120 "atomic-server" || warn "atomic health întârzie" brain_align_pg "$POSTGRES_PASSWORD" local tok; tok=$(brain_ensure_token "$POSTGRES_PASSWORD" "${ATOMIC_TOKEN:-}") [ -n "$tok" ] || die "nu am putut crea tokenul atomic" if [ "$tok" != "${ATOMIC_TOKEN:-}" ]; then sed -i "s#^ATOMIC_TOKEN=.*#ATOMIC_TOKEN=${tok}#" "$ENV_FILE"; ATOMIC_TOKEN="$tok"; ok "token atomic creat & salvat în .env" fi brain_write_env "$POSTGRES_PASSWORD" "$tok" ( cd "$BRAIN_DIR"; docker compose -f infra/docker-compose.yml up -d brain-api scheduler ) wait_health "http://$H:8090/health" 120 "brain-api" || die "brain-api nu pornește" brain_seed_taxonomy ( cd "$BRAIN_DIR"; docker compose -f infra/docker-compose.yml restart brain-api scheduler ) >/dev/null wait_health "http://$H:8090/health" 60 "brain-api (post-seed)" || die "brain-api nu revine" gate "brain stats+search" t_brain; } brick_dashboard(){ brick "DASHBOARD (UI + monitorizare)" ( cd "$DASH_DIR" cat > .env <