didi-lot2-backend/backend/production/build-local.sh
EVOTECH IT SRL 7e4f23d4c4 Optimizare scripturi build/CI si functionalitati noi Lot 2
- CI/CD GitLab (.gitlab-ci.yml): verify -> test -> build -> publish ->
  deploy cu health-gate (/api/v3/health/all) + rollback manual
- Publicare pe retele sociale (LinkedIn/Facebook) din Analysis History
  (social.ts, linkedin.ts, facebook.ts, SocialPostModal)
- Specificatii OpenAPI: agent-v3 (87 op.) si didi-framework (287 op.)
- Matrice testare acceptanta beneficiar + script dovezi API
- Fix nginx SSL: redirect port 3001 (error_page 497, absolute_redirect off)
- Adrese interne inlocuite cu hostname-uri generice (.local)
2026-07-12 13:36:08 -07:00

333 lines
19 KiB
Bash

#!/bin/bash
# =============================================================================
# DIDI Platform — Build From Scratch (topologie LOCALĂ)
# =============================================================================
# Ridică întreaga platformă backend de la zero, cu TOATE serviciile în containere
# locale pe mașina de deployment (fără cluster extern). Topologia serviciilor:
#
# Data layer : didi-postgres (PG17, DB principală LOCALĂ) + didi-cache (Redis)
# + staging-dataLayer-rabbitmq + staging-dataLayer-minio + didi-keycloak
# → docker-compose.local.yml
# Gateway : didi-kong (DBless, imagine didi-kong:latest)
# Orchestration: didi-framework + didi-agent-v3 + 13 workeri
# UI : didi-admin
#
# Seed: DIDI_full_export_2026-07-02.sql (23MB, schema + date + migrațiile 016/017).
# Se importă automat la PRIMUL boot al didi-postgres (volum gol), prin
# montarea în /docker-entrypoint-initdb.d. Redis se reface din PG cu sync-redis.
#
# Cerințe: docker, docker compose v2+. Nicio dependență de rețea externă.
# Rulare: chmod +x build-local.sh && ./build-local.sh [HOSTNAME]
# (implicit = hostname-ul mașinii curente)
#
# NB: documentul-pereche este DEPLOY_FROM_SCRATCH.md. Scriptul vechi full-build.sh
# vizează topologia cluster (PG/Kong externe) și este păstrat ca referință.
# =============================================================================
set -euo pipefail
# Hostname-ul mașinii de deployment. Implicit = hostname-ul mașinii curente
# (agnostic — funcționează pe orice mașină). Se poate suprascrie ca prim argument.
PLATFORM_HOSTNAME="${1:-$(hostname -f 2>/dev/null || hostname)}"
# IP-ul principal LAN al mașinii (pentru dashboard + redirect URIs Keycloak).
# Portabil: se poate suprascrie ca al 2-lea argument.
HOST_IP="${2:-$(hostname -I 2>/dev/null | tr ' ' '\n' | grep -vE '^(127\.|172\.1[6-9]\.|172\.2[0-9]\.|172\.3[0-1]\.|10\.0\.)' | head -1)}"
[ -z "$HOST_IP" ] && HOST_IP="$(hostname -I 2>/dev/null | awk '{print $1}')"
RED='\033[0;31m'; GREEN='\033[0;32m'; YELLOW='\033[1;33m'; CYAN='\033[0;36m'; NC='\033[0m'
# -- Paths (auto-detectate din locația scriptului — portabil pe orice mașină) ---
# Scriptul stă în backend/production/ → BACKEND = părintele lui production/
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
BACKEND="$(cd "$SCRIPT_DIR/.." && pwd)"
DATA_LAYER="$BACKEND/services/data-layer"
DIDI_DB="$DATA_LAYER/didiDatabase"
FRAMEWORK="$BACKEND/services/orchestration-layer/didiFramework"
AGENT_V3="$BACKEND/services/orchestration-layer/agent-v3"
KONG_DIR="$BACKEND/services/gateway-auth-layer/didiKong"
ADMIN_DIR="$BACKEND/admin-dashboard"
# -- PostgreSQL LOCAL (container didi-postgres) ----------------------------
PG_CONTAINER="didi-postgres"
PG_USER="bos_interface"
PG_DB="DIDI"
SEED_FILE="$DIDI_DB/DIDI_full_export_2026-07-02.sql" # sursă de adevăr; NU cel din martie
# -- Keycloak LOCAL --------------------------------------------------------
KC_PORT="28080" # host → 8080 container
KC_BASE="http://localhost:${KC_PORT}/auth" # servit sub /auth (KC_HTTP_RELATIVE_PATH)
KC_ADMIN_USER="admin"
KC_ADMIN_PASS="admin123"
log() { echo -e "${CYAN}[$(date +%H:%M:%S)]${NC} $*"; }
ok() { echo -e "${GREEN} OK${NC} $*"; }
warn() { echo -e "${YELLOW} WARN${NC} $*"; }
fail() { echo -e "${RED} FAIL${NC} $*"; exit 1; }
section() { echo; echo -e "${YELLOW}====================================================================${NC}"; echo -e "${YELLOW} $*${NC}"; echo -e "${YELLOW}====================================================================${NC}"; }
wait_healthy() {
local container="$1" max="${2:-120}" elapsed=0 status
log "Aștept $container să fie healthy (max ${max}s)..."
while [ $elapsed -lt $max ]; do
status=$(docker inspect --format='{{if .State.Health}}{{.State.Health.Status}}{{else}}running{{end}}' "$container" 2>/dev/null || echo "missing")
[ "$status" = "healthy" ] || [ "$status" = "running" ] && { ok "$container ($status)"; return 0; }
sleep 3; elapsed=$((elapsed + 3))
done
warn "$container nu a devenit healthy în ${max}s (status: $status)"; return 1
}
pg() { docker exec "$PG_CONTAINER" psql -U "$PG_USER" -d "$PG_DB" -tAc "$1" 2>/dev/null; }
# =============================================================================
section "FAZA 0: Verificări preliminare"
# =============================================================================
docker info >/dev/null 2>&1 || fail "Docker nu rulează"; ok "Docker activ"
docker compose version >/dev/null 2>&1 || fail "Docker Compose v2 lipsește"; ok "Docker Compose disponibil"
[ -f "$SEED_FILE" ] || fail "Lipsește seed-ul: $SEED_FILE"
ok "Seed găsit: $(basename "$SEED_FILE") ($(du -h "$SEED_FILE" | cut -f1))"
# =============================================================================
section "FAZA 1: Docker network"
# =============================================================================
docker network create didi-network 2>/dev/null && ok "Rețea creată" || ok "Rețeaua didi-network există deja"
# =============================================================================
section "FAZA 2: Data layer local (PG + seed, Redis, RabbitMQ, MinIO, Keycloak)"
# =============================================================================
cd "$DATA_LAYER"
# didi-postgres importă seed-ul automat la primul boot pe volum gol
# (montaj /tmp/didi_full_dump.sql → /docker-entrypoint-initdb.d/01-dump.sql, vezi compose).
# Prima instalare = containerul didi-postgres nu există încă (deci nici volumul lui).
if [ -n "$(docker ps -aq -f name=^${PG_CONTAINER}$)" ]; then
warn "didi-postgres există deja — NU re-importez seed-ul (baza e populată)."
warn "Pentru un import curat: oprește stack-ul + șterge volumul didi-postgres-data, apoi re-rulează."
SEED_ON_INIT=0
else
log "Prima instalare — pregătesc seed-ul pentru auto-import la boot..."
cp "$SEED_FILE" /tmp/didi_full_dump.sql
ok "Seed copiat în /tmp/didi_full_dump.sql (montat ca init script în didi-postgres)"
SEED_ON_INIT=1
fi
log "Pornesc data layer (docker-compose.local.yml)..."
docker compose -f docker-compose.local.yml up -d 2>&1 | tail -6
ok "Data layer pornit"
wait_healthy "$PG_CONTAINER" 90
# Aștept ca importul init (dacă e prima instalare) să termine — poate dura pe seed 23MB
if [ "${SEED_ON_INIT:-0}" = "1" ]; then
log "Aștept finalizarea importului seed (init script rulează la primul boot)..."
for i in $(seq 1 60); do
C=$(pg "SELECT count(*) FROM information_schema.schemata WHERE schema_name IN ('bos_analysis','bos_parammgmt','bos_sysadmin','bos_subscriber')" || echo 0)
[ "$C" = "4" ] && break
sleep 3
done
fi
SCHEMAS=$(pg "SELECT count(*) FROM information_schema.schemata WHERE schema_name IN ('bos_analysis','bos_parammgmt','bos_sysadmin','bos_subscriber')" || echo 0)
[ "$SCHEMAS" = "4" ] || fail "Baza DIDI nu are cele 4 scheme bos_* (găsit: $SCHEMAS). Import eșuat."
ok "Baza DIDI populată: 4 scheme bos_*"
# Sanity pe tabelele-cheie (nu doar tehnici)
log "Verific tabele critice..."
for entry in "bos_parammgmt|technique|166" "bos_parammgmt|dimension|8" "bos_parammgmt|llm_model|" \
"bos_parammgmt|component_stage_assignment|" "bos_parammgmt|input_type_profile|6" \
"bos_parammgmt|source_type|" "bos_parammgmt|verdict_category|" \
"bos_sysadmin|subscription_plan|" "bos_analysis|analysis_session|"; do
IFS='|' read -r sch tbl exp <<< "$entry"
n=$(pg "SELECT count(*) FROM ${sch}.${tbl}" || echo "ERR")
if [ "$n" = "ERR" ]; then warn " LIPSĂ: ${sch}.${tbl}"; else ok " ${sch}.${tbl} = ${n}${exp:+ (aștept ~$exp)}"; fi
done
# Localizare config LLM: seed-ul livrează modelul primar ca `Qwen3.5-397B-A17B` pe
# provider remote `llm.local` — dar vLLM-ul local servește `qwen3.5` prin routerul
# `llm-api:14011`. Aliniem model_code + provider base_url (idempotent, rulează la fiecare build).
log "Localizez config LLM (model → qwen3.5, provideri → llm-api:14011)..."
docker exec "$PG_CONTAINER" psql -U "$PG_USER" -d "$PG_DB" -c "
UPDATE bos_parammgmt.llm_model SET model_code='qwen3.5', model_name='Qwen 3.5 (local)' WHERE model_code='Qwen3.5-397B-A17B';
UPDATE bos_parammgmt.llm_provider SET base_url='http://llm-api:14011/v1' WHERE base_url LIKE 'http://llm.local:1401%';
" >/dev/null 2>&1 && ok "Config LLM localizat (qwen3.5 @ llm-api:14011)" || warn "Localizarea config LLM a eșuat — verifică manual"
wait_healthy didi-cache 30
wait_healthy staging-dataLayer-rabbitmq 90
wait_healthy staging-dataLayer-minio 60
wait_healthy didi-keycloak 180
# --- Localizare Keycloak: frontendUrl + user admin + redirect URIs pentru HOST_IP ---
# Realm-import livrează didi-admins fără user de admin și cu frontendUrl staging.
log "Localizez Keycloak (frontendUrl + user admin + redirect URIs) pentru $HOST_IP..."
KC_API="http://localhost:${KC_PORT}/auth"
for i in $(seq 1 30); do
KTOK=$(curl -s -X POST "$KC_API/realms/master/protocol/openid-connect/token" -d "client_id=admin-cli" -d "username=${KC_ADMIN_USER}" -d "password=${KC_ADMIN_PASS}" -d "grant_type=password" 2>/dev/null | sed -n 's/.*"access_token":"\([^"]*\)".*/\1/p')
[ -n "$KTOK" ] && break; sleep 3
done
if [ -n "$KTOK" ]; then
# frontendUrl → host curent (issuer-ul token-ului trebuie sa coincida cu dashboard-ul)
curl -s -X PUT "$KC_API/admin/realms/didi-admins" -H "Authorization: Bearer $KTOK" -H 'Content-Type: application/json' \
-d "{\"realm\":\"didi-admins\",\"attributes\":{\"frontendUrl\":\"https://${HOST_IP}:3001/auth\"}}" >/dev/null 2>&1
# user admin (idempotent) + parola conforma cu passwordPolicy (length10+upper+digit+lower)
curl -s -X POST "$KC_API/admin/realms/didi-admins/users" -H "Authorization: Bearer $KTOK" -H 'Content-Type: application/json' \
-d '{"username":"admin","email":"admin@didi.local","enabled":true,"emailVerified":true,"firstName":"DIDI","lastName":"Admin","credentials":[{"type":"password","value":"Admin12345","temporary":false}]}' >/dev/null 2>&1
AUID=$(curl -s "$KC_API/admin/realms/didi-admins/users?username=admin" -H "Authorization: Bearer $KTOK" 2>/dev/null | sed -n 's/.*"id":"\([^"]*\)".*/\1/p' | head -1)
# admin + moderator + senior_moderator (write-ul de moderare cere moderator/senior_moderator, NU admin)
AROLES=$(curl -s "$KC_API/admin/realms/didi-admins/roles" -H "Authorization: Bearer $KTOK" 2>/dev/null | python3 -c "import sys,json;d=json.load(sys.stdin);print(json.dumps([{'id':r['id'],'name':r['name']} for r in d if r['name'] in ('admin','moderator','senior_moderator')]))" 2>/dev/null)
[ -n "$AUID" ] && [ -n "$AROLES" ] && curl -s -X POST "$KC_API/admin/realms/didi-admins/users/$AUID/role-mappings/realm" -H "Authorization: Bearer $KTOK" -H 'Content-Type: application/json' -d "$AROLES" >/dev/null 2>&1
# redirect URIs + webOrigin pentru host curent pe clientul admin-dashboard
ACID=$(curl -s "$KC_API/admin/realms/didi-admins/clients?clientId=admin-dashboard" -H "Authorization: Bearer $KTOK" 2>/dev/null | sed -n 's/.*"id":"\([^"]*\)".*/\1/p' | head -1)
if [ -n "$ACID" ]; then
curl -s "$KC_API/admin/realms/didi-admins/clients/$ACID" -H "Authorization: Bearer $KTOK" 2>/dev/null > /tmp/ac.json
python3 - "$HOST_IP" <<'PYKC' 2>/dev/null
import json,sys
ip=sys.argv[1]; c=json.load(open('/tmp/ac.json'))
u=set(c.get('redirectUris',[])); u.update([f'https://{ip}:3001/admin/*',f'https://{ip}:3001/*']); c['redirectUris']=sorted(u)
w=set(c.get('webOrigins',[])); w.add(f'https://{ip}:3001'); c['webOrigins']=sorted(w)
json.dump(c,open('/tmp/ac.json','w'))
PYKC
curl -s -X PUT "$KC_API/admin/realms/didi-admins/clients/$ACID" -H "Authorization: Bearer $KTOK" -H 'Content-Type: application/json' -d @/tmp/ac.json >/dev/null 2>&1
fi
ok "Keycloak localizat — login: admin / Admin12345 (realm didi-admins)"
else
warn "Nu am putut obține token Keycloak — creează userul admin manual"
fi
# =============================================================================
section "FAZA 3: MinIO — bucket-uri"
# =============================================================================
if [ -f "$DATA_LAYER/didiStorage/init-buckets.sh" ]; then
log "Rulez init-buckets.sh în containerul MinIO..."
docker exec -e MINIO_HOST=localhost -e MINIO_PORT=9000 \
-e MINIO_ROOT_USER=minioadmin -e MINIO_ROOT_PASSWORD=minio123 \
staging-dataLayer-minio sh -c "$(cat "$DATA_LAYER/didiStorage/init-buckets.sh")" 2>&1 | tail -5 || warn "init-buckets a raportat erori (verifică manual)"
ok "Bucket-uri inițializate"
else
warn "init-buckets.sh lipsește — creează bucket-urile manual dacă e nevoie"
fi
# =============================================================================
section "FAZA 4: Kong (gateway DBless, imagine didi-kong:latest)"
# =============================================================================
cd "$KONG_DIR"
if ! docker images didi-kong:latest --format '{{.ID}}' | grep -q .; then
log "Build imagine didi-kong:latest..."
docker build -t didi-kong:latest . 2>&1 | tail -3
fi
docker rm -f didi-kong >/dev/null 2>&1 || true
log "Pornesc didi-kong (DBless, config local)..."
docker run -d --name didi-kong --network didi-network --restart unless-stopped \
-p 127.0.0.1:18000:8000 -p 127.0.0.1:18001:8001 -p 127.0.0.1:18443:8443 \
-e KONG_DATABASE=off \
-e KONG_DECLARATIVE_CONFIG=/kong/declarative/kong.yml \
-e "KONG_PROXY_LISTEN=0.0.0.0:8000, 0.0.0.0:8443 ssl" \
-e KONG_ADMIN_LISTEN=0.0.0.0:8001 \
-v "$KONG_DIR/declarative/kong.yml.didi11-local:/kong/declarative/kong.yml:ro" \
didi-kong:latest >/dev/null
wait_healthy didi-kong 60 || warn "Kong nu a raportat healthy — verifică: docker logs didi-kong"
# =============================================================================
section "FAZA 5: didiFramework (CRUD parametri) + sync Redis"
# =============================================================================
cd "$FRAMEWORK"
log "Build + start didi-framework..."
docker compose up -d --build 2>&1 | tail -5
wait_healthy didi-framework 90
log "Sync framework → Redis (regenerează cache-ul din PostgreSQL)..."
SYNC=$(docker exec didi-framework wget -qO- --post-data='' "http://127.0.0.1:3005/api/sync-redis" 2>/dev/null || echo FAIL)
echo "$SYNC" | grep -q '"success"' && ok "Sync Redis reușit" || warn "Sync Redis posibil eșuat: $SYNC"
KF=$(docker exec didi-cache redis-cli -a redis123 --no-auth-warning keys "didi:framework:*" 2>/dev/null | wc -l)
KC=$(docker exec didi-cache redis-cli -a redis123 --no-auth-warning keys "didi:config:*" 2>/dev/null | wc -l)
ok "Redis: $KF chei framework, $KC chei config"
[ "$KF" -ge 5 ] || warn "Prea puține chei framework ($KF) — verifică sync-ul"
# =============================================================================
section "FAZA 6: Agent V3 + workeri"
# =============================================================================
cd "$AGENT_V3"
[ -f .env ] || warn ".env lipsește în agent-v3 — analizele LLM nu vor merge fără OPENROUTER/OPENAI/GROQ keys"
# PUBLIC_API_BASE_URL (media URLs) = host curent, reachable din browser ȘI din workeri
if [ -f .env ]; then
if grep -q '^PUBLIC_API_BASE_URL=' .env; then
sed -i -E "s|^(PUBLIC_API_BASE_URL=).*|\1http://${HOST_IP}:24803|" .env
else
echo "PUBLIC_API_BASE_URL=http://${HOST_IP}:24803" >> .env
fi
ok "agent-v3 PUBLIC_API_BASE_URL → http://${HOST_IP}:24803"
fi
log "Build + start agent-v3 + workeri..."
docker compose up -d --build 2>&1 | tail -10
wait_healthy didi-agent-v3 90
W=$(docker ps --format '{{.Names}}' | grep -c "agent-v3-worker" || true)
A=$(docker ps --format '{{.Names}}' | grep -c "verdict-aggregator" || true)
ok "Workeri: $W | Aggregators: $A"
# =============================================================================
section "FAZA 7: Admin Dashboard"
# =============================================================================
if docker ps --format '{{.Names}}' | grep -q "^didi-admin"; then
ok "Admin dashboard rulează deja"
else
log "Build + start didi-admin..."
cd "$ADMIN_DIR"
# Config React baked în bundle → aliniat la HOST_IP curent (Keycloak + API pe host:3001)
if [ -f .env ]; then
sed -i -E "s|^(REACT_APP_SERVER_HOST=).*|\1${HOST_IP}|; s|^(REACT_APP_HOST=).*|\1${HOST_IP}|; s|^(REACT_APP_KEYCLOAK_URL=).*|\1https://${HOST_IP}:3001/auth|; s|^(REACT_APP_API_BASE_URL=).*|\1https://${HOST_IP}:3001|" .env
ok "admin-dashboard/.env aliniat la ${HOST_IP}"
fi
docker build -t didi-admin:latest . 2>&1 | tail -5
docker rm -f didi-admin-local >/dev/null 2>&1 || true
docker run -d --name didi-admin-local --network didi-network \
-p 3081:80 -p 3001:443 --restart unless-stopped didi-admin:latest >/dev/null
ok "Admin dashboard pornit (3081 HTTP / 3001 HTTPS)"
fi
# =============================================================================
section "FAZA 8: Observabilitate (Prometheus/Grafana/Loki/Jaeger — opțional)"
# =============================================================================
OBS_DIR="$BACKEND/observability"
if [ -f "$OBS_DIR/docker-compose.yml" ]; then
cd "$OBS_DIR"
log "Pornesc stack-ul de observabilitate..."
docker compose up -d 2>&1 | tail -4
ok "Observabilitate pornită — Grafana :3030, Prometheus :9090, Jaeger :16686"
else
warn "observability/ lipsește — sar peste (opțional)"
fi
# =============================================================================
section "VERIFICARE FINALĂ"
# =============================================================================
echo; log "Health checks:"
declare -A CHECKS=(
["PostgreSQL"]="docker exec $PG_CONTAINER pg_isready -U $PG_USER 2>/dev/null"
["Redis"]="docker exec didi-cache redis-cli -a redis123 --no-auth-warning ping 2>/dev/null"
["RabbitMQ"]="docker exec staging-dataLayer-rabbitmq rabbitmq-diagnostics -q ping 2>/dev/null"
["MinIO"]="docker exec staging-dataLayer-minio mc ready local 2>/dev/null"
["Keycloak"]="curl -s -o /dev/null -w %{http_code} ${KC_BASE}/realms/master 2>/dev/null"
["Kong"]="curl -s -o /dev/null -w %{http_code} http://127.0.0.1:18001/status 2>/dev/null"
["Framework"]="docker exec didi-framework wget -qO- http://127.0.0.1:3005/health 2>/dev/null"
["Agent-V3"]="docker exec didi-agent-v3 wget -qO- http://localhost:24803/api/v3/health 2>/dev/null"
)
for name in PostgreSQL Redis RabbitMQ MinIO Keycloak Kong Framework Agent-V3; do
r=$(eval "${CHECKS[$name]}" || echo FAIL)
echo "$r" | grep -qiE "PONG|ok|healthy|ready|service|accepting|200" && ok "$name" || warn "$name: $r"
done
echo
echo -e "${GREEN}====================================================================${NC}"
echo -e "${GREEN} BUILD LOCAL COMPLET — $PLATFORM_HOSTNAME${NC}"
echo -e "${GREEN}====================================================================${NC}"
echo
echo "Endpoint-uri:"
echo " Admin Dashboard: https://localhost:3001/admin (sau prin frontend)"
echo " Agent V3 API: http://localhost:24803/api/v3/health"
echo " Framework API: intern pe didi-network (port 3005)"
echo " Kong Gateway: http://127.0.0.1:18000 (proxy) / :18001 (admin)"
echo " Keycloak: ${KC_BASE} (admin/${KC_ADMIN_PASS}, realms didi-clients + didi-admins)"
echo " RabbitMQ UI: http://localhost:15672 (admin/rabbitmq123)"
echo " MinIO Console: http://localhost:9001 (minioadmin/minio123)"
echo " API docs (Swagger): http://localhost:8089 (dacă didi-api-docs rulează)"
echo
echo "Seed importat automat la primul boot al didi-postgres. Re-sync Redis oricând:"
echo " docker exec didi-framework wget -qO- --post-data='' http://127.0.0.1:3005/api/sync-redis"