Livrare LOT 1 - Didi
This commit is contained in:
commit
5380c3fc63
990 changed files with 133308 additions and 0 deletions
301
ai_platform/modules/didi_brain/scripts/bootstrap_deploy.sh
Normal file
301
ai_platform/modules/didi_brain/scripts/bootstrap_deploy.sh
Normal file
|
|
@ -0,0 +1,301 @@
|
|||
#!/usr/bin/env bash
|
||||
# =============================================================================
|
||||
# DidiBrain — fresh-server bootstrap
|
||||
# =============================================================================
|
||||
#
|
||||
# End-to-end deploy of the DidiBrain stack on a clean Linux server:
|
||||
#
|
||||
# 1. Preflight (docker, compose, python3, curl)
|
||||
# 2. .env validation
|
||||
# 3. docker compose build + up
|
||||
# 4. Wait for all three containers to become healthy
|
||||
# 5. Create a local Python venv for operator scripts
|
||||
# 6. Install operator dependencies (httpx, pydantic, etc.)
|
||||
# 7. Claim the Atomic instance + configure the BGE-M3 provider
|
||||
# 8. Seed the canonical tag taxonomy
|
||||
# 9. (Optional) Import the seed Wikipedia corpus
|
||||
# 10. (Optional) Run the claim extraction batch
|
||||
# 11. Final smoke test against /v1/gather
|
||||
#
|
||||
# Every step is idempotent — you can re-run this script after a crash or
|
||||
# after editing .env, and it will only do what still needs doing. No step
|
||||
# is destructive (no `down -v`, no volume deletions).
|
||||
#
|
||||
# Environment flags you can set before running:
|
||||
#
|
||||
# BRAIN_IMPORT_CORPUS 1 to import the Wikipedia seed (default 1)
|
||||
# BRAIN_RUN_EXTRACTION 1 to run claim extraction after import (default 1)
|
||||
# BRAIN_SKIP_VENV 1 to skip venv creation / reuse existing .venv
|
||||
# BRAIN_SKIP_SANITY 1 to skip the final smoke test (save a couple sec)
|
||||
#
|
||||
# Usage:
|
||||
# cd ~/didibrain
|
||||
# cp .env.example .env # then edit LLM_ROUTER_URL / EMBEDDING_URL / ...
|
||||
# ./scripts/bootstrap_deploy.sh
|
||||
#
|
||||
# =============================================================================
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# ----------------------------------------------------------------- paths
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
|
||||
INFRA_DIR="${PROJECT_ROOT}/infra"
|
||||
COMPOSE_FILE="${INFRA_DIR}/docker-compose.yml"
|
||||
ENV_FILE="${PROJECT_ROOT}/.env"
|
||||
ENV_EXAMPLE="${PROJECT_ROOT}/.env.example"
|
||||
VENV_DIR="${PROJECT_ROOT}/.venv"
|
||||
|
||||
cd "${PROJECT_ROOT}"
|
||||
|
||||
# ----------------------------------------------------------------- colors
|
||||
|
||||
RED=$'\033[31m'
|
||||
GREEN=$'\033[32m'
|
||||
YELLOW=$'\033[33m'
|
||||
BLUE=$'\033[34m'
|
||||
DIM=$'\033[2m'
|
||||
BOLD=$'\033[1m'
|
||||
RESET=$'\033[0m'
|
||||
|
||||
step() { echo "${BLUE}${BOLD}== $* ==${RESET}"; }
|
||||
info() { echo "${DIM} · $*${RESET}"; }
|
||||
ok() { echo "${GREEN} ✓ $*${RESET}"; }
|
||||
warn() { echo "${YELLOW} ! $*${RESET}"; }
|
||||
fail() { echo "${RED}${BOLD} ✗ $*${RESET}" >&2; exit 1; }
|
||||
|
||||
# ------------------------------------------------------------- step 1 preflight
|
||||
|
||||
step "1. Preflight checks"
|
||||
|
||||
require_cmd() {
|
||||
command -v "$1" >/dev/null 2>&1 || fail "required command not found: $1"
|
||||
ok "$1 found"
|
||||
}
|
||||
|
||||
require_cmd docker
|
||||
require_cmd python3
|
||||
require_cmd curl
|
||||
|
||||
# Docker Compose can be `docker compose` (v2) or `docker-compose` (v1). Prefer v2.
|
||||
if docker compose version >/dev/null 2>&1; then
|
||||
DC="docker compose"
|
||||
ok "docker compose (v2) found"
|
||||
elif command -v docker-compose >/dev/null 2>&1; then
|
||||
DC="docker-compose"
|
||||
warn "using legacy docker-compose (v1) — v2 recommended"
|
||||
else
|
||||
fail "neither 'docker compose' (v2) nor 'docker-compose' (v1) found"
|
||||
fi
|
||||
|
||||
# Docker daemon reachable?
|
||||
docker info >/dev/null 2>&1 || fail "docker daemon not reachable — is Docker running and your user in the docker group?"
|
||||
ok "docker daemon reachable"
|
||||
|
||||
# ------------------------------------------------------------- step 2 .env
|
||||
|
||||
step "2. Environment file"
|
||||
|
||||
if [[ ! -f "${ENV_FILE}" ]]; then
|
||||
if [[ -f "${ENV_EXAMPLE}" ]]; then
|
||||
warn ".env missing — copying from .env.example"
|
||||
warn "REVIEW IT AND FILL IN LLM_ROUTER_URL / EMBEDDING_URL / RERANKER_URL"
|
||||
cp "${ENV_EXAMPLE}" "${ENV_FILE}"
|
||||
fail ".env was just created from template. Edit it, then rerun this script."
|
||||
else
|
||||
fail "no .env and no .env.example in ${PROJECT_ROOT}"
|
||||
fi
|
||||
fi
|
||||
ok ".env present at ${ENV_FILE}"
|
||||
|
||||
# Minimal sanity on required variables
|
||||
check_env_var() {
|
||||
local key="$1"
|
||||
if ! grep -E "^${key}=" "${ENV_FILE}" >/dev/null 2>&1; then
|
||||
fail "${key} is missing from .env"
|
||||
fi
|
||||
local value
|
||||
value="$(grep -E "^${key}=" "${ENV_FILE}" | head -1 | cut -d= -f2-)"
|
||||
if [[ -z "${value}" ]]; then
|
||||
warn "${key} is empty in .env (may be filled by bootstrap — continuing)"
|
||||
fi
|
||||
}
|
||||
|
||||
check_env_var LLM_ROUTER_URL
|
||||
check_env_var EMBEDDING_URL
|
||||
check_env_var RERANKER_URL
|
||||
check_env_var POSTGRES_USER
|
||||
check_env_var POSTGRES_PASSWORD
|
||||
ok ".env keys look structurally correct"
|
||||
|
||||
# ------------------------------------------------------ step 3 docker compose
|
||||
|
||||
step "3. Build and start containers"
|
||||
|
||||
info "building brain-api image (cached layers reused where possible)..."
|
||||
${DC} -f "${COMPOSE_FILE}" --env-file "${ENV_FILE}" build brain-api
|
||||
|
||||
info "starting the full stack (postgres, atomic-server, brain-api)..."
|
||||
${DC} -f "${COMPOSE_FILE}" --env-file "${ENV_FILE}" up -d
|
||||
|
||||
ok "containers started"
|
||||
|
||||
# ------------------------------------------------------ step 4 wait for healthy
|
||||
|
||||
step "4. Wait for all three containers to become healthy"
|
||||
|
||||
wait_healthy() {
|
||||
local name="$1"
|
||||
local deadline=$(( $(date +%s) + 180 ))
|
||||
while (( $(date +%s) < deadline )); do
|
||||
local status
|
||||
status=$(docker inspect --format='{{.State.Health.Status}}' "${name}" 2>/dev/null || echo "missing")
|
||||
case "${status}" in
|
||||
healthy)
|
||||
ok "${name} healthy"
|
||||
return 0
|
||||
;;
|
||||
unhealthy)
|
||||
fail "${name} reports unhealthy — check 'docker logs ${name}'"
|
||||
;;
|
||||
starting)
|
||||
info "${name} starting..."
|
||||
;;
|
||||
missing)
|
||||
info "${name} not yet visible to docker..."
|
||||
;;
|
||||
*)
|
||||
info "${name} status: ${status}"
|
||||
;;
|
||||
esac
|
||||
sleep 3
|
||||
done
|
||||
fail "${name} did not reach healthy within 180 seconds"
|
||||
}
|
||||
|
||||
wait_healthy didibrain-postgres
|
||||
wait_healthy didibrain-atomic
|
||||
wait_healthy didibrain-api
|
||||
|
||||
# -------------------------------------------------------------- step 5 venv
|
||||
|
||||
step "5. Python operator venv"
|
||||
|
||||
if [[ "${BRAIN_SKIP_VENV:-0}" == "1" ]]; then
|
||||
warn "BRAIN_SKIP_VENV=1 — skipping venv creation"
|
||||
elif [[ -d "${VENV_DIR}" ]]; then
|
||||
ok "venv already exists at ${VENV_DIR}"
|
||||
else
|
||||
info "creating venv at ${VENV_DIR}..."
|
||||
python3 -m venv "${VENV_DIR}"
|
||||
ok "venv created"
|
||||
fi
|
||||
|
||||
if [[ "${BRAIN_SKIP_VENV:-0}" != "1" ]]; then
|
||||
info "installing operator dependencies..."
|
||||
"${VENV_DIR}/bin/pip" install --quiet --upgrade pip
|
||||
"${VENV_DIR}/bin/pip" install --quiet \
|
||||
"httpx>=0.28,<0.30" \
|
||||
"pydantic>=2.12,<3.0" \
|
||||
"pydantic-settings>=2.13,<3.0" \
|
||||
"structlog>=25.5,<26.0" \
|
||||
"python-dotenv>=1.2,<2.0" \
|
||||
"tenacity>=9.1,<10.0" \
|
||||
"rich>=14.3,<15.0"
|
||||
ok "operator dependencies installed"
|
||||
fi
|
||||
|
||||
PY="${VENV_DIR}/bin/python"
|
||||
export PYTHONIOENCODING=utf-8
|
||||
|
||||
# -------------------------------------------------------------- step 6 bootstrap
|
||||
|
||||
step "6. Atomic bootstrap (claim instance + provider config)"
|
||||
|
||||
info "running scripts/02_bootstrap_atomic.py (idempotent)..."
|
||||
"${PY}" "${PROJECT_ROOT}/scripts/02_bootstrap_atomic.py"
|
||||
ok "atomic bootstrapped"
|
||||
|
||||
# -------------------------------------------------------------- step 7 taxonomy
|
||||
|
||||
step "7. Seed canonical taxonomy"
|
||||
|
||||
info "running scripts/04_seed_taxonomy.py (idempotent)..."
|
||||
"${PY}" "${PROJECT_ROOT}/scripts/04_seed_taxonomy.py"
|
||||
ok "taxonomy seeded"
|
||||
|
||||
# -------------------------------------------------------- step 8 corpus import
|
||||
|
||||
if [[ "${BRAIN_IMPORT_CORPUS:-1}" == "1" ]]; then
|
||||
step "8. Import the Wikipedia seed corpus"
|
||||
info "running scripts/05_import_wikipedia_seed.py..."
|
||||
"${PY}" "${PROJECT_ROOT}/scripts/05_import_wikipedia_seed.py"
|
||||
ok "seed corpus imported"
|
||||
else
|
||||
warn "BRAIN_IMPORT_CORPUS=0 — skipping Wikipedia seed import"
|
||||
fi
|
||||
|
||||
# -------------------------------------------------------- step 9 extraction
|
||||
|
||||
if [[ "${BRAIN_RUN_EXTRACTION:-1}" == "1" && "${BRAIN_IMPORT_CORPUS:-1}" == "1" ]]; then
|
||||
step "9. Run claim extraction (may take ~15-20 min for the seed)"
|
||||
info "running scripts/07_run_extraction.py..."
|
||||
"${PY}" "${PROJECT_ROOT}/scripts/07_run_extraction.py"
|
||||
ok "claim extraction complete"
|
||||
else
|
||||
warn "skipping claim extraction"
|
||||
fi
|
||||
|
||||
# ---------------------------------------------------------- step 10 smoke test
|
||||
|
||||
if [[ "${BRAIN_SKIP_SANITY:-0}" == "1" ]]; then
|
||||
warn "BRAIN_SKIP_SANITY=1 — skipping final smoke test"
|
||||
else
|
||||
step "10. Final smoke test — /v1/gather against brain_api"
|
||||
|
||||
HEALTH_JSON="$(curl -fsS http://localhost:8090/health)"
|
||||
info "brain_api /health → ${HEALTH_JSON}"
|
||||
|
||||
GATHER_JSON="$(curl -fsS -X POST http://localhost:8090/v1/gather \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d '{"claim":"vaccines cause autism","max_evidence":3,"include_full_text":false,"run_nli":false}')"
|
||||
|
||||
CACHE_STATUS=$(echo "${GATHER_JSON}" | "${PY}" -c \
|
||||
"import sys,json; print(json.load(sys.stdin).get('brain_meta',{}).get('cache_status','?'))")
|
||||
ITEM_COUNT=$(echo "${GATHER_JSON}" | "${PY}" -c \
|
||||
"import sys,json; print(json.load(sys.stdin).get('total_evidence_items',0))")
|
||||
|
||||
info "gather: cache_status=${CACHE_STATUS} evidence_items=${ITEM_COUNT}"
|
||||
|
||||
if [[ "${CACHE_STATUS}" == "HIT" ]]; then
|
||||
ok "brain returned HIT with ${ITEM_COUNT} items — end-to-end working"
|
||||
elif [[ "${CACHE_STATUS}" == "PARTIAL" ]]; then
|
||||
ok "brain returned PARTIAL (${ITEM_COUNT} items) — end-to-end working, partial coverage"
|
||||
elif [[ "${CACHE_STATUS}" == "MISS" ]]; then
|
||||
warn "brain returned MISS — this is expected if the corpus was not imported"
|
||||
warn "(re-run with BRAIN_IMPORT_CORPUS=1 BRAIN_RUN_EXTRACTION=1 to populate)"
|
||||
else
|
||||
fail "unexpected cache_status: ${CACHE_STATUS}"
|
||||
fi
|
||||
fi
|
||||
|
||||
# ---------------------------------------------------------------- done
|
||||
|
||||
echo
|
||||
step "DONE"
|
||||
echo
|
||||
echo " brain_api: http://localhost:8090"
|
||||
echo " Swagger UI: http://localhost:8090/docs"
|
||||
echo " ReDoc: http://localhost:8090/redoc"
|
||||
echo " OpenAPI spec: http://localhost:8090/openapi.json"
|
||||
echo " atomic-server API: http://localhost:8088"
|
||||
echo " atomic API docs: http://localhost:8088/api/docs"
|
||||
echo " postgres: localhost:5434"
|
||||
echo
|
||||
echo "${DIM}Next steps:${RESET}"
|
||||
echo " · Point Didi backend at http://<this-host>:8090/v1/gather"
|
||||
echo " · Run ${BOLD}${PY} scripts/10_run_lint.py${RESET} overnight for the first contradiction audit"
|
||||
echo " · When corpus needs to grow, extend scripts/05 seed lists or feed"
|
||||
echo " web-module output back via ${BOLD}POST /v1/ingest${RESET}"
|
||||
echo
|
||||
Loading…
Add table
Add a link
Reference in a new issue