livrare lot 2
This commit is contained in:
commit
8ecc78e729
763 changed files with 164593 additions and 0 deletions
68
backend/services/data-layer/didiDatabase/.gitignore
vendored
Normal file
68
backend/services/data-layer/didiDatabase/.gitignore
vendored
Normal file
|
|
@ -0,0 +1,68 @@
|
|||
# Environment files
|
||||
.env
|
||||
.env.local
|
||||
.env.production
|
||||
|
||||
# Data directories
|
||||
data/
|
||||
backups/
|
||||
pg_data/
|
||||
pgdata/
|
||||
|
||||
# Log files
|
||||
*.log
|
||||
logs/
|
||||
log/
|
||||
|
||||
# Certificates and keys
|
||||
certs/
|
||||
*.crt
|
||||
*.key
|
||||
*.pem
|
||||
*.p12
|
||||
*.pfx
|
||||
|
||||
# Backup files
|
||||
*.dump
|
||||
*.sql
|
||||
*.sql.gz
|
||||
*.tar
|
||||
*.tar.gz
|
||||
*.backup
|
||||
|
||||
# pgAdmin data
|
||||
pgadmin_data/
|
||||
.pgadmin/
|
||||
|
||||
# Temporary files
|
||||
*.tmp
|
||||
*.temp
|
||||
*.swp
|
||||
*.swo
|
||||
*~
|
||||
|
||||
# OS files
|
||||
.DS_Store
|
||||
Thumbs.db
|
||||
desktop.ini
|
||||
|
||||
# IDE files
|
||||
.idea/
|
||||
.vscode/
|
||||
*.iml
|
||||
|
||||
# Docker volumes (local bindings)
|
||||
/data
|
||||
/backups
|
||||
/pgadmin_data
|
||||
|
||||
# Test data
|
||||
test_data/
|
||||
*.test.sql
|
||||
|
||||
# Migration tracking
|
||||
.migrations_applied
|
||||
|
||||
# Monitoring data
|
||||
prometheus_data/
|
||||
grafana_data/
|
||||
72
backend/services/data-layer/didiDatabase/Dockerfile
Normal file
72
backend/services/data-layer/didiDatabase/Dockerfile
Normal file
|
|
@ -0,0 +1,72 @@
|
|||
# PostgreSQL 15 Alpine - Rolling tag for security updates
|
||||
FROM postgres:15-alpine
|
||||
|
||||
# Set environment variables
|
||||
ENV POSTGRES_DB=misinformation_db
|
||||
ENV POSTGRES_USER=postgres
|
||||
ENV POSTGRES_PASSWORD=postgres_dev_password_123
|
||||
ENV PGDATA=/var/lib/postgresql/data/pgdata
|
||||
|
||||
# Install additional packages for production use
|
||||
RUN apk add --no-cache \
|
||||
bash \
|
||||
curl \
|
||||
postgresql-client \
|
||||
&& rm -rf /var/cache/apk/*
|
||||
|
||||
# Create necessary directories
|
||||
RUN mkdir -p /docker-entrypoint-initdb.d \
|
||||
&& mkdir -p /var/lib/postgresql/data \
|
||||
&& mkdir -p /scripts \
|
||||
&& mkdir -p /backups
|
||||
|
||||
# Copy initialization script
|
||||
COPY init.sql /docker-entrypoint-initdb.d/01-init.sql
|
||||
|
||||
# Copy health check script
|
||||
COPY health-check.sh /scripts/health-check.sh
|
||||
RUN chmod +x /scripts/health-check.sh
|
||||
|
||||
# Set proper permissions
|
||||
RUN chown -R postgres:postgres /var/lib/postgresql/data \
|
||||
&& chown -R postgres:postgres /docker-entrypoint-initdb.d \
|
||||
&& chown -R postgres:postgres /scripts \
|
||||
&& chown -R postgres:postgres /backups
|
||||
|
||||
# PostgreSQL configuration for production
|
||||
RUN echo "shared_preload_libraries = 'pg_stat_statements'" >> /usr/local/share/postgresql/postgresql.conf.sample \
|
||||
&& echo "pg_stat_statements.track = all" >> /usr/local/share/postgresql/postgresql.conf.sample \
|
||||
&& echo "log_statement = 'all'" >> /usr/local/share/postgresql/postgresql.conf.sample \
|
||||
&& echo "log_duration = on" >> /usr/local/share/postgresql/postgresql.conf.sample \
|
||||
&& echo "log_min_duration_statement = 100" >> /usr/local/share/postgresql/postgresql.conf.sample \
|
||||
&& echo "shared_buffers = 256MB" >> /usr/local/share/postgresql/postgresql.conf.sample \
|
||||
&& echo "effective_cache_size = 1GB" >> /usr/local/share/postgresql/postgresql.conf.sample \
|
||||
&& echo "maintenance_work_mem = 64MB" >> /usr/local/share/postgresql/postgresql.conf.sample \
|
||||
&& echo "checkpoint_completion_target = 0.9" >> /usr/local/share/postgresql/postgresql.conf.sample \
|
||||
&& echo "wal_buffers = 16MB" >> /usr/local/share/postgresql/postgresql.conf.sample \
|
||||
&& echo "default_statistics_target = 100" >> /usr/local/share/postgresql/postgresql.conf.sample \
|
||||
&& echo "random_page_cost = 1.1" >> /usr/local/share/postgresql/postgresql.conf.sample \
|
||||
&& echo "effective_io_concurrency = 200" >> /usr/local/share/postgresql/postgresql.conf.sample \
|
||||
&& echo "work_mem = 4MB" >> /usr/local/share/postgresql/postgresql.conf.sample \
|
||||
&& echo "min_wal_size = 1GB" >> /usr/local/share/postgresql/postgresql.conf.sample \
|
||||
&& echo "max_wal_size = 4GB" >> /usr/local/share/postgresql/postgresql.conf.sample \
|
||||
&& echo "max_worker_processes = 8" >> /usr/local/share/postgresql/postgresql.conf.sample \
|
||||
&& echo "max_parallel_workers_per_gather = 4" >> /usr/local/share/postgresql/postgresql.conf.sample \
|
||||
&& echo "max_parallel_workers = 8" >> /usr/local/share/postgresql/postgresql.conf.sample \
|
||||
&& echo "max_parallel_maintenance_workers = 4" >> /usr/local/share/postgresql/postgresql.conf.sample
|
||||
|
||||
# Expose PostgreSQL port
|
||||
EXPOSE 5432
|
||||
|
||||
# Health check
|
||||
HEALTHCHECK --interval=30s --timeout=10s --start-period=30s --retries=3 \
|
||||
CMD /scripts/health-check.sh || exit 1
|
||||
|
||||
# Use the postgres user
|
||||
USER postgres
|
||||
|
||||
# Volume for data persistence
|
||||
VOLUME ["/var/lib/postgresql/data", "/backups"]
|
||||
|
||||
# Start PostgreSQL
|
||||
CMD ["postgres"]
|
||||
536
backend/services/data-layer/didiDatabase/INDEX.md
Normal file
536
backend/services/data-layer/didiDatabase/INDEX.md
Normal file
|
|
@ -0,0 +1,536 @@
|
|||
# didiDatabase - Index
|
||||
|
||||
Documentatie completa pentru baza de date PostgreSQL a platformei DIDI. Baza principala DIDI ruleaza pe un **container LOCAL** (`didi-postgres`, PostgreSQL 17) pe masina de deployment, in reteaua Docker `didi-network`. Clusterul extern Patroni/HAProxy ramane configurat ca fallback HA, dar NU este folosit operational acum.
|
||||
|
||||
---
|
||||
|
||||
## PostgreSQL LOCAL (PRODUCTIE — activ)
|
||||
|
||||
| Parametru | Valoare |
|
||||
|-----------|---------|
|
||||
| Container | `didi-postgres` |
|
||||
| Imagine | `postgres:17-alpine` |
|
||||
| Host intern | `didi-postgres:5432` (Docker DNS pe `didi-network`) |
|
||||
| Port host | `5432` expus pe `0.0.0.0:5432->5432` |
|
||||
| Database | `DIDI` |
|
||||
| User principal | `bos_interface` / `interface` |
|
||||
|
||||
### Baze de date pe instanta
|
||||
|
||||
Instanta `didi-postgres` contine o singura baza de business, `DIDI` (4 scheme + public, ~2012 sesiuni de analiza la data documentatiei).
|
||||
|
||||
| Baza / consumator | User | Folosita de | Note |
|
||||
|-------------------|------|-------------|------|
|
||||
| DIDI | bos_interface | agent-v3, didiFramework | schemele `bos_*` |
|
||||
| DIDI (schema `public`) | bos_interface | Keycloak IAM | `KC_DB_URL=jdbc:postgresql://didi-postgres:5432/DIDI?currentSchema=public` |
|
||||
| — | — | Kong API Gateway | Kong ruleaza **DBless** (config declarativ), fara baza proprie |
|
||||
|
||||
### Cine se conecteaza
|
||||
|
||||
| Serviciu | Host | Port | Database | User | Fisier config |
|
||||
|----------|------|------|----------|------|---------------|
|
||||
| agent-v3 | didi-postgres | 5432 | DIDI | bos_interface | agent-v3/src/shared/persistence/pg-pool.ts |
|
||||
| didiFramework | didi-postgres | 5432 | DIDI | bos_interface | didiFramework/src/config/database.ts |
|
||||
| Keycloak | didi-postgres | 5432 | DIDI (schema public) | bos_interface | production/.env (`KC_DB_URL`) |
|
||||
|
||||
Containerul local `didi-postgres` este unicul PostgreSQL de productie activ. Fostul container `staging-dataLayer-postgres` NU mai exista. Toate schemele `bos_*` + `public` sunt pe `didi-postgres`.
|
||||
|
||||
agent-v3 acceseaza baza prin `shared/persistence/pg-pool.ts` (`didi-postgres:5432`, DB `DIDI`, user `bos_interface`). Dupa migration 011 scrie si in `bos_analysis.moderation_queue` (prin `moderation/queue-manager.ts`) si citeste coloanele HIL noi de pe `analysis_session`.
|
||||
|
||||
didiFramework scrie in `bos_parammgmt.moderation_config`, `sensitive_topic`, `moderation_role` (introduse de migration 011).
|
||||
|
||||
---
|
||||
|
||||
## Baza de date DIDI -- Schema completa
|
||||
|
||||
4 scheme + public, ~50 tabele total.
|
||||
|
||||
---
|
||||
|
||||
### Schema: bos_analysis (7 tabele + 1 view)
|
||||
|
||||
Scrisa de agent-v3 (pg-adapter.ts, moderation/queue-manager.ts). Citita si de didiFramework (history.ts, sync-analysis.ts).
|
||||
|
||||
#### analysis_session
|
||||
|
||||
Tabelul central -- o inregistrare per analiza.
|
||||
|
||||
| Coloana | Tip | Scop |
|
||||
|---------|-----|------|
|
||||
| session_id | TEXT PK | UUID sesiune |
|
||||
| user_id | TEXT | ID utilizator |
|
||||
| user_email | TEXT | Email utilizator |
|
||||
| input_type | TEXT | text, url, image, audio, video |
|
||||
| input_text | TEXT | Text de analizat |
|
||||
| input_url | TEXT | URL analizat |
|
||||
| input_media_url | TEXT | URL media MinIO |
|
||||
| input_hash | TEXT | Hash input (deduplicare) |
|
||||
| status | TEXT | running, completed, failed |
|
||||
| components_run | TEXT[] | Componente rulate |
|
||||
| components_skipped | TEXT[] | Componente sarite |
|
||||
| risk_score | NUMERIC | Scor risc final (0-100) |
|
||||
| risk_category | TEXT | Categorie risc |
|
||||
| risk_level | TEXT | Nivel risc |
|
||||
| confidence | NUMERIC | Incredere (0-100) |
|
||||
| confidence_level | TEXT | Nivel incredere |
|
||||
| started_at | TIMESTAMP | Start procesare |
|
||||
| completed_at | TIMESTAMP | Sfarsit procesare |
|
||||
| total_duration_ms | INTEGER | Durata totala ms |
|
||||
| scenario_applied | TEXT | Scenariu ponderi aplicat |
|
||||
| topic_applied | TEXT | Topic detectat |
|
||||
| source_app | TEXT | web (default) |
|
||||
| api_version | TEXT | v3 (default) |
|
||||
| created_at | TIMESTAMP | Data creare |
|
||||
|
||||
Coloane HIL adaugate prin migration 011 (2026-05-01):
|
||||
|
||||
| Coloana | Tip | Scop |
|
||||
|---------|-----|------|
|
||||
| review_status | TEXT default 'none' (CHECK: none\|pending\|in_review\|resolved\|declined) | HIL state |
|
||||
| human_corrected | BOOLEAN default false | true daca moderator a corectat |
|
||||
| human_corrections | JSONB NULL | Diff-style corrections {verdict?, techniques?, ai_tampered?, claims?} |
|
||||
| verified_by | TEXT NULL | keycloak_id moderator |
|
||||
| verified_at | TIMESTAMPTZ NULL | When resolved |
|
||||
| review_notes | TEXT NULL | Optional moderator notes |
|
||||
|
||||
Index partial: `idx_analysis_session_review_status WHERE review_status != 'none'` -- majoritatea sesiunilor raman 'none', sunt sarite la scan.
|
||||
|
||||
#### analysis_techniques
|
||||
|
||||
O inregistrare per sesiune -- rezultat componenta tehnici de manipulare.
|
||||
|
||||
| Coloana | Tip | Scop |
|
||||
|---------|-----|------|
|
||||
| session_id | TEXT FK | Referinta sesiune |
|
||||
| manipulation_score | NUMERIC | Scor manipulare (0-100) |
|
||||
| total_severity | NUMERIC | Severitate totala |
|
||||
| dimensions_affected | TEXT[] | Dimensiuni afectate |
|
||||
| techniques_count | INTEGER | Numar tehnici detectate |
|
||||
| techniques_detected | JSONB | Lista tehnici cu detalii |
|
||||
| coupling_context | JSONB | Context cuplare inter-tehnici |
|
||||
| llm_screening | TEXT | Model LLM screening |
|
||||
| llm_deep | TEXT | Model LLM deep analysis |
|
||||
| screening_duration_ms | INTEGER | Durata screening |
|
||||
| deep_analysis_duration_ms | INTEGER | Durata analiza profunda |
|
||||
| total_duration_ms | INTEGER | Durata totala |
|
||||
| fallbacks_screening | INTEGER | Fallback-uri screening |
|
||||
| fallbacks_deep | INTEGER | Fallback-uri deep |
|
||||
|
||||
#### analysis_ai_tampered
|
||||
|
||||
O inregistrare per sesiune -- detectie continut AI/manipulat.
|
||||
|
||||
| Coloana | Tip | Scop |
|
||||
|---------|-----|------|
|
||||
| session_id | TEXT FK | Referinta sesiune |
|
||||
| ai_probability | NUMERIC | Probabilitate AI (0-100) |
|
||||
| verdict | TEXT | Verdict AI detection |
|
||||
| risk_score | NUMERIC | Scor risc AI |
|
||||
| categories_affected | TEXT[] | Categorii afectate |
|
||||
| indicators_count | INTEGER | Numar indicatori |
|
||||
| disclosure_detected | BOOLEAN | Disclosure detectat |
|
||||
| disclosure_explicit | BOOLEAN | Disclosure explicit |
|
||||
| disclosure_text | TEXT | Text disclosure |
|
||||
| indicators_detected | JSONB | Lista indicatori |
|
||||
| coupling_context | JSONB | Context cuplare |
|
||||
| llm_screening | TEXT | Model screening |
|
||||
| llm_deep | TEXT | Model deep |
|
||||
| screening_duration_ms | INTEGER | Durata screening |
|
||||
| deep_analysis_duration_ms | INTEGER | Durata deep |
|
||||
| total_duration_ms | INTEGER | Durata totala |
|
||||
| fallbacks_screening | INTEGER | Fallback-uri screening |
|
||||
| fallbacks_deep | INTEGER | Fallback-uri deep |
|
||||
| content_type | TEXT | text, image, audio, video |
|
||||
| image_analysis | JSONB | Rezultat analiza imagine |
|
||||
|
||||
#### analysis_claims
|
||||
|
||||
O inregistrare per sesiune -- verificare afirmatii.
|
||||
|
||||
| Coloana | Tip | Scop |
|
||||
|---------|-----|------|
|
||||
| session_id | TEXT FK | Referinta sesiune |
|
||||
| total_claims | INTEGER | Total afirmatii |
|
||||
| verified_true | INTEGER | Verificate adevarate |
|
||||
| verified_false | INTEGER | Verificate false |
|
||||
| unverified | INTEGER | Neverificate |
|
||||
| opinions | INTEGER | Opinii |
|
||||
| credibility_score | NUMERIC | Scor credibilitate |
|
||||
| interpretation | TEXT | Interpretare |
|
||||
| claims_by_status | JSONB | Claims grupate pe status |
|
||||
| claims_by_type | JSONB | Claims grupate pe tip |
|
||||
| claims_verified | JSONB | Detalii verificare |
|
||||
| llm_extraction | TEXT | Model extragere |
|
||||
| llm_verification | TEXT | Model verificare |
|
||||
| extraction_duration_ms | INTEGER | Durata extragere |
|
||||
| verification_duration_ms | INTEGER | Durata verificare |
|
||||
| total_duration_ms | INTEGER | Durata totala |
|
||||
| web_searches_made | INTEGER | Cautari web efectuate |
|
||||
|
||||
#### analysis_domain
|
||||
|
||||
O inregistrare per sesiune -- analiza domeniu/sursa.
|
||||
|
||||
| Coloana | Tip | Scop |
|
||||
|---------|-----|------|
|
||||
| session_id | TEXT FK | Referinta sesiune |
|
||||
| domain | TEXT | Domeniu analizat |
|
||||
| verdict | TEXT | Verdict domeniu |
|
||||
| trust_score | NUMERIC | Scor incredere |
|
||||
| risk_level | TEXT | Nivel risc |
|
||||
| age_days | INTEGER | Varsta domeniu (zile) |
|
||||
| age_category | TEXT | Categorie varsta |
|
||||
| domain_created_at | TIMESTAMP | Data creare domeniu |
|
||||
| is_blacklisted | BOOLEAN | Pe lista neagra |
|
||||
| reputation_score | NUMERIC | Scor reputatie |
|
||||
| has_ssl | BOOLEAN | Are SSL |
|
||||
| ssl_valid | BOOLEAN | SSL valid |
|
||||
| ssl_issuer | TEXT | Emitent SSL |
|
||||
| registrar | TEXT | Registrar domeniu |
|
||||
| organization | TEXT | Organizatie |
|
||||
| country | TEXT | Tara |
|
||||
| red_flags | TEXT[] | Semnale alarma |
|
||||
| warnings | TEXT[] | Avertismente |
|
||||
| duration_ms | INTEGER | Durata analiza |
|
||||
|
||||
#### analysis_verdict
|
||||
|
||||
O inregistrare per sesiune -- verdictul final agregat.
|
||||
|
||||
| Coloana | Tip | Scop |
|
||||
|---------|-----|------|
|
||||
| session_id | TEXT FK | Referinta sesiune |
|
||||
| risk_score | NUMERIC | Scor risc final |
|
||||
| risk_category | TEXT | Categorie risc |
|
||||
| risk_category_color | TEXT | Culoare categorie |
|
||||
| risk_level | TEXT | Nivel risc |
|
||||
| risk_level_color | TEXT | Culoare nivel |
|
||||
| severity | TEXT | Severitate |
|
||||
| recommended_action | TEXT | Actiune recomandata |
|
||||
| confidence | NUMERIC | Incredere |
|
||||
| confidence_level | TEXT | Nivel incredere |
|
||||
| score_manipulation | NUMERIC | Scor componenta manipulare |
|
||||
| score_claims | NUMERIC | Scor componenta claims |
|
||||
| score_ai | NUMERIC | Scor componenta AI |
|
||||
| score_source | NUMERIC | Scor componenta sursa |
|
||||
| score_context | NUMERIC | Scor context |
|
||||
| applied_weights | JSONB | Ponderi aplicate |
|
||||
| override_applied | BOOLEAN | Override aplicat |
|
||||
| override_type | TEXT | Tip override |
|
||||
| override_reason | TEXT | Motiv override |
|
||||
| override_adjustment | NUMERIC | Ajustare override |
|
||||
| context_summary | JSONB | Sumar context |
|
||||
| components_used | TEXT[] | Componente folosite |
|
||||
| weights_source | TEXT | Sursa ponderi |
|
||||
| duration_ms | INTEGER | Durata calcul |
|
||||
| explanation_ro | TEXT | Explicatie romana (migration 001) |
|
||||
| explanation_en | TEXT | Explicatie engleza (migration 001) |
|
||||
| virality_score | NUMERIC | Scor viralitate (0-100) |
|
||||
| virality_level | TEXT | Nivel viralitate |
|
||||
| virality_factors | JSONB | Factori viralitate |
|
||||
|
||||
#### moderation_queue (adaugat prin migration 011)
|
||||
|
||||
Stare workflow HIL (Human-in-the-Loop). Un rand per sesiune marcata de triage pentru review uman.
|
||||
|
||||
| Coloana | Tip | Scop |
|
||||
|---------|-----|------|
|
||||
| queue_id | BIGSERIAL PK | Auto-increment |
|
||||
| session_id | UUID FK -> analysis_session(session_id) ON DELETE CASCADE | Referinta sesiune |
|
||||
| priority | INTEGER (1-5) | 1=highest (user_flagged), 3=low_confidence, 4=sensitive_topic |
|
||||
| enqueue_reason | TEXT | flagged \| low_confidence \| sensitive_topic \| mixed |
|
||||
| enqueue_meta | JSONB | Triage metadata (risk_score, confidence, topic detected) |
|
||||
| status | TEXT | pending \| in_review \| resolved \| declined \| auto_closed |
|
||||
| assigned_to | TEXT | keycloak_id moderator |
|
||||
| assigned_at | TIMESTAMPTZ | When claimed |
|
||||
| resolved_at | TIMESTAMPTZ | When closed |
|
||||
| resolved_by | TEXT | keycloak_id |
|
||||
| resolution_action | TEXT | approved \| corrected \| rejected |
|
||||
| time_in_queue_ms | INTEGER | enqueue -> start review |
|
||||
| time_in_review_ms | INTEGER | start review -> resolved |
|
||||
| created_at | TIMESTAMPTZ | Default now() |
|
||||
|
||||
Indecsi: `idx_moderation_queue_status_priority` (partial WHERE status IN ('pending','in_review')), `idx_moderation_queue_session`, `idx_moderation_queue_assigned`.
|
||||
|
||||
#### v_analysis_full (VIEW)
|
||||
|
||||
JOIN pe toate 6 tabelele de analiza (session + techniques + ai_tampered + claims + domain + verdict). Definit in migration 001. Selecteaza doar coloane sumar (nu JSONB-uri grele): session metadata, verdict scores, techniques summary, ai probability, claims summary, domain summary + explanation_ro/en.
|
||||
|
||||
Migration 011 NU modifica view-ul: coloanele HIL noi de pe `analysis_session` (review_status, human_corrected etc.) sunt acoperite automat de `SELECT s.*`.
|
||||
|
||||
---
|
||||
|
||||
### Schema: bos_parammgmt (~40 tabele)
|
||||
|
||||
Scrisa si citita exclusiv de didiFramework. Contine toti parametrii de configurare ai platformei. Sincronizata in Redis prin POST /api/sync-redis.
|
||||
|
||||
Search path setat in database.ts: `SET search_path TO bos_parammgmt, public`.
|
||||
|
||||
#### Tabel de baza
|
||||
|
||||
| Tabel | Scop | Rute CRUD |
|
||||
|-------|------|-----------|
|
||||
| parameter | Tabel parinte versionare (parameter_id, parameter_type, valid_from/to) | intern (FK din toate celelalte) |
|
||||
|
||||
#### Tehnici de manipulare (ierarhie 4 nivele)
|
||||
|
||||
| Tabel | Scop | Rute CRUD |
|
||||
|-------|------|-----------|
|
||||
| dimension | Dimensiuni top-level (code, name, weight) | /api/dimensions |
|
||||
| subdimension | Sub-dimensiuni (FK dimension) | /api/subdimensions |
|
||||
| technique | Tehnici individuale (FK subdimension, severity, confidence, detectability) | /api/techniques |
|
||||
| technique_indicator | Indicatori detectie per tehnica (name, description, max_intensity 1-3) | /api/indicators |
|
||||
| technique_validation_rule | Reguli validare per tehnica | /api/validation-rules |
|
||||
|
||||
#### Evaluare sursa
|
||||
|
||||
| Tabel | Scop | Rute CRUD |
|
||||
|-------|------|-----------|
|
||||
| platform | Platforme social media (code, name, score) | /api/platforms |
|
||||
| platform_modifier | Modificatori platforma (condition, score) | /api/platform-modifiers |
|
||||
| source_credibility | Factori credibilitate sursa | /api/source-credibility |
|
||||
| source_type | Tipuri sursa (base_score) | intern |
|
||||
| source_assessment | Evaluare sursa | intern |
|
||||
| domain_age_score | Scor varsta domeniu (range-uri, impact) | /api/domain-age-scores |
|
||||
| domain_risk_level | Nivele risc domeniu (range-uri, interpretare) | /api/domain-risk-levels |
|
||||
| domain_red_flag | Red flags domeniu (condition, severity, action) | /api/domain-red-flags |
|
||||
| author_classification | Clasificari autor (code, name, score) | /api/author-classifications |
|
||||
| author_credibility | Credibilitate autor (impact) | /api/author-credibility |
|
||||
|
||||
#### Claims
|
||||
|
||||
| Tabel | Scop | Rute CRUD |
|
||||
|-------|------|-----------|
|
||||
| claim | Statusuri claim (TRUE, FALSE, UNVERIFIED, OPINION) | /api/claims/status |
|
||||
| claim_type | Tipuri claim (factual, statistic, cauzal, etc.) | /api/claims/types |
|
||||
| confidence | Nivele incredere (level, color, action, range) | /api/claims/confidence |
|
||||
| interpretation | Interpretare scor credibilitate (range-uri) | /api/claims/interpretation |
|
||||
|
||||
#### Verdicte si scoruri
|
||||
|
||||
| Tabel | Scop | Rute CRUD |
|
||||
|-------|------|-----------|
|
||||
| verdict_category | Categorii verdict (code, range, color) | /api/verdicts/categories |
|
||||
| risk_mapping | Mapping risc (level, range, color) | /api/verdicts/risk |
|
||||
| severity_assessment | Evaluare severitate (category, range, action) | /api/verdicts/severity |
|
||||
|
||||
#### Ponderi
|
||||
|
||||
| Tabel | Scop | Rute CRUD |
|
||||
|-------|------|-----------|
|
||||
| component_weight | Ponderi componente (manipulation, claims, source, ai, context) | /api/weights/components |
|
||||
| weight_scenario | Scenarii ponderi (per topic: health, politics, etc.) | /api/weights/scenarios |
|
||||
| multiplier | Multiplicatori (topic, temporal, reach) | /api/weights/multipliers |
|
||||
|
||||
#### Provideri LLM
|
||||
|
||||
| Tabel | Scop | Rute CRUD |
|
||||
|-------|------|-----------|
|
||||
| llm_provider | Configurare provideri (base_url, auth_type, rate_limit) | /api/providers/configs |
|
||||
| llm_model | Modele LLM (context_window, cost, capabilities) | /api/providers/models |
|
||||
| component_provider_assignment | Assignment componenta -> model (legacy, pre-migration-002) | /api/providers/assignments |
|
||||
| provider_api_key | Chei API per provider (criptate, usage tracking) | /api/providers/keys |
|
||||
|
||||
#### Configurare unificata componente (adaugat prin migration 002, extinsa cu tier prin 006)
|
||||
|
||||
| Tabel | Scop | Rute CRUD |
|
||||
|-------|------|-----------|
|
||||
| component_stage_assignment | Assignment model pe etapa + **tier** (free/premium) cu fallback chain. Unique: `(component_code, stage_code, tier, fallback_order)`. | /api/providers/assignments (suporta `?tier=X` filter) |
|
||||
| component_prompt | Prompturi LLM per componenta/etapa (system_prompt, user_template) | /api/providers/prompts |
|
||||
| component_config | Config JSONB catch-all per componenta (scoring, patterns, vision models) | intern (sync-redis) |
|
||||
|
||||
**Component codes prezente dupa migrations 006-009**:
|
||||
- `techniques` (stages: techniques_screening, techniques_deep)
|
||||
- `ai-tampered` (stages: ai_tampered_screening, ai_tampered_deep)
|
||||
- `claims` (stages: claims_extraction, claims_verification)
|
||||
- `source-assessment` (stages: source_assessment_extraction, source_assessment_evaluation)
|
||||
- `vision` (stage: image_analysis — OCR + AI detection + video frames, Etapa 4)
|
||||
- `verdict` (stage: verdict_review — LLM verdict reviewer care ajusteaza scorul final + explicatii RO/EN, Etapa 5)
|
||||
|
||||
Fiecare componenta/stage are **2 tiers** (`free` + `premium`), fiecare cu propriul fallback chain (primary + 2-3 fallbacks). Ex: `techniques_screening` are 4 randuri `tier='free'` + 4 randuri `tier='premium'`.
|
||||
|
||||
Tier-ul final folosit la runtime se deriveaza din `planType` al userului (returnat de check-credits):
|
||||
- `plan_type` 1-3 (Freemium/Starter/Basic) → `tier='free'`
|
||||
- `plan_type` 4-6 (Pro/Business/Enterprise) → `tier='premium'`
|
||||
|
||||
#### Chei API extensie browser
|
||||
|
||||
| Tabel | Scop | Rute CRUD |
|
||||
|-------|------|-----------|
|
||||
| extension_api_key | Chei API extensie browser (key, user_id, usage_count) | /api/extension-keys |
|
||||
|
||||
#### Profiluri verdict per input type (adaugat 2026-03-21)
|
||||
|
||||
| Tabel | Scop | Rute CRUD |
|
||||
|-------|------|-----------|
|
||||
| input_type_profile | 6 profiluri verdict (text, image, audio, video, url) cu ponderi per componenta, reguli INCONCLUSIVE, disclosure multipliers | /api/input-profiles |
|
||||
| profile_override_config | Override-uri per profil (8 tipuri × 6 profiluri = 48 randuri) | /api/input-profiles/:code/overrides |
|
||||
|
||||
Coloane noi in tabele existente:
|
||||
- `claim_type.unverified_weight` NUMERIC(3,2) — ponderea UV per tip claim (0.25-0.50)
|
||||
- `claim.credibility_weight` NUMERIC(3,2) — ponderea credibilitate per status claim (0.00-1.00)
|
||||
|
||||
#### HIL Moderation config (adaugat prin migration 011, 2026-05-01)
|
||||
|
||||
| Tabel | Scop | Rute CRUD |
|
||||
|-------|------|-----------|
|
||||
| moderation_config | Single-row settings (CHECK config_id=1): triage thresholds (confidence_low, risk_grey_min/max, queue_relax_at, queue_strict_at) + brain client config (brain_enabled, brain_url, lookup/write timeouts, brain_confidence_min_silver, brain_semantic_threshold, brain_per_component JSONB). 14 fields total. Sincronizat in Redis ca `didi:config:moderation:v1:settings`. | /api/moderation-config |
|
||||
| sensitive_topic | Topics care declanseaza HIL review (seed: elections, health, war, covid, climate). topic_code regex `[a-z0-9_]+` UNIQUE; soft delete via is_active. Sincronizat in Redis ca `didi:config:moderation:v1:sensitive_topics`. | /api/sensitive-topics |
|
||||
| moderation_role | Mapping Keycloak role -> HIL permissions (seed: moderator, senior_moderator). Toggles: can_resolve, can_escalate, can_force_gold_brain, is_active. role_code este PK (immutable). Sincronizat in Redis ca `didi:config:moderation:v1:roles`. | /api/moderation-roles |
|
||||
|
||||
---
|
||||
|
||||
### Schema: bos_sysadmin (5 tabele)
|
||||
|
||||
Scrisa si citita de didiFramework (auth.ts, admin.ts, subscriptions.ts). Management utilizatori si abonamente.
|
||||
|
||||
| Tabel | Scop | Rute |
|
||||
|-------|------|------|
|
||||
| internet_user | Utilizator platforma (internet_user_id, person_id FK, credits_remained, credits_spent) | /api/auth/me (auto-creare), /api/admin/users |
|
||||
| user_credential | Credentiale (email, keycloak_id, enrollment_type, subscription_status) | /api/auth/me, /api/admin/users |
|
||||
| subscription | Abonament activ (internet_user_id FK, plan FK, status, activation_date) | /api/subscriptions |
|
||||
| subscription_plan | Planuri abonament (plan_name, plan_type, price, credits, limite storage/media, costuri per tip) | /api/admin/plans |
|
||||
| ai_credit_usage | Log consum credite (session_id, user_id, credits_used, input_type) | /api/auth/deduct-credits |
|
||||
|
||||
---
|
||||
|
||||
### Schema: bos_subscriber (4 tabele)
|
||||
|
||||
Scrisa de didiFramework la inregistrare utilizator. Date personale.
|
||||
|
||||
| Tabel | Scop |
|
||||
|-------|------|
|
||||
| person | Entitate persoana (person_id, person_type, status) |
|
||||
| address | Adresa (address_id, address_type) |
|
||||
| persoana_fizica | Persoana fizica romaneasca (nume, prenume, FK person, FK address) |
|
||||
| contact | Contact (person_id FK, contact_type_id, contact_info) |
|
||||
|
||||
---
|
||||
|
||||
### Schema: public
|
||||
|
||||
| Tabel | Scop |
|
||||
|-------|------|
|
||||
| waitlist | Lista de asteptare pre-lansare (vezi sectiunea container local) |
|
||||
|
||||
---
|
||||
|
||||
## Migratii aplicate
|
||||
|
||||
| Fisier | Ce face | Aplicata de |
|
||||
|--------|---------|-------------|
|
||||
| didiFramework/sql/migrations/001_add_explanation_columns.sql | Adauga explanation_ro, explanation_en la analysis_verdict + creeaza view v_analysis_full | didiFramework la pornire |
|
||||
| didiFramework/sql/migrations/002_add_component_pilot_config.sql | Adauga tabele component_stage_assignment, component_prompt, component_config | didiFramework la pornire |
|
||||
| didiFramework/sql/migrations/006_add_tier_column.sql | `component_stage_assignment.tier varchar(20) DEFAULT 'free'` + unique constraint pe (component_code, stage_code, tier, fallback_order) | Manual |
|
||||
| didiFramework/sql/migrations/007_seed_premium_assignments.sql | Seed 32 rows `tier='premium'` pentru 8 stages LLM (techniques/ai-tampered/claims/source-assessment) | Manual |
|
||||
| didiFramework/sql/migrations/008_seed_vision_assignments.sql | Seed 7 rows pentru component `vision` stage `image_analysis` (3 free + 4 premium) | Manual |
|
||||
| didiFramework/sql/migrations/009_seed_verdict_assignments.sql | Seed 8 rows pentru component `verdict` stage `verdict_review` (4 free + 4 premium) | Manual |
|
||||
| didiFramework/sql/migrations/011_add_moderation.sql | HIL Moderation foundation: 6 coloane pe `analysis_session`, tabela `moderation_queue`, 3 tabele config in bos_parammgmt (moderation_config, sensitive_topic, moderation_role) + seeds. Companion `011_rollback.sql`. session_id este UUID, FK foloseste UUID. | Manual |
|
||||
|
||||
Migratiile ulterioare (012 topic_volatility, 013 user_audit_log, 014 atomic_path_prefix, 015 social_post, 016 input_profile_versions, 017 model_catalog_attributes) sunt incluse integral in seed-ul canonic `DIDI_full_export_2026-07-02.sql`. Un restore curat al seed-ului produce schema completa la zi (fara a mai rula migratiile manual). Cateva dintre ele sunt descrise mai jos in "Schema additions".
|
||||
|
||||
---
|
||||
|
||||
## Container `staging-dataLayer-postgres` (ISTORIC — inexistent)
|
||||
|
||||
> Nota istorica: un container `staging-dataLayer-postgres` (postgres:15-alpine, database `misinformation_db`) a servit candva doar tabela `public.waitlist` + ~22 tabele legacy goale din vechiul orchestrator Python. **Acest container NU mai exista.** Baza de business (inclusiv `public.waitlist`, daca este folosita) este acum in database-ul `DIDI` de pe containerul `didi-postgres`. Orice referinta la `staging-dataLayer-postgres`, `misinformation_db` sau la path-ul arhiva `didiDatabase-legacy/` este stale si nu mai reflecta realitatea.
|
||||
|
||||
---
|
||||
|
||||
## Diagrama conexiuni
|
||||
|
||||
```
|
||||
+-----------------------------------+
|
||||
| didi-postgres:5432 |
|
||||
| Container LOCAL (postgres:17) |
|
||||
| didi-network |
|
||||
+-----------------------------------+
|
||||
| Database: DIDI |
|
||||
+-----------------------------------+
|
||||
| bos_analysis (agent-v3) |
|
||||
| bos_parammgmt (didiFramework) |
|
||||
| bos_sysadmin (didiFramework) |
|
||||
| bos_subscriber (didiFramework) |
|
||||
| public (Keycloak schema) |
|
||||
+-----------------------------------+
|
||||
^ ^ ^
|
||||
| | |
|
||||
agent-v3 didiFramework Keycloak
|
||||
(schema public)
|
||||
|
||||
Kong ruleaza DBless (fara baza proprie).
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Fisiere in directorul didiDatabase
|
||||
|
||||
```
|
||||
DIDI_full_export_2026-07-02.sql -- SEED CANONIC (23 MB): pg_dump complet DIDI
|
||||
(schema + date + migratiile 001-017). Restorabil
|
||||
cu --clean --if-exists --no-owner.
|
||||
Dockerfile -- Build imagine postgres cu init (pastrat pentru rebuild container)
|
||||
MIGRATION.md -- Note migrare PostgreSQL (atentie: contine si sectiuni stale despre cluster)
|
||||
REBUILD.md -- Reteta rebuild baza pe alt host din seed-ul canonic
|
||||
ha-cluster/ -- Config optional HA (docker-compose + haproxy.cfg) pentru fallback cluster
|
||||
.gitignore -- Exclude .env, data/
|
||||
INDEX.md -- Aceasta documentatie
|
||||
```
|
||||
|
||||
Nota seed: fisierul canonic actual este `DIDI_full_export_2026-07-02.sql`. Seed-ul vechi `DIDI_full_export_2026-03-22.sql` (fara migratiile 016/017) si pachetul demo (`DIDI_demo_seed_2026-07-02.sql` + `demo-seed/`) au fost arhivate **in afara repo-ului** (`/home/admin365/didi_seed_archive_2026-07-08/`) — livrarea foloseste DOAR full seed-ul curent.
|
||||
|
||||
---
|
||||
|
||||
## Ce NU face containerul local `didi-postgres`
|
||||
|
||||
- Nu are replicare (instanta singulara); HA se obtine doar comutand pe fallback-ul cluster din `ha-cluster/`
|
||||
- Nu are backup automat integrat (backup manual din seed / pg_dump)
|
||||
- Nu are SSL/TLS intern
|
||||
- Este sursa unica de adevar pentru datele DIDI; Redis (`didi:config:*`, `didi:framework:*`) e cache derivat, regenerat cu `sync-redis`
|
||||
|
||||
---
|
||||
|
||||
## Schema additions (2026-05-04 → 2026-05-05)
|
||||
|
||||
### `bos_parammgmt.sensitive_topic` — extins (migration 012)
|
||||
|
||||
ALTER ADD: `volatility ('volatile'|'evolving'|'stable')`, `cache_ttl_hours integer (1-26280)`, `recency_window_days integer (1-365)`, `half_life_days numeric (>0)`. Seed: war/elections=volatile@24h/7d/3d, health/covid=evolving@168h/14d/14d, climate=stable@720h/180d/180d, fraud_test=defaults. Used by brain `topic_volatility.py` to override classifier TTL per topic.
|
||||
|
||||
### `bos_sysadmin.user_audit_log` — nou (migration 013)
|
||||
|
||||
```
|
||||
audit_id bigserial PK
|
||||
internet_user_id integer (NULL pentru keycloak-only useri)
|
||||
target_email text
|
||||
target_keycloak_id text
|
||||
actor_keycloak_id text -- extras din JWT (sub claim)
|
||||
actor_email text
|
||||
action text NOT NULL -- user.{update,delete,sync,email_verified,subscription,roles,group,reset_password}
|
||||
payload jsonb DEFAULT '{}' -- diff before/after sau parametri operațiune
|
||||
request_ip text
|
||||
user_agent text
|
||||
created_at timestamptz NOT NULL DEFAULT now()
|
||||
```
|
||||
|
||||
4 indexuri: user (partial), actor, action+time, time. Powers tab "Audit Log" în UserManagement DIDI admin.
|
||||
|
||||
### Brain tables (alongside Atomic, prefix `brain_*`, public schema)
|
||||
|
||||
| Tabela | Scop | Cheie unique |
|
||||
|---|---|---|
|
||||
| `brain_analysis_atom` (existed) | Cache rezultate full-component LLM (techniques/ai_tampered/claims). +7 coloane noi: `volatility`, `topic_codes text[]`, `entity_bindings jsonb`, `ttl_hours_used`, `last_audited_at`, `audit_history jsonb` (last-50 cap), `consecutive_audit_passes` | `(content_hash, component, prompt_hash)` |
|
||||
| `brain_verification_cache` (existed) | Cache verdict LLM per claim. Same +7 coloane | `(claim_hash, tier)` |
|
||||
| `brain_fact_status` (NOU, 2026-05-04) | Current truth pentru triplete `(subject, predicate, object)`. Coloane: `current_truth bool|NULL`, `current_version_id`, `current_confidence`, `last_verified_at`, `last_evidence_urls jsonb`, `volatility`, `topic_codes`, `next_check_at`, `check_interval_hours`, `moderator_locked bool`, `moderator_user_id`, `moderator_notes` | `canonical_form_hash` |
|
||||
| `brain_fact_version` (NOU) | Temporal versioning. `truth_value bool`, `confidence`, `valid_from`, `valid_to (NULL=current)`, `source_atom_ids text[]`, `evidence_urls jsonb`, `llm_reasoning`, `created_by ('auto'|'moderator'|'breaking_news_watcher'|'auditor'|'extractor')`, `moderator_user_id`, `notes` | bigserial; FK fact_id → fact_status ON DELETE CASCADE |
|
||||
| `brain_audit_log` (NOU) | Cache mutation log: judge decisions, mass invalidations, gold promotions, fact truth changes. Coloane: `action`, `target_table`, `target_id`, `actor`, `payload jsonb` | bigserial |
|
||||
|
||||
GIN index-uri pe `topic_codes` (pentru topic-scoped invalidate). Partial index pe `cache_tier IN ('gold','silver') AND volatility != 'stable'` pentru auditor sweep. Schema migrează idempotent la fiecare brain `db.connect()`.
|
||||
|
||||
### DB live counts (2026-05-05, pe `didi-postgres`)
|
||||
|
||||
```
|
||||
internet_users: 21 | brain_fact_status: 4
|
||||
user_credentials: 20 | brain_fact_version: 1 (Putin → TRUE locked smoke-admin)
|
||||
subscriptions: 20 | brain_audit_log: ~10 (mostly fact_truth_changed + reset_password)
|
||||
subscription_plans: 20 | user_audit_log: live (logged on every PUT/DELETE/role/group)
|
||||
```
|
||||
101
backend/services/data-layer/didiDatabase/MIGRATION.md
Normal file
101
backend/services/data-layer/didiDatabase/MIGRATION.md
Normal file
|
|
@ -0,0 +1,101 @@
|
|||
# PostgreSQL — pe cluster Patroni HA (status curent)
|
||||
|
||||
> **TL;DR**: DIDI folosește **clusterul Patroni** extern (3 noduri PG + 3 etcd + 2 HAProxy LB). Containerul local `staging-dataLayer-postgres` din `data-layer/docker-compose.yml` păstrează **doar tabelul `waitlist`** — toate datele de business sunt pe cluster.
|
||||
|
||||
---
|
||||
|
||||
## Ce era aici (legacy)
|
||||
|
||||
Cândva, `staging-dataLayer-postgres` (Postgres 15 Alpine, container Docker) servea toate datele DIDI. Avea ~22 tabele în schemele `analyses`, `catalog`, `execution`, `pipelines`, `users` — toate din vechiul orchestrator Python. Acum sunt **goale, nefolosite**, schema veche arhivată în `/home/admin365/old_deprecated_code_archive/didiDatabase-legacy/`.
|
||||
|
||||
## Ce e acum
|
||||
|
||||
### Cluster Patroni (productie)
|
||||
|
||||
| Componentă | Hostname | IP | Port | Rol |
|
||||
|---|---|---|---|---|
|
||||
| pg-node1 | `pg-node1-test` | `10.11.50.160` | 5432 | Replica streaming |
|
||||
| pg-node2 | `pg-node2-test` | `10.11.50.161` | 5432 | Replica streaming |
|
||||
| **pg-node3** | `pg-node3-test` | `10.11.50.162` | 5432 | **Leader curent** |
|
||||
| etcd-node1/2/3 | — | `10.11.50.163-165` | 2379 | Quorum |
|
||||
| HAProxy LB1 | `haproxy-lb-test` | `10.11.50.166` | 5000 (RW), 5001 (RO) | Primary |
|
||||
| HAProxy LB2 | `haproxy-lb2-test` | `10.11.50.169` | 5000, 5001 | Secondary |
|
||||
| pgBackRest | `pg-backup-test` | `10.11.50.168` | — | Backup zilnic + NFS |
|
||||
|
||||
**Endpoint-uri pentru aplicații DIDI:**
|
||||
|
||||
| Scop | Endpoint | Notă |
|
||||
|---|---|---|
|
||||
| **WRITE** (orice modificare) | `10.11.50.167:5000` | DIDI configurat aici (HAProxy LB) |
|
||||
| READ (raportări) | `10.11.50.167:5001` | replica load-balanced |
|
||||
|
||||
> `.166`, `.167` și `.169` sunt toate HAProxy LB valide spre același cluster Patroni. DIDI folosește `.167` istoric. Verificat 2026-04-28: toate trei dau aceleași date (1782 sesiuni).
|
||||
|
||||
### Database principal: `DIDI`
|
||||
|
||||
User: `bos_interface` / parolă în vault-ul de credențiale `name='PostgreSQL Cluster Patroni (admin)'`.
|
||||
|
||||
4 scheme + public:
|
||||
- `bos_analysis` (6 tabele + view) — scrise de agent-v3
|
||||
- `bos_parammgmt` (~40 tabele) — scrise de didiFramework, sincronizate în Redis
|
||||
- `bos_sysadmin` (5 tabele) — utilizatori, credite, abonamente
|
||||
- `bos_subscriber` (4 tabele) — date personale
|
||||
- `public.waitlist` — pe **containerul local**, nu cluster
|
||||
|
||||
### Database-uri suplimentare pe același cluster
|
||||
|
||||
- `kong_db` (user `kong`) — folosit de Kong **cluster** (vezi `gateway-auth-layer/didiKong/MIGRATION.md`)
|
||||
- `keycloak_db` (user `keycloak`) — folosit de Keycloak
|
||||
|
||||
## Ce mai e local (containerul `staging-dataLayer-postgres`)
|
||||
|
||||
Definit în `data-layer/docker-compose.yml`. **Nu** e pe rețea externă — doar Docker network. Singurul tabel activ: `public.waitlist` în DB `misinformation_db` (3 înregistrări).
|
||||
|
||||
Folosit doar de `didiFramework/src/routes/waitlist.ts` prin pool separat (`stagingPool` cu host `staging-dataLayer-postgres`).
|
||||
|
||||
Schemele legacy (`analyses`, `catalog`, etc.) sunt goale.
|
||||
|
||||
### De ce nu am migrat waitlist pe cluster?
|
||||
|
||||
Decizie pragmatică: waitlist e public-facing (anyone-can-signup), volum mic, nu necesită HA. Containerul local e suficient. Migrare ulterioară opțională.
|
||||
|
||||
## Connection patterns în cod
|
||||
|
||||
```typescript
|
||||
// agent-v3/src/shared/persistence/pg-pool.ts
|
||||
host: '10.11.50.167', port: 5000, database: 'DIDI', user: 'bos_interface'
|
||||
|
||||
// didiFramework/src/config/database.ts (production data)
|
||||
host: '10.11.50.167', port: 5000, database: 'DIDI'
|
||||
|
||||
// didiFramework/src/routes/waitlist.ts (special — local container)
|
||||
host: 'staging-dataLayer-postgres', port: 5432, database: 'misinformation_db'
|
||||
```
|
||||
|
||||
## Verificare connectivity
|
||||
|
||||
```bash
|
||||
# Cu psql container (din host)
|
||||
docker run --rm --network host -e PGPASSWORD=<pwd> postgres:15-alpine \
|
||||
psql -h 10.11.50.167 -p 5000 -U bos_interface -d DIDI \
|
||||
-c "SELECT inet_server_addr() AS leader, now()"
|
||||
|
||||
# Patroni REST API status
|
||||
curl -s http://10.11.50.162:8008/cluster | python3 -m json.tool
|
||||
```
|
||||
|
||||
## Backup
|
||||
|
||||
pgBackRest zilnic (full săptămânal + incremental zilnic) pe `10.11.50.168` cu storage NFS la `10.11.10.150`. Toate DB-urile DIDI intră automat în stanza globală — nu trebuie config per-app.
|
||||
|
||||
## Linkuri rapide
|
||||
|
||||
- Ghid utilizare cluster: `landingzone/postgres-patroni/README.md` (repo `git.finesynergy.eu/lucian/landingzone`)
|
||||
- Onboarding aplicație nouă: `landingzone/postgres-patroni/CLAUDE_PROMPT.md`
|
||||
- HAProxy stats: `http://10.11.50.166:7000/stats`
|
||||
|
||||
## Status
|
||||
|
||||
- ✅ Migrare făcută înaintea acestui mono-repo (cluster Patroni e canonical)
|
||||
- ✅ Container local păstrat doar pentru waitlist
|
||||
- ⚠️ DIDI configurat pe HAProxy LB `.167` (canonical landingzone e `.166`); ambele rutează la același leader — schimbare cosmetică opțională
|
||||
63
backend/services/data-layer/didiDatabase/REBUILD.md
Normal file
63
backend/services/data-layer/didiDatabase/REBUILD.md
Normal file
|
|
@ -0,0 +1,63 @@
|
|||
# Rebuild bază de date pe alt host — rețetă
|
||||
|
||||
## Ce e local vs derivat (important înainte de rebuild)
|
||||
|
||||
| Store | Rol | Se seed-uiește? |
|
||||
|---|---|---|
|
||||
| **PostgreSQL** (`didi-postgres`, local pe didi11) | **sursă de adevăr** — 97 tabele, 4 scheme (bos_parammgmt, bos_analysis, bos_sysadmin, bos_subscriber) | **DA** — din dump-ul de mai jos |
|
||||
| **Redis** (`didi-cache`) chei `didi:config:*` + `didi:framework:*` | **cache derivat** din Postgres (populat de `sync-redis` din `component_config/prompt/stage_assignment`, `llm_model`, `moderation_*`) | **NU** — se regenerează cu `sync-redis` |
|
||||
| Redis `didi:pipeline:*` / `didi:queue:*` | stare runtime sesiuni (TTL) | NU — efemer |
|
||||
|
||||
**Concluzie:** NU sunt date dublate în sensul de „două surse de adevăr". Seed-uiești
|
||||
DOAR Postgres; Redis se reface singur dintr-o comandă. Nu există fișier de seed
|
||||
pentru Redis și nici nu e nevoie.
|
||||
|
||||
## Fișierul de seed
|
||||
|
||||
`DIDI_full_export_2026-07-02.sql` (23 MB) — pg_dump complet: schema + date + toate
|
||||
migrațiile (inclusiv 016 input_type_profile_version, 017 model catalog attributes).
|
||||
Restorabil (`--clean --if-exists --no-owner`). Validat: restore curat pe Postgres
|
||||
gol → 99 tabele, date reale (23 modele LLM, 83 stage assignments, 6 profiluri).
|
||||
|
||||
> Seed-ul vechi `DIDI_full_export_2026-03-22.sql` (fără migrațiile 016/017) și pachetul
|
||||
> demo (`DIDI_demo_seed_2026-07-02.sql` + `demo-seed/`) au fost arhivate în afara repo-ului
|
||||
> (`/home/admin365/didi_seed_archive_2026-07-08/`) — livrarea folosește DOAR full seed-ul curent.
|
||||
|
||||
## Pași rebuild
|
||||
|
||||
```bash
|
||||
# 1. Pornește un Postgres (local container SAU clusterul extern — vezi mai jos)
|
||||
# Aici: containerul local, ca pe didi11.
|
||||
docker compose -f services/data-layer/docker-compose.local.yml up -d didi-postgres
|
||||
until docker exec didi-postgres pg_isready -U bos_interface; do sleep 2; done
|
||||
|
||||
# 2. Restaurează schema + datele
|
||||
docker exec -i didi-postgres psql -U bos_interface -d DIDI \
|
||||
< services/data-layer/didiDatabase/DIDI_full_export_2026-07-02.sql
|
||||
# (un singur warning benign 'transaction_timeout' pe versiuni PG <17 — se ignoră)
|
||||
|
||||
# 3. Pornește restul serviciilor (agent-v3, framework, workeri) — se conectează
|
||||
# la didi-postgres prin PG_HOST/DB_HOST din compose.
|
||||
cd services/orchestration-layer/agent-v3 && docker compose up -d
|
||||
cd ../didiFramework && docker compose up -d didi-framework
|
||||
|
||||
# 4. Regenerează cache-ul Redis din Postgres (config + framework params)
|
||||
docker exec didi-framework sh -c 'wget -qO- --post-data="" http://127.0.0.1:3005/api/sync-redis'
|
||||
|
||||
# 5. (verificare) Redis populat + un răspuns 200 pe framework
|
||||
docker exec didi-cache redis-cli -a redis123 --no-auth-warning dbsize
|
||||
curl -sf http://localhost:3005/health
|
||||
```
|
||||
|
||||
## Local vs cluster extern
|
||||
|
||||
Serviciile sunt agnostice — `PG_HOST`/`DB_HOST` din compose decid ținta:
|
||||
- **didi11 (acum):** `didi-postgres` (container local, 5432).
|
||||
- **Producție/cluster:** setează `PG_HOST=10.11.50.167 PG_PORT=5000` (VIP Patroni/HAProxy).
|
||||
Același dump se restaurează în oricare; la cluster, restaurează pe leaderul RW (`:5000`).
|
||||
|
||||
## HA opțional
|
||||
|
||||
Dacă vrei Postgres HA pe noul host (nu single-node), vezi
|
||||
`ha-cluster/` (Patroni + etcd + HAProxy) — restaurează dump-ul pe `:5000` după
|
||||
`patronictl list` arată un leader.
|
||||
104
backend/services/data-layer/didiDatabase/ha-cluster/README.md
Normal file
104
backend/services/data-layer/didiDatabase/ha-cluster/README.md
Normal file
|
|
@ -0,0 +1,104 @@
|
|||
# DIDI PostgreSQL HA — Patroni + etcd + HAProxy (IaC livrabil)
|
||||
|
||||
Pachet **reproductibil** care livrează clusterul HA PostgreSQL al platformei DiDi
|
||||
ca Infrastructure-as-Code. Aceeași arhitectură rulează în producție pe VM-uri
|
||||
dedicate (vezi `../MIGRATION.md`); acest compose o reproduce integral pe un
|
||||
singur host pentru demo, recepție, DR-rehearsal și medii de test.
|
||||
|
||||
## Arhitectură
|
||||
|
||||
```
|
||||
┌────────────────────┐
|
||||
apps ──5000──▶ │ HAProxy │ ──▶ /primary (Patroni REST :8008)
|
||||
apps ──5001──▶ │ (LB + healthcheck)│ ──▶ /replica
|
||||
└─────────┬──────────┘
|
||||
┌───────────────┼───────────────┐
|
||||
┌─────▼─────┐ ┌─────▼─────┐ ┌─────▼─────┐
|
||||
│ pg-node1 │ │ pg-node2 │ │ pg-node3 │ Spilo = PostgreSQL 16
|
||||
│ Patroni │ │ Patroni │ │ Patroni │ + Patroni (Zalando)
|
||||
└─────┬─────┘ └─────┬─────┘ └─────┬─────┘
|
||||
└───────────────┼───────────────┘
|
||||
┌────────▼────────┐
|
||||
│ etcd1/2/3 (DCS) │ quorum leader-election
|
||||
└─────────────────┘
|
||||
```
|
||||
|
||||
| Rol | Producție (VM-uri) | Acest pachet |
|
||||
|---|---|---|
|
||||
| PG + Patroni ×3 | 10.11.50.160–162 | `pg-node1..3` (Spilo 16) |
|
||||
| etcd quorum ×3 | 10.11.50.163–165 | `etcd1..3` (v3.5) |
|
||||
| HAProxy | 10.11.50.166 + 169 (VIP .167) | `haproxy` :5000/:5001 |
|
||||
| Backup | pgBackRest (10.11.50.168, NFS) | vezi §Backup |
|
||||
|
||||
## Pornire
|
||||
|
||||
```bash
|
||||
docker compose up -d
|
||||
# election durează ~30-60s; verifică:
|
||||
docker exec didi-ha-pg1 patronictl list
|
||||
```
|
||||
|
||||
Conectare (contract identic cu producția):
|
||||
|
||||
```bash
|
||||
PGPASSWORD=didi-super-secret psql -h localhost -p 5000 -U postgres # RW (leader)
|
||||
PGPASSWORD=didi-super-secret psql -h localhost -p 5001 -U postgres # RO (replici)
|
||||
```
|
||||
|
||||
Restaurare schema DIDI (bos_parammgmt / bos_analysis / bos_sysadmin / bos_subscriber):
|
||||
|
||||
```bash
|
||||
PGPASSWORD=didi-super-secret psql -h localhost -p 5000 -U postgres \
|
||||
-f ../DIDI_full_export_2026-07-02.sql
|
||||
```
|
||||
|
||||
## Test failover (drill de recepție)
|
||||
|
||||
```bash
|
||||
# 1. află liderul
|
||||
docker exec didi-ha-pg1 patronictl list
|
||||
# 2. omoară-l
|
||||
docker stop didi-ha-pg2 # (dacă pg2 e leader)
|
||||
# 3. Patroni promovează o replică în secunde; HAProxy reroutează :5000
|
||||
# automat (healthcheck /primary la 3s, fall 3). Aplicațiile nu schimbă
|
||||
# nimic — se reconectează pe același endpoint.
|
||||
docker exec didi-ha-pg1 patronictl list
|
||||
# 4. reintră nodul căzut ca replică:
|
||||
docker start didi-ha-pg2
|
||||
```
|
||||
|
||||
Switchover planificat (fără downtime):
|
||||
|
||||
```bash
|
||||
docker exec didi-ha-pg1 patronictl switchover didi --force
|
||||
```
|
||||
|
||||
## Parametri
|
||||
|
||||
| Env | Default | Rol |
|
||||
|---|---|---|
|
||||
| `PG_SUPERUSER_PASSWORD` | `didi-super-secret` | postgres superuser |
|
||||
| `PG_ADMIN_PASSWORD` | `didi-admin-secret` | admin role |
|
||||
| `PG_STANDBY_PASSWORD` | `didi-standby-secret` | replicare streaming |
|
||||
|
||||
**Schimbă-le obligatoriu în producție** (`.env` lângă compose).
|
||||
|
||||
## Backup
|
||||
|
||||
În producție backup-ul e pgBackRest (full zilnic + WAL archiving pe NFS,
|
||||
nod dedicat). Pe acest pachet, echivalentul minim:
|
||||
|
||||
```bash
|
||||
docker exec didi-ha-pg1 su postgres -c \
|
||||
'pg_basebackup -h localhost -p 5432 -D /tmp/didi-backup -Ft -z -Xs'
|
||||
```
|
||||
|
||||
## Relația cu livrabilul Lot 2
|
||||
|
||||
- Modulul 5 (Baze de date SQL) cere PostgreSQL cu HA; oferta specifică
|
||||
Patroni + HAProxy. Acest director este implementarea IaC livrată —
|
||||
reproductibilă pe orice host Docker, plus instanțierea de producție
|
||||
documentată în `MIGRATION.md`.
|
||||
- Aplicațiile (agent-v3, didiFramework) sunt agnostice: `PG_HOST:PG_PORT`
|
||||
arată fie spre VIP-ul de producție (`10.11.50.167:5000`), fie spre acest
|
||||
cluster local (`localhost:5000`) — același contract, zero modificări de cod.
|
||||
|
|
@ -0,0 +1,116 @@
|
|||
# ============================================================================
|
||||
# DIDI PostgreSQL HA cluster — Patroni + etcd + HAProxy (IaC, reproducible)
|
||||
#
|
||||
# Containerized mirror of the production topology (see ../MIGRATION.md):
|
||||
# prod: 3× PG/Patroni (10.11.50.160-162) + 3× etcd (163-165)
|
||||
# + 2× HAProxy (166/169, VIP 167) + pgBackRest (168)
|
||||
# here: 3× Spilo (Patroni+PG, Zalando) + 3× etcd + 1× HAProxy
|
||||
# → same failover semantics, single-host footprint for
|
||||
# demo/recepție/DR-rehearsal.
|
||||
#
|
||||
# Endpoints (identical contract to production):
|
||||
# localhost:5000 → leader (read-write) [HAProxy checks Patroni /primary]
|
||||
# localhost:5001 → replicas (read-only) [HAProxy checks Patroni /replica]
|
||||
# localhost:7000 → HAProxy stats UI
|
||||
#
|
||||
# Usage:
|
||||
# docker compose up -d
|
||||
# # wait ~30s for leader election, then:
|
||||
# psql -h localhost -p 5000 -U postgres # password: $PG_SUPERUSER_PASSWORD
|
||||
# # restore DIDI schema:
|
||||
# psql -h localhost -p 5000 -U postgres -f ../DIDI_full_export_2026-07-02.sql
|
||||
# # failover drill:
|
||||
# docker compose stop $(docker compose ps --format '{{.Name}}' | head -1)
|
||||
# # → a replica is promoted in seconds; :5000 keeps serving writes.
|
||||
# ============================================================================
|
||||
|
||||
x-etcd-common: &etcd-common
|
||||
image: quay.io/coreos/etcd:v3.5.16
|
||||
restart: unless-stopped
|
||||
networks: [didi-ha]
|
||||
environment: &etcd-env
|
||||
ETCD_INITIAL_CLUSTER: etcd1=http://etcd1:2380,etcd2=http://etcd2:2380,etcd3=http://etcd3:2380
|
||||
ETCD_INITIAL_CLUSTER_STATE: new
|
||||
ETCD_INITIAL_CLUSTER_TOKEN: didi-pg-ha
|
||||
ETCD_AUTO_COMPACTION_RETENTION: "1"
|
||||
ETCD_ENABLE_V2: "true"
|
||||
|
||||
x-spilo-common: &spilo-common
|
||||
image: ghcr.io/zalando/spilo-16:3.3-p3
|
||||
restart: unless-stopped
|
||||
networks: [didi-ha]
|
||||
environment: &spilo-env
|
||||
SCOPE: didi # Patroni cluster name (etcd namespace)
|
||||
PGVERSION: "16"
|
||||
ETCD3_HOSTS: "'etcd1:2379','etcd2:2379','etcd3:2379'"
|
||||
PGPASSWORD_SUPERUSER: ${PG_SUPERUSER_PASSWORD:-didi-super-secret}
|
||||
PGPASSWORD_ADMIN: ${PG_ADMIN_PASSWORD:-didi-admin-secret}
|
||||
PGPASSWORD_STANDBY: ${PG_STANDBY_PASSWORD:-didi-standby-secret}
|
||||
ALLOW_NOSSL: "true"
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "curl -sf http://localhost:8008/health || exit 1"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 6
|
||||
start_period: 60s
|
||||
|
||||
services:
|
||||
etcd1:
|
||||
<<: *etcd-common
|
||||
container_name: didi-ha-etcd1
|
||||
command: etcd --name etcd1
|
||||
--listen-peer-urls http://0.0.0.0:2380 --initial-advertise-peer-urls http://etcd1:2380
|
||||
--listen-client-urls http://0.0.0.0:2379 --advertise-client-urls http://etcd1:2379
|
||||
etcd2:
|
||||
<<: *etcd-common
|
||||
container_name: didi-ha-etcd2
|
||||
command: etcd --name etcd2
|
||||
--listen-peer-urls http://0.0.0.0:2380 --initial-advertise-peer-urls http://etcd2:2380
|
||||
--listen-client-urls http://0.0.0.0:2379 --advertise-client-urls http://etcd2:2379
|
||||
etcd3:
|
||||
<<: *etcd-common
|
||||
container_name: didi-ha-etcd3
|
||||
command: etcd --name etcd3
|
||||
--listen-peer-urls http://0.0.0.0:2380 --initial-advertise-peer-urls http://etcd3:2380
|
||||
--listen-client-urls http://0.0.0.0:2379 --advertise-client-urls http://etcd3:2379
|
||||
|
||||
pg-node1:
|
||||
<<: *spilo-common
|
||||
container_name: didi-ha-pg1
|
||||
hostname: pg-node1
|
||||
depends_on: [etcd1, etcd2, etcd3]
|
||||
volumes: [pg1-data:/home/postgres/pgdata]
|
||||
pg-node2:
|
||||
<<: *spilo-common
|
||||
container_name: didi-ha-pg2
|
||||
hostname: pg-node2
|
||||
depends_on: [etcd1, etcd2, etcd3]
|
||||
volumes: [pg2-data:/home/postgres/pgdata]
|
||||
pg-node3:
|
||||
<<: *spilo-common
|
||||
container_name: didi-ha-pg3
|
||||
hostname: pg-node3
|
||||
depends_on: [etcd1, etcd2, etcd3]
|
||||
volumes: [pg3-data:/home/postgres/pgdata]
|
||||
|
||||
haproxy:
|
||||
image: haproxy:2.9-alpine
|
||||
container_name: didi-ha-haproxy
|
||||
restart: unless-stopped
|
||||
networks: [didi-ha]
|
||||
depends_on: [pg-node1, pg-node2, pg-node3]
|
||||
ports:
|
||||
- "5000:5000" # read-write → Patroni leader
|
||||
- "5001:5001" # read-only → replicas
|
||||
- "7000:7000" # stats UI
|
||||
volumes:
|
||||
- ./haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
|
||||
|
||||
volumes:
|
||||
pg1-data:
|
||||
pg2-data:
|
||||
pg3-data:
|
||||
|
||||
networks:
|
||||
didi-ha:
|
||||
name: didi-ha
|
||||
|
|
@ -0,0 +1,46 @@
|
|||
# HAProxy for DIDI PostgreSQL HA — routes by Patroni REST health checks.
|
||||
# Mirrors the production LB config (10.11.50.166/169 → VIP 167).
|
||||
#
|
||||
# :5000 → the ONE node whose Patroni answers 200 on /primary (leader, RW)
|
||||
# :5001 → nodes answering 200 on /replica (round-robin, RO)
|
||||
#
|
||||
# On failover Patroni flips the health endpoints; HAProxy reroutes in
|
||||
# (inter × fall) ≈ 9s worst case without client config changes.
|
||||
|
||||
global
|
||||
maxconn 300
|
||||
log stdout format raw local0
|
||||
|
||||
defaults
|
||||
log global
|
||||
mode tcp
|
||||
retries 2
|
||||
timeout client 30m
|
||||
timeout connect 4s
|
||||
timeout server 30m
|
||||
timeout check 5s
|
||||
|
||||
listen stats
|
||||
mode http
|
||||
bind *:7000
|
||||
stats enable
|
||||
stats uri /
|
||||
|
||||
listen postgres_write
|
||||
bind *:5000
|
||||
option httpchk GET /primary
|
||||
http-check expect status 200
|
||||
default-server inter 3s fall 3 rise 2 on-marked-down shutdown-sessions
|
||||
server pg-node1 pg-node1:5432 check port 8008
|
||||
server pg-node2 pg-node2:5432 check port 8008
|
||||
server pg-node3 pg-node3:5432 check port 8008
|
||||
|
||||
listen postgres_read
|
||||
bind *:5001
|
||||
balance roundrobin
|
||||
option httpchk GET /replica
|
||||
http-check expect status 200
|
||||
default-server inter 3s fall 3 rise 2 on-marked-down shutdown-sessions
|
||||
server pg-node1 pg-node1:5432 check port 8008
|
||||
server pg-node2 pg-node2:5432 check port 8008
|
||||
server pg-node3 pg-node3:5432 check port 8008
|
||||
Loading…
Add table
Add a link
Reference in a new issue