livrare lot 2
This commit is contained in:
commit
8ecc78e729
763 changed files with 164593 additions and 0 deletions
104
backend/observability/prometheus/prometheus.yml
Normal file
104
backend/observability/prometheus/prometheus.yml
Normal file
|
|
@ -0,0 +1,104 @@
|
|||
global:
|
||||
scrape_interval: 30s
|
||||
evaluation_interval: 30s
|
||||
external_labels:
|
||||
cluster: didi-prod
|
||||
environment: production
|
||||
|
||||
rule_files:
|
||||
- /etc/prometheus/rules/*.yml
|
||||
|
||||
alerting:
|
||||
alertmanagers:
|
||||
- static_configs:
|
||||
- targets:
|
||||
- didi-alertmanager:9093
|
||||
|
||||
scrape_configs:
|
||||
# Prometheus self-monitoring
|
||||
- job_name: prometheus
|
||||
static_configs:
|
||||
- targets: ['localhost:9090']
|
||||
|
||||
# Backend Node.js services (expun /metrics via prom-client)
|
||||
- job_name: agent-v3
|
||||
metrics_path: /metrics
|
||||
static_configs:
|
||||
- targets: ['didi-agent-v3:24803']
|
||||
labels: { service: agent-v3, layer: orchestration }
|
||||
|
||||
- job_name: didi-framework
|
||||
metrics_path: /metrics
|
||||
static_configs:
|
||||
- targets: ['didi-framework:3005']
|
||||
labels: { service: framework, layer: orchestration }
|
||||
|
||||
# AI Platform Python services (expun /metrics via prometheus_client)
|
||||
- job_name: ai-platform
|
||||
metrics_path: /metrics
|
||||
static_configs:
|
||||
- targets:
|
||||
- 'didiAI-llm-api:14011'
|
||||
- 'didiAI-embeddings-api:14100'
|
||||
- 'didiAI-rerank-api:14200'
|
||||
- 'didiAI-audio-api:54300'
|
||||
- 'didiAI-video-api:54600'
|
||||
- 'didiAI-web-api:51100'
|
||||
- 'didiAI-catalog-api:11000'
|
||||
- 'didiAI-dashboard:51300'
|
||||
- 'didibrain-api:8090'
|
||||
labels: { layer: ai-platform }
|
||||
|
||||
# Workers (expun /metrics pe portul intern)
|
||||
- job_name: agent-v3-workers
|
||||
metrics_path: /metrics
|
||||
static_configs:
|
||||
- targets:
|
||||
- 'agent-v3-worker-techniques-1:24803'
|
||||
- 'agent-v3-worker-techniques-2:24803'
|
||||
- 'agent-v3-worker-ai-tampered-1:24803'
|
||||
- 'agent-v3-worker-ai-tampered-2:24803'
|
||||
- 'agent-v3-worker-claims-1:24803'
|
||||
- 'agent-v3-worker-claims-2:24803'
|
||||
- 'agent-v3-worker-claims-3:24803'
|
||||
- 'agent-v3-worker-domain-1:24803'
|
||||
- 'agent-v3-worker-domain-2:24803'
|
||||
- 'agent-v3-worker-media-preprocess-1:24803'
|
||||
- 'agent-v3-worker-media-preprocess-2:24803'
|
||||
- 'agent-v3-verdict-aggregator-1:24803'
|
||||
- 'agent-v3-verdict-aggregator-2:24803'
|
||||
labels: { service: agent-v3, layer: workers }
|
||||
|
||||
# Infrastructure metrics (cadvisor + node-exporter already running on host)
|
||||
- job_name: cadvisor
|
||||
static_configs:
|
||||
- targets: ['heimdall_cadvisor:8080']
|
||||
labels: { layer: infrastructure }
|
||||
|
||||
- job_name: node-exporter
|
||||
static_configs:
|
||||
- targets: ['heimdall_node_exporter:9100']
|
||||
labels: { layer: infrastructure }
|
||||
|
||||
# OTel Collector self-metrics + traces (metric forwarder)
|
||||
- job_name: otel-collector
|
||||
static_configs:
|
||||
- targets: ['didi-otel-collector:8889']
|
||||
labels: { layer: observability }
|
||||
|
||||
# PostgreSQL via postgres_exporter (opt-in — add postgres-exporter container if needed)
|
||||
# - job_name: postgres-exporter
|
||||
# static_configs:
|
||||
# - targets: ['postgres-exporter:9187']
|
||||
|
||||
# RabbitMQ has built-in prometheus support since 3.10 — enable rabbitmq_prometheus plugin
|
||||
- job_name: rabbitmq
|
||||
metrics_path: /metrics
|
||||
static_configs:
|
||||
- targets: ['staging-dataLayer-rabbitmq:15692']
|
||||
labels: { service: rabbitmq, layer: data }
|
||||
|
||||
# Redis via redis_exporter (opt-in)
|
||||
# - job_name: redis-exporter
|
||||
# static_configs:
|
||||
# - targets: ['redis-exporter:9121']
|
||||
119
backend/observability/prometheus/rules/alerts.yml
Normal file
119
backend/observability/prometheus/rules/alerts.yml
Normal file
|
|
@ -0,0 +1,119 @@
|
|||
groups:
|
||||
- name: didi-platform-availability
|
||||
interval: 1m
|
||||
rules:
|
||||
- alert: ServiceDown
|
||||
expr: up == 0
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
team: platform
|
||||
annotations:
|
||||
summary: "Service {{ $labels.job }} ({{ $labels.instance }}) is DOWN"
|
||||
description: "{{ $labels.job }} has been unreachable for >5m. Last scrape: {{ $value }}"
|
||||
|
||||
- alert: HighErrorRate
|
||||
expr: |
|
||||
sum(rate(didi_http_requests_total{status=~"5.."}[5m])) by (service)
|
||||
/
|
||||
sum(rate(didi_http_requests_total[5m])) by (service)
|
||||
> 0.05
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
team: platform
|
||||
annotations:
|
||||
summary: "High 5xx error rate on {{ $labels.service }}"
|
||||
description: "{{ $labels.service }} has >5% 5xx errors for >10m"
|
||||
|
||||
- name: didi-pipeline-performance
|
||||
interval: 1m
|
||||
rules:
|
||||
- alert: PipelineLatencyHigh
|
||||
expr: |
|
||||
histogram_quantile(0.95,
|
||||
sum(rate(didi_pipeline_duration_seconds_bucket[5m])) by (le, component)
|
||||
) > 60
|
||||
for: 15m
|
||||
labels:
|
||||
severity: warning
|
||||
team: platform
|
||||
annotations:
|
||||
summary: "Pipeline component {{ $labels.component }} P95 latency >60s"
|
||||
description: "P95 latency on {{ $labels.component }} is {{ $value }}s"
|
||||
|
||||
- alert: RabbitMQQueueBacklog
|
||||
expr: rabbitmq_queue_messages_ready > 100
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
team: platform
|
||||
annotations:
|
||||
summary: "Queue {{ $labels.queue }} backlog >100 messages"
|
||||
description: "{{ $labels.queue }} has {{ $value }} unprocessed messages"
|
||||
|
||||
- alert: DeadLetterMessages
|
||||
expr: increase(didi_dlq_messages_total[15m]) > 0
|
||||
labels:
|
||||
severity: warning
|
||||
team: platform
|
||||
annotations:
|
||||
summary: "Messages landing in analysis_dlq ({{ $labels.component }})"
|
||||
description: "{{ $value }} dead-lettered messages in 15m — check didi:queue:dlq:recent in Redis for session ids"
|
||||
|
||||
- name: didi-infrastructure
|
||||
interval: 1m
|
||||
rules:
|
||||
- alert: HighCPU
|
||||
expr: 100 - (avg by(instance) (irate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 85
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
team: ops
|
||||
annotations:
|
||||
summary: "CPU >85% on {{ $labels.instance }}"
|
||||
|
||||
- alert: LowDiskSpace
|
||||
expr: |
|
||||
(node_filesystem_avail_bytes{mountpoint="/"}
|
||||
/ node_filesystem_size_bytes{mountpoint="/"})
|
||||
< 0.15
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
team: ops
|
||||
annotations:
|
||||
summary: "Disk space <15% on {{ $labels.instance }}"
|
||||
|
||||
- alert: GPUMemoryHigh
|
||||
expr: nvidia_gpu_memory_used_bytes / nvidia_gpu_memory_total_bytes > 0.95
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
team: ai-platform
|
||||
annotations:
|
||||
summary: "GPU {{ $labels.gpu }} memory >95% on {{ $labels.instance }}"
|
||||
|
||||
- name: didi-business
|
||||
interval: 5m
|
||||
rules:
|
||||
- alert: NoAnalysesIn30Min
|
||||
expr: |
|
||||
increase(didi_analyses_completed_total[30m]) == 0
|
||||
for: 30m
|
||||
labels:
|
||||
severity: warning
|
||||
team: product
|
||||
annotations:
|
||||
summary: "No analyses completed in last 30 minutes"
|
||||
description: "Platform may be down or no traffic — investigate"
|
||||
|
||||
- alert: StripeWebhookFailures
|
||||
expr: |
|
||||
rate(didi_stripe_webhook_failures_total[5m]) > 0.1
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
team: revenue
|
||||
annotations:
|
||||
summary: "Stripe webhook failures detected"
|
||||
Loading…
Add table
Add a link
Reference in a new issue