livrare lot 2

backend/observability/prometheus/rules/alerts.yml

@@ -0,0 +1,119 @@
+groups:
+  - name: didi-platform-availability
+    interval: 1m
+    rules:
+      - alert: ServiceDown
+        expr: up == 0
+        for: 5m
+        labels:
+          severity: critical
+          team: platform
+        annotations:
+          summary: "Service {{ $labels.job }} ({{ $labels.instance }}) is DOWN"
+          description: "{{ $labels.job }} has been unreachable for >5m. Last scrape: {{ $value }}"
+
+      - alert: HighErrorRate
+        expr: |
+          sum(rate(didi_http_requests_total{status=~"5.."}[5m])) by (service)
+            /
+          sum(rate(didi_http_requests_total[5m])) by (service)
+          > 0.05
+        for: 10m
+        labels:
+          severity: warning
+          team: platform
+        annotations:
+          summary: "High 5xx error rate on {{ $labels.service }}"
+          description: "{{ $labels.service }} has >5% 5xx errors for >10m"
+
+  - name: didi-pipeline-performance
+    interval: 1m
+    rules:
+      - alert: PipelineLatencyHigh
+        expr: |
+          histogram_quantile(0.95,
+            sum(rate(didi_pipeline_duration_seconds_bucket[5m])) by (le, component)
+          ) > 60
+        for: 15m
+        labels:
+          severity: warning
+          team: platform
+        annotations:
+          summary: "Pipeline component {{ $labels.component }} P95 latency >60s"
+          description: "P95 latency on {{ $labels.component }} is {{ $value }}s"
+
+      - alert: RabbitMQQueueBacklog
+        expr: rabbitmq_queue_messages_ready > 100
+        for: 10m
+        labels:
+          severity: warning
+          team: platform
+        annotations:
+          summary: "Queue {{ $labels.queue }} backlog >100 messages"
+          description: "{{ $labels.queue }} has {{ $value }} unprocessed messages"
+
+      - alert: DeadLetterMessages
+        expr: increase(didi_dlq_messages_total[15m]) > 0
+        labels:
+          severity: warning
+          team: platform
+        annotations:
+          summary: "Messages landing in analysis_dlq ({{ $labels.component }})"
+          description: "{{ $value }} dead-lettered messages in 15m — check didi:queue:dlq:recent in Redis for session ids"
+
+  - name: didi-infrastructure
+    interval: 1m
+    rules:
+      - alert: HighCPU
+        expr: 100 - (avg by(instance) (irate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 85
+        for: 10m
+        labels:
+          severity: warning
+          team: ops
+        annotations:
+          summary: "CPU >85% on {{ $labels.instance }}"
+
+      - alert: LowDiskSpace
+        expr: |
+          (node_filesystem_avail_bytes{mountpoint="/"}
+           / node_filesystem_size_bytes{mountpoint="/"})
+          < 0.15
+        for: 5m
+        labels:
+          severity: critical
+          team: ops
+        annotations:
+          summary: "Disk space <15% on {{ $labels.instance }}"
+
+      - alert: GPUMemoryHigh
+        expr: nvidia_gpu_memory_used_bytes / nvidia_gpu_memory_total_bytes > 0.95
+        for: 10m
+        labels:
+          severity: warning
+          team: ai-platform
+        annotations:
+          summary: "GPU {{ $labels.gpu }} memory >95% on {{ $labels.instance }}"
+
+  - name: didi-business
+    interval: 5m
+    rules:
+      - alert: NoAnalysesIn30Min
+        expr: |
+          increase(didi_analyses_completed_total[30m]) == 0
+        for: 30m
+        labels:
+          severity: warning
+          team: product
+        annotations:
+          summary: "No analyses completed in last 30 minutes"
+          description: "Platform may be down or no traffic — investigate"
+
+      - alert: StripeWebhookFailures
+        expr: |
+          rate(didi_stripe_webhook_failures_total[5m]) > 0.1
+        for: 5m
+        labels:
+          severity: critical
+          team: revenue
+        annotations:
+          summary: "Stripe webhook failures detected"