119 lines
3.7 KiB
YAML
119 lines
3.7 KiB
YAML
groups:
|
|
- name: didi-platform-availability
|
|
interval: 1m
|
|
rules:
|
|
- alert: ServiceDown
|
|
expr: up == 0
|
|
for: 5m
|
|
labels:
|
|
severity: critical
|
|
team: platform
|
|
annotations:
|
|
summary: "Service {{ $labels.job }} ({{ $labels.instance }}) is DOWN"
|
|
description: "{{ $labels.job }} has been unreachable for >5m. Last scrape: {{ $value }}"
|
|
|
|
- alert: HighErrorRate
|
|
expr: |
|
|
sum(rate(didi_http_requests_total{status=~"5.."}[5m])) by (service)
|
|
/
|
|
sum(rate(didi_http_requests_total[5m])) by (service)
|
|
> 0.05
|
|
for: 10m
|
|
labels:
|
|
severity: warning
|
|
team: platform
|
|
annotations:
|
|
summary: "High 5xx error rate on {{ $labels.service }}"
|
|
description: "{{ $labels.service }} has >5% 5xx errors for >10m"
|
|
|
|
- name: didi-pipeline-performance
|
|
interval: 1m
|
|
rules:
|
|
- alert: PipelineLatencyHigh
|
|
expr: |
|
|
histogram_quantile(0.95,
|
|
sum(rate(didi_pipeline_duration_seconds_bucket[5m])) by (le, component)
|
|
) > 60
|
|
for: 15m
|
|
labels:
|
|
severity: warning
|
|
team: platform
|
|
annotations:
|
|
summary: "Pipeline component {{ $labels.component }} P95 latency >60s"
|
|
description: "P95 latency on {{ $labels.component }} is {{ $value }}s"
|
|
|
|
- alert: RabbitMQQueueBacklog
|
|
expr: rabbitmq_queue_messages_ready > 100
|
|
for: 10m
|
|
labels:
|
|
severity: warning
|
|
team: platform
|
|
annotations:
|
|
summary: "Queue {{ $labels.queue }} backlog >100 messages"
|
|
description: "{{ $labels.queue }} has {{ $value }} unprocessed messages"
|
|
|
|
- alert: DeadLetterMessages
|
|
expr: increase(didi_dlq_messages_total[15m]) > 0
|
|
labels:
|
|
severity: warning
|
|
team: platform
|
|
annotations:
|
|
summary: "Messages landing in analysis_dlq ({{ $labels.component }})"
|
|
description: "{{ $value }} dead-lettered messages in 15m — check didi:queue:dlq:recent in Redis for session ids"
|
|
|
|
- name: didi-infrastructure
|
|
interval: 1m
|
|
rules:
|
|
- alert: HighCPU
|
|
expr: 100 - (avg by(instance) (irate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 85
|
|
for: 10m
|
|
labels:
|
|
severity: warning
|
|
team: ops
|
|
annotations:
|
|
summary: "CPU >85% on {{ $labels.instance }}"
|
|
|
|
- alert: LowDiskSpace
|
|
expr: |
|
|
(node_filesystem_avail_bytes{mountpoint="/"}
|
|
/ node_filesystem_size_bytes{mountpoint="/"})
|
|
< 0.15
|
|
for: 5m
|
|
labels:
|
|
severity: critical
|
|
team: ops
|
|
annotations:
|
|
summary: "Disk space <15% on {{ $labels.instance }}"
|
|
|
|
- alert: GPUMemoryHigh
|
|
expr: nvidia_gpu_memory_used_bytes / nvidia_gpu_memory_total_bytes > 0.95
|
|
for: 10m
|
|
labels:
|
|
severity: warning
|
|
team: ai-platform
|
|
annotations:
|
|
summary: "GPU {{ $labels.gpu }} memory >95% on {{ $labels.instance }}"
|
|
|
|
- name: didi-business
|
|
interval: 5m
|
|
rules:
|
|
- alert: NoAnalysesIn30Min
|
|
expr: |
|
|
increase(didi_analyses_completed_total[30m]) == 0
|
|
for: 30m
|
|
labels:
|
|
severity: warning
|
|
team: product
|
|
annotations:
|
|
summary: "No analyses completed in last 30 minutes"
|
|
description: "Platform may be down or no traffic — investigate"
|
|
|
|
- alert: StripeWebhookFailures
|
|
expr: |
|
|
rate(didi_stripe_webhook_failures_total[5m]) > 0.1
|
|
for: 5m
|
|
labels:
|
|
severity: critical
|
|
team: revenue
|
|
annotations:
|
|
summary: "Stripe webhook failures detected"
|