93 lines
3.5 KiB
YAML
93 lines
3.5 KiB
YAML
|
groups:
|
||
|
|
- name: monitoring
|
||
|
|
rules:
|
||
|
|
- alert: PrometheusTargetDown
|
||
|
|
expr: up == 0
|
||
|
|
for: 5m
|
||
|
|
labels:
|
||
|
|
severity: critical
|
||
|
|
annotations:
|
||
|
|
summary: "Prometheus target is down: {{ $labels.instance }}"
|
||
|
|
description: "Job {{ $labels.job }} has been unreachable for more than five minutes."
|
||
|
|
|
||
|
|
- alert: HostHighCPU
|
||
|
|
expr: 100 * (1 - avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m]))) > 90
|
||
|
|
for: 15m
|
||
|
|
labels:
|
||
|
|
severity: warning
|
||
|
|
annotations:
|
||
|
|
summary: "Host CPU usage is above 90%"
|
||
|
|
description: "CPU usage on {{ $labels.instance }} has remained high for 15 minutes."
|
||
|
|
|
||
|
|
- alert: HostLowMemory
|
||
|
|
expr: 100 * node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes < 10
|
||
|
|
for: 10m
|
||
|
|
labels:
|
||
|
|
severity: warning
|
||
|
|
annotations:
|
||
|
|
summary: "Host available memory is below 10%"
|
||
|
|
description: "Available memory on {{ $labels.instance }} has remained low for 10 minutes."
|
||
|
|
|
||
|
|
- alert: HostDiskSpaceLow
|
||
|
|
expr: 100 * node_filesystem_avail_bytes{fstype!~"tmpfs|overlay|squashfs"} / node_filesystem_size_bytes{fstype!~"tmpfs|overlay|squashfs"} < 10
|
||
|
|
for: 15m
|
||
|
|
labels:
|
||
|
|
severity: critical
|
||
|
|
annotations:
|
||
|
|
summary: "Host filesystem has less than 10% free space"
|
||
|
|
description: "{{ $labels.mountpoint }} on {{ $labels.instance }} is nearly full."
|
||
|
|
|
||
|
|
- alert: HostFilesystemReadOnly
|
||
|
|
expr: node_filesystem_readonly{fstype!~"tmpfs|squashfs"} == 1
|
||
|
|
for: 5m
|
||
|
|
labels:
|
||
|
|
severity: critical
|
||
|
|
annotations:
|
||
|
|
summary: "Host filesystem became read-only"
|
||
|
|
description: "{{ $labels.mountpoint }} on {{ $labels.instance }} is read-only."
|
||
|
|
|
||
|
|
- alert: ContainerMissing
|
||
|
|
expr: time() - container_last_seen{name!=""} > 180
|
||
|
|
for: 5m
|
||
|
|
labels:
|
||
|
|
severity: warning
|
||
|
|
annotations:
|
||
|
|
summary: "Container disappeared: {{ $labels.name }}"
|
||
|
|
description: "cAdvisor has not seen the container for more than three minutes."
|
||
|
|
|
||
|
|
- alert: ContainerHighCPU
|
||
|
|
expr: sum by (name) (rate(container_cpu_usage_seconds_total{name!=""}[5m])) * 100 > 90
|
||
|
|
for: 15m
|
||
|
|
labels:
|
||
|
|
severity: warning
|
||
|
|
annotations:
|
||
|
|
summary: "Container CPU usage is high: {{ $labels.name }}"
|
||
|
|
description: "Container CPU consumption has exceeded 90% of one core for 15 minutes."
|
||
|
|
|
||
|
|
- alert: BlackboxProbeFailed
|
||
|
|
expr: probe_success == 0
|
||
|
|
for: 5m
|
||
|
|
labels:
|
||
|
|
severity: critical
|
||
|
|
annotations:
|
||
|
|
summary: "Endpoint probe failed: {{ $labels.instance }}"
|
||
|
|
description: "The endpoint has failed blackbox probes for five minutes."
|
||
|
|
|
||
|
|
- alert: CertificateExpiringSoon
|
||
|
|
expr: probe_ssl_earliest_cert_expiry - time() < 14 * 24 * 60 * 60
|
||
|
|
for: 1h
|
||
|
|
labels:
|
||
|
|
severity: warning
|
||
|
|
annotations:
|
||
|
|
summary: "TLS certificate expires within 14 days"
|
||
|
|
description: "The certificate for {{ $labels.instance }} expires soon."
|
||
|
|
|
||
|
|
- alert: ProductionBackupMissing
|
||
|
|
expr: absent(haixun_backup_last_success_timestamp_seconds) or (time() - haixun_backup_last_success_timestamp_seconds > 26 * 60 * 60)
|
||
|
|
for: 30m
|
||
|
|
labels:
|
||
|
|
severity: critical
|
||
|
|
annotations:
|
||
|
|
summary: "Production backup is missing or stale"
|
||
|
|
description: "No successful MongoDB and MinIO backup has been recorded in the last 26 hours."
|