diff --git a/monitoring/alert_rules.yml b/monitoring/alert_rules.yml index 64faa13..2debefb 100644 --- a/monitoring/alert_rules.yml +++ b/monitoring/alert_rules.yml @@ -142,13 +142,26 @@ groups: description: "{{ $labels.name }} has restarted more than 2 times in 15 minutes." - alert: ContainerHighCPU - expr: rate(container_cpu_usage_seconds_total{name=~"performancewest-.*"}[5m]) * 100 > 80 - for: 10m + expr: | + ( + sum by (instance, name) ( + rate(container_cpu_usage_seconds_total{name=~"performancewest-.*"}[5m]) + ) + / + sum by (instance, name) ( + container_spec_cpu_quota{name=~"performancewest-.*"} + / container_spec_cpu_period{name=~"performancewest-.*"} + ) + * 100 > 80 + ) + and on (instance, name) + sum by (instance, name) (container_spec_cpu_quota{name=~"performancewest-.*"}) > 0 + for: 30m labels: severity: warning annotations: summary: "Container {{ $labels.name }} high CPU ({{ $value | printf \"%.0f\" }}%)" - description: "Container CPU usage above 80% for 10 minutes." + description: "Container CPU usage above 80% of its CPU quota for 30 minutes." - alert: ContainerHighMemory expr: |