diff --git a/monitoring/alert_rules.yml b/monitoring/alert_rules.yml index 2debefb..5baa112 100644 --- a/monitoring/alert_rules.yml +++ b/monitoring/alert_rules.yml @@ -142,26 +142,25 @@ groups: description: "{{ $labels.name }} has restarted more than 2 times in 15 minutes." - alert: ContainerHighCPU + # Our containers run WITHOUT a CPU quota (CpuQuota=0), so the old + # `rate(...) * 100 > 80` treated 1 full core = 100% and fired every time + # forgejo/erpnext briefly used ~0.8 of a core — on a 12-core host that is + # only ~7% of the machine, pure noise. Normalize against the whole host + # (machine_cpu_cores) so the number means "share of the entire server this + # one container is eating." A single unlimited container sustained above + # half the host for 30m is a genuine runaway worth paging on. expr: | - ( - sum by (instance, name) ( - rate(container_cpu_usage_seconds_total{name=~"performancewest-.*"}[5m]) - ) - / - sum by (instance, name) ( - container_spec_cpu_quota{name=~"performancewest-.*"} - / container_spec_cpu_period{name=~"performancewest-.*"} - ) - * 100 > 80 + sum by (instance, name) ( + rate(container_cpu_usage_seconds_total{name=~"performancewest-.*"}[5m]) ) - and on (instance, name) - sum by (instance, name) (container_spec_cpu_quota{name=~"performancewest-.*"}) > 0 + / scalar(max(machine_cpu_cores)) + * 100 > 50 for: 30m labels: severity: warning annotations: - summary: "Container {{ $labels.name }} high CPU ({{ $value | printf \"%.0f\" }}%)" - description: "Container CPU usage above 80% of its CPU quota for 30 minutes." + summary: "Container {{ $labels.name }} high CPU ({{ $value | printf \"%.0f\" }}% of host)" + description: "Container using more than 50% of the whole host's CPU for 30 minutes." - alert: ContainerHighMemory expr: |