justin/new-site
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Fix ContainerHighCPU: normalize to host cores (containers are unquota'd)

Browse source
This commit is contained in:
justin 2026-07-02 15:19:27 -05:00
parent e722350af1
commit 5ae0483074
1 changed files with 13 additions and 14 deletions
Download patch file Download diff file Expand all files Collapse all files

27
monitoring/alert_rules.yml
View file

@ -142,26 +142,25 @@ groups:
description: "{{ $labels.name }} has restarted more than 2 times in 15 minutes."
- alert: ContainerHighCPU
# Our containers run WITHOUT a CPU quota (CpuQuota=0), so the old
# `rate(...) * 100 > 80` treated 1 full core = 100% and fired every time
# forgejo/erpnext briefly used ~0.8 of a core — on a 12-core host that is
# only ~7% of the machine, pure noise. Normalize against the whole host
# (machine_cpu_cores) so the number means "share of the entire server this
# one container is eating." A single unlimited container sustained above
# half the host for 30m is a genuine runaway worth paging on.
expr: |
(
sum by (instance, name) (
rate(container_cpu_usage_seconds_total{name=~"performancewest-.*"}[5m])
)
/
sum by (instance, name) (
container_spec_cpu_quota{name=~"performancewest-.*"}
/ container_spec_cpu_period{name=~"performancewest-.*"}
)
* 100 > 80
sum by (instance, name) (
rate(container_cpu_usage_seconds_total{name=~"performancewest-.*"}[5m])
)
and on (instance, name)
sum by (instance, name) (container_spec_cpu_quota{name=~"performancewest-.*"}) > 0
/ scalar(max(machine_cpu_cores))
* 100 > 50
for: 30m
labels:
severity: warning
annotations:
summary: "Container {{ $labels.name }} high CPU ({{ $value | printf \"%.0f\" }}%)"
description: "Container CPU usage above 80% of its CPU quota for 30 minutes."
summary: "Container {{ $labels.name }} high CPU ({{ $value | printf \"%.0f\" }}% of host)"
description: "Container using more than 50% of the whole host's CPU for 30 minutes."
- alert: ContainerHighMemory
expr: |