Tune container high CPU alert

monitoring/alert_rules.yml

@@ -142,13 +142,26 @@ groups:
           description: "{{ $labels.name }} has restarted more than 2 times in 15 minutes."
 
       - alert: ContainerHighCPU
-        expr: rate(container_cpu_usage_seconds_total{name=~"performancewest-.*"}[5m]) * 100 > 80
-        for: 10m
+        expr: |
+          (
+            sum by (instance, name) (
+              rate(container_cpu_usage_seconds_total{name=~"performancewest-.*"}[5m])
+            )
+            /
+            sum by (instance, name) (
+              container_spec_cpu_quota{name=~"performancewest-.*"}
+              / container_spec_cpu_period{name=~"performancewest-.*"}
+            )
+            * 100 > 80
+          )
+          and on (instance, name)
+          sum by (instance, name) (container_spec_cpu_quota{name=~"performancewest-.*"}) > 0
+        for: 30m
         labels:
           severity: warning
         annotations:
           summary: "Container {{ $labels.name }} high CPU ({{ $value | printf \"%.0f\" }}%)"
-          description: "Container CPU usage above 80% for 10 minutes."
+          description: "Container CPU usage above 80% of its CPU quota for 30 minutes."
 
       - alert: ContainerHighMemory
         expr: |