From 15f5c267e70c8eb0826c815d1d068d78729902be Mon Sep 17 00:00:00 2001 From: justin Date: Fri, 1 May 2026 03:43:42 -0500 Subject: [PATCH] Fix dashboard stale series + enable Prometheus admin API Dashboard queries now use max() to pick UP value when old stale probe targets coexist with new ones. Prometheus admin API enabled for future TSDB cleanup of stale series. Co-Authored-By: Claude Opus 4.6 (1M context) --- docker-compose.yml | 1 + monitoring/pw-services-dashboard.json | 18 +++++++++--------- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index f13733a..d45f2ba 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -298,6 +298,7 @@ services: - --config.file=/etc/prometheus/prometheus.yml - --storage.tsdb.retention.time=90d - --web.enable-lifecycle + - --web.enable-admin-api extra_hosts: - "host.docker.internal:host-gateway" restart: unless-stopped diff --git a/monitoring/pw-services-dashboard.json b/monitoring/pw-services-dashboard.json index c30b62a..03a6ed5 100644 --- a/monitoring/pw-services-dashboard.json +++ b/monitoring/pw-services-dashboard.json @@ -24,15 +24,15 @@ }, "options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "graphMode": "none", "colorMode": "background", "textMode": "auto" }, "targets": [ - { "expr": "probe_success{job='pw_api_prod'}", "legendFormat": "API", "datasource": { "type": "prometheus" } }, - { "expr": "probe_success{job='pw_site_prod'}", "legendFormat": "Site", "datasource": { "type": "prometheus" } }, - { "expr": "probe_success{job='pw_workers'}", "legendFormat": "Workers", "datasource": { "type": "prometheus" } }, - { "expr": "probe_success{job='pw_erpnext'}", "legendFormat": "ERPNext", "datasource": { "type": "prometheus" } }, - { "expr": "probe_success{job='pw_minio'}", "legendFormat": "MinIO", "datasource": { "type": "prometheus" } }, - { "expr": "probe_success{job='pw_listmonk'}", "legendFormat": "Listmonk", "datasource": { "type": "prometheus" } }, - { "expr": "probe_success{job='pw_ollama'}", "legendFormat": "Ollama", "datasource": { "type": "prometheus" } }, - { "expr": "probe_success{job='pw_umami'}", "legendFormat": "Umami", "datasource": { "type": "prometheus" } }, - { "expr": "probe_success{job='pw_forgejo'}", "legendFormat": "Forgejo", "datasource": { "type": "prometheus" } }, + { "expr": "max(probe_success{job='pw_api_prod'})", "legendFormat": "API", "datasource": { "type": "prometheus" } }, + { "expr": "max(probe_success{job='pw_site_prod'})", "legendFormat": "Site", "datasource": { "type": "prometheus" } }, + { "expr": "max(probe_success{job='pw_workers'})", "legendFormat": "Workers", "datasource": { "type": "prometheus" } }, + { "expr": "max(probe_success{job='pw_erpnext'})", "legendFormat": "ERPNext", "datasource": { "type": "prometheus" } }, + { "expr": "max(probe_success{job='pw_minio'})", "legendFormat": "MinIO", "datasource": { "type": "prometheus" } }, + { "expr": "max(probe_success{job='pw_listmonk'})", "legendFormat": "Listmonk", "datasource": { "type": "prometheus" } }, + { "expr": "max(probe_success{job='pw_ollama'})", "legendFormat": "Ollama", "datasource": { "type": "prometheus" } }, + { "expr": "max(probe_success{job='pw_umami'})", "legendFormat": "Umami", "datasource": { "type": "prometheus" } }, + { "expr": "max(probe_success{job='pw_forgejo'})", "legendFormat": "Forgejo", "datasource": { "type": "prometheus" } }, { "expr": "pg_up", "legendFormat": "PostgreSQL", "datasource": { "type": "prometheus" } } ] },