Add Prometheus + Grafana + Alertmanager monitoring stack
Full observability stack with Telegram alerting: Components: - Prometheus: metrics collection, 90-day retention - Grafana: dashboards at monitoring.performancewest.net - Alertmanager: routes alerts to Telegram bot - node-exporter: OS metrics (CPU, RAM, disk, network) - cAdvisor: container metrics (CPU, memory, restarts) - postgres-exporter: PostgreSQL connection/query metrics - nginx-exporter: request rate, 5xx errors, connections - blackbox-exporter: HTTP/TCP endpoint probing + SSL cert checks Alert rules: - Service down (HTTP probe, TCP port, container missing) - Container restart loops - High CPU/memory/disk/load - PostgreSQL down or high connections - SSL cert expiring (14d warning, 3d critical) - Slow HTTP responses, high 5xx rate Blackbox probes all public endpoints: performancewest.net, api, dev, crm, lists, analytics, minio, crypto, pay Telegram alerts: critical=1h repeat, warning=6h repeat, auto-resolve notifications Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
justin
parent
97e8664cbf
commit
a4a5500bfc
13 changed files with 581 additions and 0 deletions
162
monitoring/alert_rules.yml
Normal file
162
monitoring/alert_rules.yml
Normal file
|
|
@ -0,0 +1,162 @@
|
|||
groups:
|
||||
# ══════════════════════════════════════════════════════════════════════
|
||||
# Service Down Alerts
|
||||
# ══════════════════════════════════════════════════════════════════════
|
||||
- name: service_down
|
||||
rules:
|
||||
- alert: EndpointDown
|
||||
expr: probe_success{job="blackbox_http"} == 0
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "{{ $labels.instance }} is DOWN"
|
||||
description: "HTTP probe failed for {{ $labels.instance }} for more than 2 minutes."
|
||||
|
||||
- alert: TCPPortDown
|
||||
expr: probe_success{job="blackbox_tcp"} == 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "TCP port {{ $labels.instance }} is DOWN"
|
||||
description: "TCP connection failed to {{ $labels.instance }} for more than 1 minute."
|
||||
|
||||
- alert: ContainerDown
|
||||
expr: |
|
||||
absent(container_last_seen{name=~"performancewest-(api|site|workers|erpnext|listmonk|minio|umami)-1"})
|
||||
or time() - container_last_seen{name=~"performancewest-(api|site|workers|erpnext|listmonk|minio|umami)-1"} > 60
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Container {{ $labels.name }} is DOWN"
|
||||
description: "Docker container {{ $labels.name }} has not been seen for more than 1 minute."
|
||||
|
||||
- alert: ContainerRestarting
|
||||
expr: increase(container_start_time_seconds{name=~"performancewest-.*"}[15m]) > 2
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Container {{ $labels.name }} is restart-looping"
|
||||
description: "Container {{ $labels.name }} has restarted more than 2 times in 15 minutes."
|
||||
|
||||
# ══════════════════════════════════════════════════════════════════════
|
||||
# Host Resource Alerts
|
||||
# ══════════════════════════════════════════════════════════════════════
|
||||
- name: host_resources
|
||||
rules:
|
||||
- alert: HighCPU
|
||||
expr: 100 - (avg by(instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 85
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "High CPU usage ({{ $value | printf \"%.1f\" }}%)"
|
||||
description: "CPU usage has been above 85% for 10 minutes."
|
||||
|
||||
- alert: HighMemory
|
||||
expr: (1 - node_memory_AvailableBytes / node_memory_MemTotalBytes) * 100 > 90
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "High memory usage ({{ $value | printf \"%.1f\" }}%)"
|
||||
description: "Memory usage has been above 90% for 5 minutes."
|
||||
|
||||
- alert: DiskSpaceLow
|
||||
expr: (1 - node_filesystem_avail_bytes{mountpoint="/"} / node_filesystem_size_bytes{mountpoint="/"}) * 100 > 80
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Disk usage high ({{ $value | printf \"%.1f\" }}%)"
|
||||
description: "Root filesystem is more than 80% full."
|
||||
|
||||
- alert: DiskSpaceCritical
|
||||
expr: (1 - node_filesystem_avail_bytes{mountpoint="/"} / node_filesystem_size_bytes{mountpoint="/"}) * 100 > 92
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Disk usage CRITICAL ({{ $value | printf \"%.1f\" }}%)"
|
||||
description: "Root filesystem is more than 92% full. Immediate action required."
|
||||
|
||||
- alert: HighLoadAverage
|
||||
expr: node_load15 > 8
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "High load average ({{ $value | printf \"%.1f\" }})"
|
||||
description: "15-minute load average has been above 8 for 10 minutes."
|
||||
|
||||
# ══════════════════════════════════════════════════════════════════════
|
||||
# Database Alerts
|
||||
# ══════════════════════════════════════════════════════════════════════
|
||||
- name: database
|
||||
rules:
|
||||
- alert: PostgresDown
|
||||
expr: pg_up == 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "PostgreSQL is DOWN"
|
||||
description: "PostgreSQL exporter cannot connect to the database."
|
||||
|
||||
- alert: PostgresHighConnections
|
||||
expr: pg_stat_activity_count > 80
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "High PostgreSQL connections ({{ $value }})"
|
||||
description: "PostgreSQL active connections exceeding 80."
|
||||
|
||||
# ══════════════════════════════════════════════════════════════════════
|
||||
# SSL Certificate Alerts
|
||||
# ══════════════════════════════════════════════════════════════════════
|
||||
- name: ssl
|
||||
rules:
|
||||
- alert: SSLCertExpiringSoon
|
||||
expr: probe_ssl_earliest_cert_expiry - time() < 14 * 24 * 3600
|
||||
for: 1h
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "SSL cert expiring in < 14 days for {{ $labels.instance }}"
|
||||
description: "Certificate for {{ $labels.instance }} expires in {{ $value | humanizeDuration }}."
|
||||
|
||||
- alert: SSLCertExpiryCritical
|
||||
expr: probe_ssl_earliest_cert_expiry - time() < 3 * 24 * 3600
|
||||
for: 10m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "SSL cert expiring in < 3 days for {{ $labels.instance }}"
|
||||
description: "Certificate for {{ $labels.instance }} expires in {{ $value | humanizeDuration }}. Certbot renewal may be broken."
|
||||
|
||||
# ══════════════════════════════════════════════════════════════════════
|
||||
# Response Time Alerts
|
||||
# ══════════════════════════════════════════════════════════════════════
|
||||
- name: latency
|
||||
rules:
|
||||
- alert: SlowHTTPResponse
|
||||
expr: probe_http_duration_seconds{phase="transfer", job="blackbox_http"} > 5
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Slow response from {{ $labels.instance }} ({{ $value | printf \"%.1f\" }}s)"
|
||||
description: "HTTP response time exceeds 5 seconds for {{ $labels.instance }}."
|
||||
|
||||
- alert: HighNginx5xxRate
|
||||
expr: rate(nginx_http_requests_total{status=~"5.."}[5m]) > 0.5
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "High nginx 5xx error rate"
|
||||
description: "More than 0.5 req/s returning 5xx errors."
|
||||
40
monitoring/alertmanager.yml
Normal file
40
monitoring/alertmanager.yml
Normal file
|
|
@ -0,0 +1,40 @@
|
|||
global:
|
||||
resolve_timeout: 5m
|
||||
|
||||
route:
|
||||
receiver: telegram
|
||||
group_by: [alertname, instance]
|
||||
group_wait: 30s
|
||||
group_interval: 5m
|
||||
repeat_interval: 4h
|
||||
routes:
|
||||
- match:
|
||||
severity: critical
|
||||
receiver: telegram
|
||||
repeat_interval: 1h
|
||||
- match:
|
||||
severity: warning
|
||||
receiver: telegram
|
||||
repeat_interval: 6h
|
||||
|
||||
receivers:
|
||||
- name: telegram
|
||||
telegram_configs:
|
||||
- bot_token: "${TELEGRAM_BOT_TOKEN}"
|
||||
chat_id: ${TELEGRAM_CHAT_ID}
|
||||
parse_mode: HTML
|
||||
message: |
|
||||
{{ if eq .Status "firing" }}🔴{{ else }}✅{{ end }} <b>{{ .Status | toUpper }}</b>
|
||||
{{ range .Alerts }}
|
||||
<b>{{ .Labels.alertname }}</b>
|
||||
{{ .Annotations.summary }}
|
||||
{{ if .Annotations.description }}<i>{{ .Annotations.description }}</i>{{ end }}
|
||||
{{ end }}
|
||||
<code>Server: pw-server | {{ .ExternalURL }}</code>
|
||||
|
||||
inhibit_rules:
|
||||
- source_match:
|
||||
severity: critical
|
||||
target_match:
|
||||
severity: warning
|
||||
equal: [alertname, instance]
|
||||
15
monitoring/blackbox.yml
Normal file
15
monitoring/blackbox.yml
Normal file
|
|
@ -0,0 +1,15 @@
|
|||
modules:
|
||||
http_2xx:
|
||||
prober: http
|
||||
timeout: 10s
|
||||
http:
|
||||
valid_http_versions: ["HTTP/1.1", "HTTP/2.0"]
|
||||
valid_status_codes: [200, 301, 302]
|
||||
follow_redirects: true
|
||||
preferred_ip_protocol: ip4
|
||||
tls_config:
|
||||
insecure_skip_verify: false
|
||||
|
||||
tcp_connect:
|
||||
prober: tcp
|
||||
timeout: 5s
|
||||
9
monitoring/grafana-datasources.yml
Normal file
9
monitoring/grafana-datasources.yml
Normal file
|
|
@ -0,0 +1,9 @@
|
|||
apiVersion: 1
|
||||
|
||||
datasources:
|
||||
- name: Prometheus
|
||||
type: prometheus
|
||||
access: proxy
|
||||
url: http://prometheus:9090
|
||||
isDefault: true
|
||||
editable: false
|
||||
83
monitoring/prometheus.yml
Normal file
83
monitoring/prometheus.yml
Normal file
|
|
@ -0,0 +1,83 @@
|
|||
global:
|
||||
scrape_interval: 30s
|
||||
evaluation_interval: 30s
|
||||
|
||||
rule_files:
|
||||
- /etc/prometheus/alert_rules.yml
|
||||
|
||||
alerting:
|
||||
alertmanagers:
|
||||
- static_configs:
|
||||
- targets:
|
||||
- alertmanager:9093
|
||||
|
||||
scrape_configs:
|
||||
# ── Prometheus self-monitoring ──────────────────────────────────────
|
||||
- job_name: prometheus
|
||||
static_configs:
|
||||
- targets: ["localhost:9090"]
|
||||
|
||||
# ── Host OS metrics (node_exporter) ────────────────────────────────
|
||||
- job_name: node
|
||||
static_configs:
|
||||
- targets: ["node-exporter:9100"]
|
||||
|
||||
# ── Docker container metrics (cAdvisor) ────────────────────────────
|
||||
- job_name: cadvisor
|
||||
static_configs:
|
||||
- targets: ["cadvisor:8080"]
|
||||
|
||||
# ── PostgreSQL (prod) ──────────────────────────────────────────────
|
||||
- job_name: postgres_prod
|
||||
static_configs:
|
||||
- targets: ["postgres-exporter:9187"]
|
||||
labels:
|
||||
instance: prod
|
||||
|
||||
# ── nginx ──────────────────────────────────────────────────────────
|
||||
- job_name: nginx
|
||||
static_configs:
|
||||
- targets: ["nginx-exporter:9113"]
|
||||
|
||||
# ── Blackbox probes (HTTP endpoint monitoring) ─────────────────────
|
||||
- job_name: blackbox_http
|
||||
metrics_path: /probe
|
||||
params:
|
||||
module: [http_2xx]
|
||||
static_configs:
|
||||
- targets:
|
||||
- https://performancewest.net
|
||||
- https://api.performancewest.net/api/v1/fcc/search?q=test
|
||||
- https://dev.performancewest.net
|
||||
- https://api.dev.performancewest.net/api/v1/fcc/search?q=test
|
||||
- https://crm.performancewest.net
|
||||
- https://lists.performancewest.net
|
||||
- https://analytics.performancewest.net
|
||||
- https://minio.performancewest.net/minio/health/live
|
||||
- https://crypto.performancewest.net
|
||||
- https://pay.performancewest.net
|
||||
relabel_configs:
|
||||
- source_labels: [__address__]
|
||||
target_label: __param_target
|
||||
- source_labels: [__param_target]
|
||||
target_label: instance
|
||||
- target_label: __address__
|
||||
replacement: blackbox-exporter:9115
|
||||
|
||||
# ── Blackbox TCP probes (port monitoring) ──────────────────────────
|
||||
- job_name: blackbox_tcp
|
||||
metrics_path: /probe
|
||||
params:
|
||||
module: [tcp_connect]
|
||||
static_configs:
|
||||
- targets:
|
||||
- api-postgres:5432
|
||||
- erpnext-mariadb:3306
|
||||
- erpnext-redis:6379
|
||||
relabel_configs:
|
||||
- source_labels: [__address__]
|
||||
target_label: __param_target
|
||||
- source_labels: [__param_target]
|
||||
target_label: instance
|
||||
- target_label: __address__
|
||||
replacement: blackbox-exporter:9115
|
||||
Loading…
Add table
Add a link
Reference in a new issue