From f856434642c72a2ad7fc5950db004f1f77cc57a2 Mon Sep 17 00:00:00 2001
From: justin <justin@liquidator.optimal-reality.com>
Date: Fri, 1 May 2026 03:33:48 -0500
Subject: [PATCH] Fix service probes: correct endpoints and permissive HTTP
 module

- Workers: use http_internal module (HTTP/1.0 SimpleHTTPServer)
- ERPNext: use /api/method/ping, accept 401/403 (still means alive)
- Listmonk: use /health not /api/health (403 without auth)
- Forgejo: port 3000 not 3030
- Dev API: probe via HTTPS public URL (blackbox can't reach Docker)
- Added http_internal blackbox module accepting HTTP/1.0 + 401/403

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
---
 monitoring/blackbox.yml   | 10 ++++++++++
 monitoring/prometheus.yml | 16 ++++++++--------
 2 files changed, 18 insertions(+), 8 deletions(-)

diff --git a/monitoring/blackbox.yml b/monitoring/blackbox.yml
index ab91b22..2ceee14 100644
--- a/monitoring/blackbox.yml
+++ b/monitoring/blackbox.yml
@@ -10,6 +10,16 @@ modules:
       tls_config:
         insecure_skip_verify: false
 
+  # Permissive module for internal services (Python SimpleHTTPServer, etc.)
+  http_internal:
+    prober: http
+    timeout: 10s
+    http:
+      valid_http_versions: ["HTTP/1.0", "HTTP/1.1", "HTTP/2.0"]
+      valid_status_codes: [200, 301, 302, 401, 403]
+      follow_redirects: false
+      preferred_ip_protocol: ip4
+
   tcp_connect:
     prober: tcp
     timeout: 5s
diff --git a/monitoring/prometheus.yml b/monitoring/prometheus.yml
index 5cf9069..d2bb381 100644
--- a/monitoring/prometheus.yml
+++ b/monitoring/prometheus.yml
@@ -75,7 +75,7 @@ scrape_configs:
       module: [http_2xx]
     static_configs:
       - targets:
-          - http://host.docker.internal:3002/api/v1/status
+          - https://api.dev.performancewest.net/api/v1/status
         labels:
           service: api
           env: dev
@@ -106,11 +106,11 @@ scrape_configs:
       - target_label: __address__
         replacement: blackbox-exporter:9115
 
-  # ── Workers (Python job server) ────────────────────────────────────
+  # ── Workers (Python job server — HTTP/1.0 SimpleHTTPServer) ────────
   - job_name: pw_workers
     metrics_path: /probe
     params:
-      module: [http_2xx]
+      module: [http_internal]
     static_configs:
       - targets:
           - http://workers:8090/health
@@ -125,14 +125,14 @@ scrape_configs:
       - target_label: __address__
         replacement: blackbox-exporter:9115
 
-  # ── ERPNext CRM ────────────────────────────────────────────────────
+  # ── ERPNext CRM (login page returns 200) ───────────────────────────
   - job_name: pw_erpnext
     metrics_path: /probe
     params:
-      module: [http_2xx]
+      module: [http_internal]
     static_configs:
       - targets:
-          - http://erpnext:8000/api/method/frappe.client.get_count?doctype=Customer
+          - http://erpnext:8000/api/method/ping
         labels:
           service: erpnext
           env: prod
@@ -170,7 +170,7 @@ scrape_configs:
       module: [http_2xx]
     static_configs:
       - targets:
-          - http://listmonk:9000/api/health
+          - http://listmonk:9000/health
         labels:
           service: listmonk
           env: prod
@@ -227,7 +227,7 @@ scrape_configs:
       module: [http_2xx]
     static_configs:
       - targets:
-          - http://host.docker.internal:3030/
+          - http://host.docker.internal:3000/
         labels:
           service: forgejo
           env: prod