From 7c39a858ccfb56807d96aaf878e3b4028b88c0b9 Mon Sep 17 00:00:00 2001
From: justin <justin@liquidator.optimal-reality.com>
Date: Mon, 8 Jun 2026 21:06:41 -0500
Subject: [PATCH] monitoring: daily warmup IP-reputation Telegram alert

End-of-day (20:00 Central) check of campaign deliverability across both sending
pools (main out05-09 + healthcare hcout). Sends a Telegram alert ONLY when there
is a reputation problem -- delivery below 65% or a spam/policy-block (550-5.7.1)
spike above 150/day -- so healthy days stay silent. Reuses the existing
TELEGRAM_BOT_TOKEN/CHAT_ID from /opt/performancewest/.env. Logs every run to
/var/log/pw-warmup-healthcheck.log for history. Excludes internal/probe noise so
the delivery figure reflects real external recipients.
---
 infra/cron/pw-warmup-tg-alert          |  7 ++++
 infra/monitoring/pw-warmup-tg-alert.sh | 48 ++++++++++++++++++++++++++
 2 files changed, 55 insertions(+)
 create mode 100644 infra/cron/pw-warmup-tg-alert
 create mode 100755 infra/monitoring/pw-warmup-tg-alert.sh

diff --git a/infra/cron/pw-warmup-tg-alert b/infra/cron/pw-warmup-tg-alert
new file mode 100644
index 0000000..420f803
--- /dev/null
+++ b/infra/cron/pw-warmup-tg-alert
@@ -0,0 +1,7 @@
+# Daily warmup IP-reputation check + Telegram alert. Runs 20:00 Central (after
+# the day's sends complete), alerts ONLY on a problem (delivery below 65% or
+# >150 spam/policy 550-5.7.1 blocks); healthy days stay silent. Logs every run
+# to /var/log/pw-warmup-healthcheck.log. Script: infra/monitoring/pw-warmup-tg-alert.sh
+# -> /usr/local/bin/pw-warmup-tg-alert. Reads TELEGRAM_BOT_TOKEN/CHAT_ID from
+# /opt/performancewest/.env.
+0 20 * * * deploy /usr/local/bin/pw-warmup-tg-alert >> /var/log/pw-warmup-healthcheck.log 2>&1
diff --git a/infra/monitoring/pw-warmup-tg-alert.sh b/infra/monitoring/pw-warmup-tg-alert.sh
new file mode 100755
index 0000000..9a073a6
--- /dev/null
+++ b/infra/monitoring/pw-warmup-tg-alert.sh
@@ -0,0 +1,48 @@
+#!/bin/bash
+# Daily warmup IP-reputation check. Sends a Telegram alert ONLY if there is a
+# reputation problem (silent on healthy days). Creds from /opt/performancewest/.env.
+set -uo pipefail
+LOG=/var/log/mail.log
+TODAY=$(date '+%b %d')
+REPORT=/var/log/pw-warmup-healthcheck.log
+MIN_DELIVERY=65
+MAX_SPAMBLOCK=150
+MIN_SENT=50
+BOT=$(grep -E '^TELEGRAM_BOT_TOKEN=' /opt/performancewest/.env | head -1 | cut -d= -f2-)
+CHAT=$(grep -E '^TELEGRAM_CHAT_ID=' /opt/performancewest/.env | head -1 | cut -d= -f2-)
+tg() {
+  [ -n "$BOT" ] && [ -n "$CHAT" ] || return 0
+  curl -s --max-time 10 "https://api.telegram.org/bot${BOT}/sendMessage" --data-urlencode "chat_id=${CHAT}" --data-urlencode "text=$1" >/dev/null 2>&1
+}
+mlog() { sudo grep -E "$1" "$LOG" 2>/dev/null | grep "^$TODAY"; }
+EXT='to=<[^>]*@(performancewest|carrierone|perfwest)'
+NOISE='probe|example.org|rt-252'
+MSENT=$(mlog 'out0[5-9]/smtp' | grep 'status=sent' | grep -vE "$EXT" | grep -viE "$NOISE" | wc -l)
+MBOUNCE=$(mlog 'out0[5-9]/smtp' | grep 'status=bounced' | grep -vE "$EXT" | grep -viE "$NOISE" | wc -l)
+MSPAM=$(mlog 'out0[5-9]/smtp' | grep 'status=bounced' | grep -c '550-5.7.1')
+MTOTAL=$((MSENT + MBOUNCE)); MDEL=0
+[ "$MTOTAL" -gt 0 ] && MDEL=$(python3 -c "print(round(100*$MSENT/$MTOTAL))")
+HSENT=$(mlog 'hcout[0-9]/smtp' | grep -c 'status=sent')
+HBOUNCE=$(mlog 'hcout[0-9]/smtp' | grep -c 'status=bounced')
+HSPAM=$(mlog 'hcout[0-9]/smtp' | grep 'status=bounced' | grep -c '550-5.7.1')
+HTOTAL=$((HSENT + HBOUNCE)); HDEL=0
+[ "$HTOTAL" -gt 0 ] && HDEL=$(python3 -c "print(round(100*$HSENT/$HTOTAL))")
+PROBLEMS=''
+if [ "$MSENT" -ge "$MIN_SENT" ]; then
+  [ "$MDEL" -lt "$MIN_DELIVERY" ] && PROBLEMS="${PROBLEMS}- Main pool delivery ${MDEL}% (below ${MIN_DELIVERY}%)\n"
+  [ "$MSPAM" -gt "$MAX_SPAMBLOCK" ] && PROBLEMS="${PROBLEMS}- Main pool spam/policy blocks: ${MSPAM}\n"
+fi
+if [ "$HSENT" -ge "$MIN_SENT" ]; then
+  [ "$HDEL" -lt "$MIN_DELIVERY" ] && PROBLEMS="${PROBLEMS}- HC stream delivery ${HDEL}%\n"
+  [ "$HSPAM" -gt "$MAX_SPAMBLOCK" ] && PROBLEMS="${PROBLEMS}- HC stream spam/policy blocks: ${HSPAM}\n"
+fi
+{
+  echo "==== TG WARMUP CHECK $(date) ===="
+  echo "MAIN: sent=$MSENT bounced=$MBOUNCE delivery=${MDEL}% spamblock=$MSPAM"
+  echo "HC:   sent=$HSENT bounced=$HBOUNCE delivery=${HDEL}% spamblock=$HSPAM"
+  echo "problems: ${PROBLEMS:-none}"
+} >> "$REPORT" 2>&1
+if [ -n "$PROBLEMS" ]; then
+  MSG=$(printf '⚠️ Performance West IP reputation alert (%s)\n\nMain pool: %d%% delivery, %d sent, %d bounced, %d spam-blocks\nHC stream: %d%% delivery, %d sent, %d spam-blocks\n\nIssues:\n%b' "$TODAY" "$MDEL" "$MSENT" "$MBOUNCE" "$MSPAM" "$HDEL" "$HSENT" "$HSPAM" "$PROBLEMS")
+  tg "$MSG"
+fi