899b880e7f
The FMCSA census was a one-time snapshot (last loaded ~May 30) with NO refresh timer -- carriers newly falling out of MCS-150/UCR compliance were never picked up. New scripts/workers/fmcsa_source_refresh.py orchestrates the full pipeline (census download -> enrichment -> deficiency flag -> verify new emails -> MX-tag new) and runs weekly via cron pw-fmcsa-refresh (Sun 09:00 UTC), codified in the mail-pipeline Ansible role. Idempotent + incremental: the census upsert preserves email_verified / listmonk_sent_at / deficiency_flags, so existing carriers keep their send state and only census fields refresh; new DOTs flow into verification then campaigns. A carrier who refiled gets a fresh mcs150_parsed, so the builder's overdue WHERE clause stops targeting them automatically. Verify is capped per run (20k) so it never stalls on millions of rows. (Healthcare already auto-catches newly-revalidation-overdue providers within its 63k institutional pool via pw-hc-refresh Mon/Wed/Fri.)
120 lines
4.5 KiB
YAML
120 lines
4.5 KiB
YAML
---
|
|
# mail-pipeline role
|
|
#
|
|
# Codifies the outbound email-campaign pipeline that previously lived ONLY on
|
|
# the host (none of this was in IaC before -- a fresh rebuild would have silently
|
|
# shipped NO campaigns, NO IP warmup/ramp, and NO bounce processing):
|
|
#
|
|
# - /etc/cron.d/pw-* daily campaign builders + IP-warmup/ramp drivers
|
|
# - /usr/local/bin/pw-* warmup/ramp/healthcheck helper scripts
|
|
# - /usr/local/bin/postfix-*-bounce-notify.sh bounce watchers
|
|
# - pw-bounce-watcher / pw-hc-bounce-watcher systemd watcher services
|
|
#
|
|
# The campaign BUILDER logic (scripts/build_*.py) is synced with the app/workers
|
|
# code; this role only deploys the host-level glue (cron + helper scripts +
|
|
# services). The OpenDKIM signing + mail.log logrotate live in the `mail` role.
|
|
|
|
# ── log + state dirs ────────────────────────────────────────────────────────
|
|
# The deploy user CANNOT write /var/log, so the deploy-owned cron jobs log to
|
|
# /opt/performancewest/logs. A missing dir makes the `>>` redirect fail before
|
|
# the command runs (cron then mails the error to deploy@ -> self-bounce).
|
|
- name: Ensure deploy-owned cron log directory exists
|
|
ansible.builtin.file:
|
|
path: "{{ project_dir }}/logs"
|
|
state: directory
|
|
owner: "{{ deploy_user }}"
|
|
group: "{{ deploy_user }}"
|
|
mode: "0775"
|
|
|
|
# ── warmup / ramp helper scripts (run as root: edit main.cf, restart cntrs) ──
|
|
- name: Deploy mail warmup/ramp/healthcheck helper scripts
|
|
ansible.builtin.copy:
|
|
src: "{{ playbook_dir }}/../../{{ item.src }}"
|
|
dest: "/usr/local/bin/{{ item.dest }}"
|
|
owner: root
|
|
group: root
|
|
mode: "0755"
|
|
loop:
|
|
- { src: "infra/postfix/pw-mta-warmup.sh", dest: "pw-mta-warmup" }
|
|
- { src: "infra/postfix/pw-listmonk-rampcap.sh", dest: "pw-listmonk-rampcap" }
|
|
- { src: "infra/postfix/pw-hc-rampcap.sh", dest: "pw-hc-rampcap" }
|
|
- { src: "infra/monitoring/pw-warmup-tg-alert.sh", dest: "pw-warmup-tg-alert" }
|
|
|
|
# ── bounce watchers (tail mail.log -> Listmonk bounce webhook) ──────────────
|
|
- name: Deploy bounce-watcher scripts
|
|
ansible.builtin.copy:
|
|
src: "{{ playbook_dir }}/../../{{ item.src }}"
|
|
dest: "/usr/local/bin/{{ item.dest }}"
|
|
owner: root
|
|
group: root
|
|
mode: "0755"
|
|
loop:
|
|
- { src: "scripts/bounce-watcher.sh", dest: "postfix-bounce-notify.sh" }
|
|
- { src: "scripts/hc-bounce-watcher.sh", dest: "postfix-hc-bounce-notify.sh" }
|
|
notify:
|
|
- Restart pw-bounce-watcher
|
|
- Restart pw-hc-bounce-watcher
|
|
|
|
- name: Deploy bounce-watcher systemd units
|
|
ansible.builtin.copy:
|
|
src: "{{ playbook_dir }}/../../infra/systemd/{{ item }}"
|
|
dest: "/etc/systemd/system/{{ item }}"
|
|
owner: root
|
|
group: root
|
|
mode: "0644"
|
|
loop:
|
|
- pw-bounce-watcher.service
|
|
- pw-hc-bounce-watcher.service
|
|
notify:
|
|
- Reload systemd
|
|
- Restart pw-bounce-watcher
|
|
- Restart pw-hc-bounce-watcher
|
|
|
|
- name: Enable + start bounce-watcher services
|
|
ansible.builtin.systemd:
|
|
name: "{{ item }}"
|
|
enabled: true
|
|
state: started
|
|
daemon_reload: true
|
|
loop:
|
|
- pw-bounce-watcher.service
|
|
- pw-hc-bounce-watcher.service
|
|
|
|
# ── listmonk bounce-sync poller (host python, every 5 min via root crontab) ──
|
|
- name: Deploy listmonk bounce-sync poller
|
|
ansible.builtin.copy:
|
|
src: "{{ playbook_dir }}/../../scripts/listmonk-bounce-sync.py"
|
|
dest: /usr/local/bin/listmonk-bounce-sync.py
|
|
owner: root
|
|
group: root
|
|
mode: "0755"
|
|
|
|
- name: Schedule listmonk bounce-sync (root crontab, every 5 min)
|
|
ansible.builtin.cron:
|
|
name: listmonk-bounce-sync
|
|
minute: "*/5"
|
|
job: "/usr/bin/python3 /usr/local/bin/listmonk-bounce-sync.py >> /var/log/bounce-sync.log 2>&1"
|
|
|
|
# ── campaign + warmup cron.d files ──────────────────────────────────────────
|
|
# These reference scripts/ in {{ project_dir }} and the docker compose stack, so
|
|
# they are deployed verbatim from infra/cron/ (the canonical, reviewed copies).
|
|
- name: Deploy campaign + warmup cron.d files
|
|
ansible.builtin.copy:
|
|
src: "{{ playbook_dir }}/../../infra/cron/{{ item }}"
|
|
dest: "/etc/cron.d/{{ item }}"
|
|
owner: root
|
|
group: root
|
|
mode: "0644"
|
|
loop:
|
|
- pw-trucking-campaign-builder
|
|
- pw-ifta-campaign
|
|
- pw-ucr-campaign
|
|
- pw-hc-campaign
|
|
- pw-hc-nppes
|
|
- pw-hc-refresh
|
|
- pw-fmcsa-refresh
|
|
- pw-mta-warmup
|
|
- pw-listmonk-rampcap
|
|
- pw-hc-rampcap
|
|
- pw-ip-rehab
|
|
- pw-warmup-tg-alert
|