0b7a35a58e
- build_trucking_campaigns.py: nightly script that creates 8 Listmonk campaigns per day (4 TZ x 2 types: MCS-150 overdue 2k/TZ, inactive USDOT 1k/TZ) at 4AM ET / 5AM ET (CT) / 6AM ET (MT) / 7AM ET (PT). Deduplicates via listmonk_sent_at column. - migration 083: add listmonk_sent_at + listmonk_campaign_type to fmcsa_carriers - email_verifier.py: bump max_workers from 5 to 20 for 4x faster throughput - cron: daily pw-trucking-campaigns at 08:00 UTC (3 AM EST) Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
210 lines
8.7 KiB
YAML
210 lines
8.7 KiB
YAML
---
|
||
# worker-crons defaults
|
||
#
|
||
# Each entry in `worker_crons` deploys a systemd .service + .timer pair.
|
||
# Times use systemd OnCalendar syntax (see `man systemd.time`).
|
||
#
|
||
# The service runs inside the workers container via
|
||
# docker compose exec -T workers python -m <module>
|
||
#
|
||
# Why systemd timers not crond: timers log to journald (queryable with
|
||
# `journalctl -u <name>.service`), support OnBootSec/Persistent for catch-up
|
||
# after a downed host, and inherit the docker daemon restart policy
|
||
# without additional tooling.
|
||
|
||
project_dir: /opt/performancewest
|
||
|
||
worker_crons:
|
||
# USF quarterly factor — polls USAC daily at 09:00 Chicago local.
|
||
- name: pw-usf-factor-monitor
|
||
description: Poll USAC for new USF quarterly contribution factor
|
||
module: scripts.workers.usf_factor_monitor
|
||
on_calendar: "*-*-* 14:00:00 UTC" # 09:00 CT
|
||
persistent: true
|
||
|
||
# De minimis factor presence — 3am daily, alerts if missing.
|
||
- name: pw-deminimis-factor-check
|
||
description: Alert if fcc_deminimis_factors missing for current/next year
|
||
module: scripts.workers.deminimis_factor_check
|
||
on_calendar: "*-*-* 03:00:00 UTC"
|
||
persistent: true
|
||
|
||
# Cold wallet sweep — every 30 min.
|
||
- name: pw-cold-wallet-sweep
|
||
description: Sweep SHKeeper hot-wallet excess to cold wallet
|
||
module: scripts.workers.cold_wallet_sweeper
|
||
on_calendar: "*-*-* *:00,30:00 UTC"
|
||
persistent: false
|
||
|
||
# Crypto payment worker — every 60s while we have pending jobs.
|
||
- name: pw-crypto-payment-worker
|
||
description: Advance crypto_payment_jobs state machine
|
||
module: scripts.workers.crypto_payment_worker
|
||
on_calendar: "*-*-* *:*:00 UTC"
|
||
persistent: false
|
||
|
||
# Relay deposit monitor — every 5 min, polls IMAP for Relay alerts.
|
||
- name: pw-relay-deposit-monitor
|
||
description: Parse Relay email alerts into relay_deposits
|
||
module: scripts.workers.relay_deposit_monitor
|
||
on_calendar: "*-*-* *:00/5:00 UTC"
|
||
persistent: false
|
||
|
||
# Commission worker — 02:00 daily (flips 14-day-eligible rows to eligible).
|
||
- name: pw-commission-worker
|
||
description: Flip commission_ledger rows to eligible after 14-day holdback
|
||
module: scripts.workers.commission_worker
|
||
on_calendar: "*-*-* 02:00:00 UTC"
|
||
persistent: true
|
||
|
||
# Renewal worker — 04:00 daily.
|
||
- name: pw-renewal-worker
|
||
description: Send renewal reminders + auto-renew RA/annual report/CDR
|
||
module: scripts.workers.renewal_worker
|
||
on_calendar: "*-*-* 04:00:00 UTC"
|
||
persistent: true
|
||
|
||
# CDR retention sweeper — 05:00 daily (purges past-retention CDR).
|
||
- name: pw-cdr-retention
|
||
description: Purge CDR older than the per-profile retention window
|
||
module: scripts.workers.cdr_retention_sweeper
|
||
on_calendar: "*-*-* 05:00:00 UTC"
|
||
persistent: true
|
||
|
||
# CDR unlock nudge — 10:00 daily (emails customers with locked studies).
|
||
- name: pw-cdr-unlock-nudge
|
||
description: Nudge customers whose CDR study is locked behind paywall
|
||
module: scripts.workers.cdr_unlock_nudge
|
||
on_calendar: "*-*-* 15:00:00 UTC" # 10:00 CT
|
||
persistent: true
|
||
|
||
# Payment reminder — 11:00 daily.
|
||
- name: pw-payment-reminder
|
||
description: Remind customers with unpaid compliance orders
|
||
module: scripts.workers.payment_reminder
|
||
on_calendar: "*-*-* 16:00:00 UTC" # 11:00 CT
|
||
persistent: true
|
||
|
||
# RMD removed scraper — weekly, Wednesday 08:00 (tracks FCC RMD removals).
|
||
- name: pw-fcc-rmd-removed
|
||
description: Scrape FCC public list of RMD-removed carriers
|
||
module: scripts.workers.fcc_rmd_removed_scraper
|
||
on_calendar: "Wed *-*-* 13:00:00 UTC"
|
||
persistent: true
|
||
|
||
# Client email processor — every 15 min, IMAPs regulatory mailboxes for
|
||
# CRTC pipeline events (incoming agency replies, domain activation, etc.).
|
||
- name: pw-client-email-processor
|
||
description: Parse regulatory mailbox replies for CRTC pipeline advancement
|
||
module: scripts.workers.client_email_processor
|
||
on_calendar: "*-*-* *:00/15:00 UTC"
|
||
persistent: false
|
||
|
||
# AMB location scraper — daily 06:00 UTC, pulls current per-mailbox pricing
|
||
# and sold-out status for the CRTC AMB picker.
|
||
- name: pw-amb-location-scraper
|
||
description: Scrape current Anytime Mailbox location + pricing catalog
|
||
module: scripts.workers.amb_location_scraper
|
||
on_calendar: "*-*-* 06:00:00 UTC"
|
||
persistent: true
|
||
|
||
# Compliance alert mailing list — weekly Sunday 06:00 UTC.
|
||
# Queries local DB for carriers behind on filings and upserts to Listmonk.
|
||
- name: pw-compliance-alert-list
|
||
description: Update Listmonk compliance alert mailing list from local FCC data
|
||
module: scripts.workers.compliance_alert_list
|
||
on_calendar: "Sun *-*-* 06:00:00 UTC"
|
||
persistent: true
|
||
|
||
# Entity cache refresh — daily 07:00 UTC (2am CT).
|
||
# Downloads business entity data from state Socrata APIs (CO, IA, CT, OR, NY)
|
||
# into entity_cache for corporation status checks.
|
||
- name: pw-entity-cache-refresh
|
||
description: Refresh entity_cache from state SOS open data portals
|
||
module: scripts.formation.bulk_download --all
|
||
on_calendar: "*-*-* 07:00:00 UTC"
|
||
persistent: true
|
||
|
||
# Multi-state entity scraper — daily 08:00 UTC (3am CT).
|
||
# Scrapes state SOS portals via Playwright for states without bulk APIs.
|
||
# Covers 44 states (excludes CO, IA, CT, OR, NY, FL which use bulk data).
|
||
- name: pw-entity-scraper
|
||
description: Scrape business entities from state SOS portals via Playwright
|
||
module: scripts.workers.entity_scraper --all
|
||
on_calendar: "*-*-* 08:00:00 UTC"
|
||
persistent: true
|
||
|
||
# Post-completion emails — every 15 minutes.
|
||
# Sends completion confirmations + 24h follow-up (survey + referral).
|
||
- name: pw-completion-emails
|
||
description: Send post-completion and follow-up emails
|
||
module: scripts.workers.completion_emails
|
||
on_calendar: "*-*-* *:0/15:00 UTC"
|
||
persistent: false
|
||
|
||
# Florida entity downloader — daily 07:30 UTC.
|
||
# Downloads daily diffs from FL Sunbiz SFTP (free public access).
|
||
- name: pw-fl-entity-downloader
|
||
description: Download Florida Sunbiz corporation data via SFTP
|
||
module: scripts.workers.fl_entity_downloader --daily
|
||
on_calendar: "*-*-* 07:30:00 UTC"
|
||
persistent: true
|
||
|
||
# RMD email scraper — daily 09:00 UTC (4am CT).
|
||
# Scrapes contact emails from FCC ServiceNow SP API for RMD carriers.
|
||
- name: pw-rmd-email-scraper
|
||
description: Scrape RMD contact emails from FCC ServiceNow portal
|
||
module: scripts.workers.fcc_rmd_scraper --phase scrape --limit 500
|
||
on_calendar: "*-*-* 09:00:00 UTC"
|
||
persistent: true
|
||
|
||
# RMD filing auditor — weekly Saturday 10:00 UTC (5am CT).
|
||
# Audits RMD filings for 2026 compliance deficiencies (PDF analysis).
|
||
- name: pw-rmd-auditor
|
||
description: Audit RMD filings for 2026 compliance deficiencies
|
||
module: scripts.workers.fcc_rmd_auditor --batch --year 2026 --no-ollama
|
||
on_calendar: "Sat *-*-* 10:00:00 UTC"
|
||
persistent: true
|
||
|
||
# Playwright selector health check — daily 12:00 UTC (7am CT).
|
||
# Proactively detects when FCC/USAC portal UI changes would break automation.
|
||
# Alerts via Telegram + email if selectors are missing.
|
||
- name: pw-playwright-health
|
||
description: Check FCC/USAC portal selectors are still valid
|
||
module: scripts.workers.services.telecom.playwright_monitor
|
||
on_calendar: "*-*-* 12:00:00 UTC"
|
||
persistent: true
|
||
|
||
# Pipeline orchestrator — every 5 min.
|
||
# Advances sequential service fulfillment (new carrier bundles, entity upgrades).
|
||
# Checks dependencies, forwards data between steps, dispatches next service.
|
||
- name: pw-pipeline-orchestrator
|
||
description: Advance multi-step compliance pipelines
|
||
module: scripts.workers.pipeline_orchestrator
|
||
on_calendar: "*-*-* *:0/5:00 UTC"
|
||
persistent: false
|
||
|
||
# FMCSA email monitor — every 15 min (offset :05), checks for filing confirmations.
|
||
- name: pw-fmcsa-email-monitor
|
||
description: Check for FMCSA filing confirmation emails
|
||
module: scripts.workers.client_email_processor
|
||
on_calendar: "*-*-* *:5/15:00 UTC"
|
||
persistent: false
|
||
|
||
# 499-Q quarterly filing reminders — daily 13:00 UTC (8am CT).
|
||
# Sends reminder emails at 30/14/7 days before each quarterly due date.
|
||
# Creates compliance_orders for each quarter when the 499-A+Q bundle is filed.
|
||
- name: pw-499q-notify
|
||
description: Send 499-Q quarterly filing reminders and intake links
|
||
module: scripts.workers.quarterly_499q_notify
|
||
on_calendar: "*-*-* 13:00:00 UTC"
|
||
persistent: true
|
||
|
||
# Trucking campaign builder — 3 AM EST (08:00 UTC) daily.
|
||
# Creates 8 Listmonk campaigns for the next day's sends (4 TZ × MCS-150 + inactive USDOT).
|
||
# 2,000 carriers/TZ for MCS-150; 1,000 carriers/TZ for inactive USDOT.
|
||
- name: pw-trucking-campaigns
|
||
description: Build daily trucking Listmonk campaigns (MCS-150 overdue + inactive USDOT)
|
||
module: scripts.build_trucking_campaigns
|
||
on_calendar: "*-*-* 08:00:00 UTC"
|
||
persistent: true
|