9abd08c305
Adds --telegram mode to coupon_ab_scoreboard.py + a daily cron (18:00 UTC). Pushes the per-arm clicks+conversions table to Telegram ONLY when there was at least one PAID sale in the window; stays silent otherwise (per request, no all-zero spam). Creds read from /opt/performancewest/.env like the other monitors. Verified live: silent at 0 sales (14d), sends at 5 sales (30d).
235 lines
10 KiB
Python
235 lines
10 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
coupon_ab_scoreboard.py — The trucking price A/B/C scoreboard.
|
|
|
|
Answers two questions per discount arm (20% / 30% / 40% / full-price control):
|
|
1. Does the discount drive the CTA click? -> HUMAN clicks from Umami
|
|
(campaign-click events, already bot-filtered by pw-bot-filter.js), attributed
|
|
to an arm by the ?code= the link carried.
|
|
2. Does it drive the sale? -> PAID orders from compliance_orders,
|
|
attributed to an arm by discount_code (discount arms) or, for the control
|
|
arm (no code), by re-hashing customer_email the SAME way the builder bucketed
|
|
it (sha256(email) % len(arms)).
|
|
|
|
Both DBs live on the same Postgres. Reads:
|
|
- umami DB: website_event (event_name='campaign-click', has url_query/event_data)
|
|
- performancewest DB: discount_codes (campaign-daily:<date>:<pct> marker -> arm),
|
|
compliance_orders (paid conversions + customer_email)
|
|
|
|
Usage (run on the host, or anywhere with psql access to the containers):
|
|
python3 scripts/coupon_ab_scoreboard.py # last 14 days
|
|
python3 scripts/coupon_ab_scoreboard.py --days 30
|
|
python3 scripts/coupon_ab_scoreboard.py --arms 20,30,40,0
|
|
|
|
The arm set MUST match CAMPAIGN_COUPON_AB_PCTS so the control re-hash lines up.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import urllib.parse
|
|
import urllib.request
|
|
from collections import defaultdict
|
|
|
|
# How to reach the two databases. On the host we exec into the postgres
|
|
# container; override with env if running elsewhere.
|
|
PG_CONTAINER = os.getenv("PW_PG_CONTAINER", "performancewest-api-postgres-1")
|
|
UMAMI_CONTAINER = os.getenv("PW_UMAMI_CONTAINER", "performancewest-umami-postgres-1")
|
|
PW_DB = os.getenv("PW_DB", "performancewest")
|
|
UMAMI_DB = os.getenv("PW_UMAMI_DB", "umami")
|
|
PW_USER = os.getenv("PW_DB_USER", "pw")
|
|
UMAMI_USER = os.getenv("PW_UMAMI_USER", "umami")
|
|
|
|
|
|
def psql(container: str, db: str, user: str, sql: str) -> list[list[str]]:
|
|
"""Run SQL via `docker exec ... psql -tAF$'\\t'` and return rows of fields."""
|
|
out = subprocess.run(
|
|
["docker", "exec", "-i", container, "psql", "-U", user, "-d", db,
|
|
"-tAF", "\t", "-c", sql],
|
|
capture_output=True, text=True, timeout=60,
|
|
)
|
|
if out.returncode != 0:
|
|
raise RuntimeError(f"psql {db} failed: {out.stderr[:300]}")
|
|
return [line.split("\t") for line in out.stdout.splitlines() if line.strip()]
|
|
|
|
|
|
def arm_for_email(email: str, arms: list[int]) -> int:
|
|
"""Reproduce the builder's deterministic bucketing (pick_coupon_for_email).
|
|
|
|
pcts are sorted; bucket = sha256(lower(email)) % len(pcts).
|
|
"""
|
|
pcts = sorted(arms)
|
|
if len(pcts) == 1:
|
|
return pcts[0]
|
|
h = hashlib.sha256((email or "").strip().lower().encode()).hexdigest()
|
|
return pcts[int(h, 16) % len(pcts)]
|
|
|
|
|
|
def send_telegram(text: str) -> bool:
|
|
"""Post a message to the ops Telegram chat. Creds from /opt/performancewest/.env
|
|
(TELEGRAM_BOT_TOKEN / TELEGRAM_CHAT_ID), same as the other monitors. Returns
|
|
True if a request was attempted with creds present.
|
|
"""
|
|
env_path = os.getenv("PW_ENV_FILE", "/opt/performancewest/.env")
|
|
bot = chat = ""
|
|
try:
|
|
with open(env_path) as f:
|
|
for line in f:
|
|
if line.startswith("TELEGRAM_BOT_TOKEN="):
|
|
bot = line.split("=", 1)[1].strip()
|
|
elif line.startswith("TELEGRAM_CHAT_ID="):
|
|
chat = line.split("=", 1)[1].strip()
|
|
except OSError:
|
|
pass
|
|
bot = os.getenv("TELEGRAM_BOT_TOKEN", bot)
|
|
chat = os.getenv("TELEGRAM_CHAT_ID", chat)
|
|
if not bot or not chat:
|
|
return False
|
|
data = urllib.parse.urlencode(
|
|
{"chat_id": chat, "text": text, "parse_mode": "HTML"}
|
|
).encode()
|
|
req = urllib.request.Request(
|
|
f"https://api.telegram.org/bot{bot}/sendMessage", data=data
|
|
)
|
|
try:
|
|
urllib.request.urlopen(req, timeout=10).read()
|
|
return True
|
|
except Exception:
|
|
return False
|
|
|
|
|
|
|
|
def main() -> int:
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument("--days", type=int, default=14)
|
|
ap.add_argument("--arms", default=os.getenv("CAMPAIGN_COUPON_AB_PCTS", "20,30,40,0"),
|
|
help="comma list matching CAMPAIGN_COUPON_AB_PCTS, e.g. 20,30,40,0")
|
|
ap.add_argument("--telegram", action="store_true",
|
|
help="push the scoreboard to Telegram, but ONLY when there is >=1 paid sale in the window")
|
|
args = ap.parse_args()
|
|
|
|
arms = [int(a) for a in args.arms.split(",") if a.strip().lstrip("-").isdigit()]
|
|
if not arms:
|
|
print("no arms parsed", file=sys.stderr)
|
|
return 1
|
|
days = args.days
|
|
|
|
# ── 1. Map every daily A/B code -> its arm pct (from the description marker) ──
|
|
code_rows = psql(PG_CONTAINER, PW_DB, PW_USER, f"""
|
|
SELECT upper(code), discount_value
|
|
FROM discount_codes
|
|
WHERE description LIKE 'campaign-daily:%'
|
|
AND created_at > now() - interval '{days} days'
|
|
""")
|
|
code_to_pct = {c: int(v) for c, v in code_rows}
|
|
|
|
# ── 2. HUMAN clicks per arm (Umami campaign-click, bot-filtered already) ──────
|
|
# The arm rides in event_data 'code' (new) OR the url_query ?code= (older). We
|
|
# read both and map code->pct. Clicks with no code = control arm (pct 0) IF the
|
|
# control arm exists, but a no-code click could also just be a non-coupon link;
|
|
# we only count no-code clicks toward control when they hit an /order/ page.
|
|
click_rows = psql(UMAMI_CONTAINER, UMAMI_DB, UMAMI_USER, f"""
|
|
SELECT lower(coalesce(url_path,'')) AS path,
|
|
upper(coalesce(url_query,'')) AS q
|
|
FROM website_event
|
|
WHERE event_name = 'campaign-click'
|
|
AND created_at > now() - interval '{days} days'
|
|
""")
|
|
clicks = defaultdict(int)
|
|
for path, q in click_rows:
|
|
code = ""
|
|
if "CODE=" in q:
|
|
# extract CODE=XXXXX from the query string
|
|
seg = q.split("CODE=", 1)[1]
|
|
for sep in ("&", "@", " "):
|
|
seg = seg.split(sep, 1)[0]
|
|
code = seg
|
|
if code and code in code_to_pct:
|
|
clicks[code_to_pct[code]] += 1
|
|
elif not code and "/ORDER/" in path.upper() and 0 in arms:
|
|
clicks[0] += 1 # no-code order click = full-price control arm
|
|
|
|
# ── 3. PAID conversions per arm ──────────────────────────────────────────────
|
|
order_rows = psql(PG_CONTAINER, PW_DB, PW_USER, f"""
|
|
SELECT upper(coalesce(discount_code,'')) AS code,
|
|
lower(coalesce(customer_email,'')) AS email,
|
|
service_fee_cents
|
|
FROM compliance_orders
|
|
WHERE payment_status = 'paid'
|
|
AND created_at > now() - interval '{days} days'
|
|
AND customer_email !~* 'performancewest|carrierone|e2e|test|@example|synthetic|pipeline'
|
|
""")
|
|
paid = defaultdict(int)
|
|
revenue = defaultdict(int)
|
|
for code, email, fee in order_rows:
|
|
fee_cents = int(fee) if fee and fee.isdigit() else 0
|
|
if code and code in code_to_pct:
|
|
arm = code_to_pct[code]
|
|
elif code and code.isalpha() and len(code) == 5:
|
|
# an A/B code from outside the window; skip rather than misattribute
|
|
continue
|
|
else:
|
|
# control arm (no code) -> recover the arm by re-hashing the email
|
|
arm = arm_for_email(email, arms)
|
|
paid[arm] += 1
|
|
revenue[arm] += fee_cents
|
|
|
|
# ── 4. Build the scoreboard rows ─────────────────────────────────────────────
|
|
tot_c = sum(clicks.values())
|
|
tot_o = sum(paid.values())
|
|
tot_r = sum(revenue.values()) / 100.0
|
|
|
|
hdr = f"{'arm':>8} {'human_clicks':>13} {'paid_orders':>12} {'revenue':>10} {'click->order':>12}"
|
|
lines = [
|
|
f"=== Trucking price A/B/C scoreboard (last {days} days) ===",
|
|
f"arms: {arms} (control = full price, no code)",
|
|
"",
|
|
hdr,
|
|
"-" * len(hdr),
|
|
]
|
|
for pct in sorted(arms):
|
|
label = f"{pct}% off" if pct > 0 else "control"
|
|
c = clicks.get(pct, 0)
|
|
o = paid.get(pct, 0)
|
|
rev = revenue.get(pct, 0) / 100.0
|
|
cvr = f"{(o / c * 100):.1f}%" if c else "-"
|
|
lines.append(f"{label:>8} {c:>13} {o:>12} {('$%.0f' % rev):>10} {cvr:>12}")
|
|
lines.append("-" * len(hdr))
|
|
lines.append(f"{'TOTAL':>8} {tot_c:>13} {tot_o:>12} {('$%.0f' % tot_r):>10} "
|
|
f"{((tot_o/tot_c*100) if tot_c else 0):>11.1f}%")
|
|
table = "\n".join(lines)
|
|
|
|
if args.telegram:
|
|
# Daily push, but ONLY when there is at least one paid sale in the window.
|
|
# A discount price test is only meaningful once money moves; on no-sale days
|
|
# we stay silent (per request) rather than spam an all-zero table.
|
|
if tot_o <= 0:
|
|
print(table)
|
|
print("\n[telegram] no paid sales in window — staying silent (no report sent).")
|
|
return 0
|
|
msg = (f"Trucking price A/B/C ({days}d)\n"
|
|
f"{tot_o} paid / ${tot_r:.0f} rev / {tot_c} human clicks\n\n"
|
|
f"<pre>{table}</pre>")
|
|
sent = send_telegram(msg)
|
|
print(table)
|
|
print(f"\n[telegram] {'sent' if sent else 'NOT sent (missing creds?)'} "
|
|
f"({tot_o} paid sales triggered the report).")
|
|
return 0
|
|
|
|
# ── Console output ───────────────────────────────────────────────────────────
|
|
print(table)
|
|
print("\nClicks are HUMAN (bot-filtered via Umami before-send). Conversions are")
|
|
print("PAID Stripe orders. Control conversions recovered by email re-hash.")
|
|
if tot_o < 30:
|
|
print(f"\n NOTE: only {tot_o} paid conversions in window — not yet enough for a")
|
|
print(" statistically meaningful price comparison (want ~hundreds/arm). Clicks")
|
|
print(" are the early CTA signal; conversions are the real answer once volume builds.")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|