#!/usr/bin/env python3 """ coupon_ab_scoreboard.py — The trucking price A/B/C scoreboard. Answers two questions per discount arm (20% / 30% / 40% / full-price control): 1. Does the discount drive the CTA click? -> HUMAN clicks from Umami (campaign-click events, already bot-filtered by pw-bot-filter.js), attributed to an arm by the ?code= the link carried. 2. Does it drive the sale? -> PAID orders from compliance_orders, attributed to an arm by discount_code (discount arms) or, for the control arm (no code), by re-hashing customer_email the SAME way the builder bucketed it (sha256(email) % len(arms)). Both DBs live on the same Postgres. Reads: - umami DB: website_event (event_name='campaign-click', has url_query/event_data) - performancewest DB: discount_codes (campaign-daily:: marker -> arm), compliance_orders (paid conversions + customer_email) Usage (run on the host, or anywhere with psql access to the containers): python3 scripts/coupon_ab_scoreboard.py # last 14 days python3 scripts/coupon_ab_scoreboard.py --days 30 python3 scripts/coupon_ab_scoreboard.py --arms 20,30,40,0 The arm set MUST match CAMPAIGN_COUPON_AB_PCTS so the control re-hash lines up. """ from __future__ import annotations import argparse import hashlib import json import os import subprocess import sys import urllib.parse import urllib.request from collections import defaultdict # How to reach the two databases. On the host we exec into the postgres # container; override with env if running elsewhere. PG_CONTAINER = os.getenv("PW_PG_CONTAINER", "performancewest-api-postgres-1") UMAMI_CONTAINER = os.getenv("PW_UMAMI_CONTAINER", "performancewest-umami-postgres-1") PW_DB = os.getenv("PW_DB", "performancewest") UMAMI_DB = os.getenv("PW_UMAMI_DB", "umami") PW_USER = os.getenv("PW_DB_USER", "pw") UMAMI_USER = os.getenv("PW_UMAMI_USER", "umami") def psql(container: str, db: str, user: str, sql: str) -> list[list[str]]: """Run SQL via `docker exec ... psql -tAF$'\\t'` and return rows of fields.""" out = subprocess.run( ["docker", "exec", "-i", container, "psql", "-U", user, "-d", db, "-tAF", "\t", "-c", sql], capture_output=True, text=True, timeout=60, ) if out.returncode != 0: raise RuntimeError(f"psql {db} failed: {out.stderr[:300]}") return [line.split("\t") for line in out.stdout.splitlines() if line.strip()] def arm_for_email(email: str, arms: list[int]) -> int: """Reproduce the builder's deterministic bucketing (pick_coupon_for_email). pcts are sorted; bucket = sha256(lower(email)) % len(pcts). """ pcts = sorted(arms) if len(pcts) == 1: return pcts[0] h = hashlib.sha256((email or "").strip().lower().encode()).hexdigest() return pcts[int(h, 16) % len(pcts)] def send_telegram(text: str) -> bool: """Post a message to the ops Telegram chat. Creds from /opt/performancewest/.env (TELEGRAM_BOT_TOKEN / TELEGRAM_CHAT_ID), same as the other monitors. Returns True if a request was attempted with creds present. """ env_path = os.getenv("PW_ENV_FILE", "/opt/performancewest/.env") bot = chat = "" try: with open(env_path) as f: for line in f: if line.startswith("TELEGRAM_BOT_TOKEN="): bot = line.split("=", 1)[1].strip() elif line.startswith("TELEGRAM_CHAT_ID="): chat = line.split("=", 1)[1].strip() except OSError: pass bot = os.getenv("TELEGRAM_BOT_TOKEN", bot) chat = os.getenv("TELEGRAM_CHAT_ID", chat) if not bot or not chat: return False data = urllib.parse.urlencode( {"chat_id": chat, "text": text, "parse_mode": "HTML"} ).encode() req = urllib.request.Request( f"https://api.telegram.org/bot{bot}/sendMessage", data=data ) try: urllib.request.urlopen(req, timeout=10).read() return True except Exception: return False def main() -> int: ap = argparse.ArgumentParser() ap.add_argument("--days", type=int, default=14) ap.add_argument("--arms", default=os.getenv("CAMPAIGN_COUPON_AB_PCTS", "20,30,40,0"), help="comma list matching CAMPAIGN_COUPON_AB_PCTS, e.g. 20,30,40,0") ap.add_argument("--telegram", action="store_true", help="push the scoreboard to Telegram, but ONLY when there is >=1 paid sale in the window") args = ap.parse_args() arms = [int(a) for a in args.arms.split(",") if a.strip().lstrip("-").isdigit()] if not arms: print("no arms parsed", file=sys.stderr) return 1 days = args.days # ── 1. Map every daily A/B code -> its arm pct (from the description marker) ── code_rows = psql(PG_CONTAINER, PW_DB, PW_USER, f""" SELECT upper(code), discount_value FROM discount_codes WHERE description LIKE 'campaign-daily:%' AND created_at > now() - interval '{days} days' """) code_to_pct = {c: int(v) for c, v in code_rows} # ── 2. HUMAN clicks per arm (Umami campaign-click, bot-filtered already) ────── # The arm rides in event_data 'code' (new) OR the url_query ?code= (older). We # read both and map code->pct. Clicks with no code = control arm (pct 0) IF the # control arm exists, but a no-code click could also just be a non-coupon link; # we only count no-code clicks toward control when they hit an /order/ page. click_rows = psql(UMAMI_CONTAINER, UMAMI_DB, UMAMI_USER, f""" SELECT lower(coalesce(url_path,'')) AS path, upper(coalesce(url_query,'')) AS q FROM website_event WHERE event_name = 'campaign-click' AND created_at > now() - interval '{days} days' """) clicks = defaultdict(int) for path, q in click_rows: code = "" if "CODE=" in q: # extract CODE=XXXXX from the query string seg = q.split("CODE=", 1)[1] for sep in ("&", "@", " "): seg = seg.split(sep, 1)[0] code = seg if code and code in code_to_pct: clicks[code_to_pct[code]] += 1 elif not code and "/ORDER/" in path.upper() and 0 in arms: clicks[0] += 1 # no-code order click = full-price control arm # ── 3. PAID conversions per arm ────────────────────────────────────────────── order_rows = psql(PG_CONTAINER, PW_DB, PW_USER, f""" SELECT upper(coalesce(discount_code,'')) AS code, lower(coalesce(customer_email,'')) AS email, service_fee_cents FROM compliance_orders WHERE payment_status = 'paid' AND created_at > now() - interval '{days} days' AND customer_email !~* 'performancewest|carrierone|e2e|test|@example|synthetic|pipeline' """) paid = defaultdict(int) revenue = defaultdict(int) for code, email, fee in order_rows: fee_cents = int(fee) if fee and fee.isdigit() else 0 if code and code in code_to_pct: arm = code_to_pct[code] elif code and code.isalpha() and len(code) == 5: # an A/B code from outside the window; skip rather than misattribute continue else: # control arm (no code) -> recover the arm by re-hashing the email arm = arm_for_email(email, arms) paid[arm] += 1 revenue[arm] += fee_cents # ── 4. Build the scoreboard rows ───────────────────────────────────────────── tot_c = sum(clicks.values()) tot_o = sum(paid.values()) tot_r = sum(revenue.values()) / 100.0 hdr = f"{'arm':>8} {'human_clicks':>13} {'paid_orders':>12} {'revenue':>10} {'click->order':>12}" lines = [ f"=== Trucking price A/B/C scoreboard (last {days} days) ===", f"arms: {arms} (control = full price, no code)", "", hdr, "-" * len(hdr), ] for pct in sorted(arms): label = f"{pct}% off" if pct > 0 else "control" c = clicks.get(pct, 0) o = paid.get(pct, 0) rev = revenue.get(pct, 0) / 100.0 cvr = f"{(o / c * 100):.1f}%" if c else "-" lines.append(f"{label:>8} {c:>13} {o:>12} {('$%.0f' % rev):>10} {cvr:>12}") lines.append("-" * len(hdr)) lines.append(f"{'TOTAL':>8} {tot_c:>13} {tot_o:>12} {('$%.0f' % tot_r):>10} " f"{((tot_o/tot_c*100) if tot_c else 0):>11.1f}%") table = "\n".join(lines) if args.telegram: # Daily push, but ONLY when there is at least one paid sale in the window. # A discount price test is only meaningful once money moves; on no-sale days # we stay silent (per request) rather than spam an all-zero table. if tot_o <= 0: print(table) print("\n[telegram] no paid sales in window — staying silent (no report sent).") return 0 msg = (f"Trucking price A/B/C ({days}d)\n" f"{tot_o} paid / ${tot_r:.0f} rev / {tot_c} human clicks\n\n" f"
{table}
") sent = send_telegram(msg) print(table) print(f"\n[telegram] {'sent' if sent else 'NOT sent (missing creds?)'} " f"({tot_o} paid sales triggered the report).") return 0 # ── Console output ─────────────────────────────────────────────────────────── print(table) print("\nClicks are HUMAN (bot-filtered via Umami before-send). Conversions are") print("PAID Stripe orders. Control conversions recovered by email re-hash.") if tot_o < 30: print(f"\n NOTE: only {tot_o} paid conversions in window — not yet enough for a") print(" statistically meaningful price comparison (want ~hundreds/arm). Clicks") print(" are the early CTA signal; conversions are the real answer once volume builds.") return 0 if __name__ == "__main__": raise SystemExit(main())