#!/usr/bin/env python3 """ coupon_ab_scoreboard.py — The trucking price A/B/C scoreboard. Answers two questions per discount arm (20% / 30% / 40% / full-price control): 1. Does the discount drive the CTA click? -> HUMAN clicks from Umami (campaign-click events, already bot-filtered by pw-bot-filter.js), attributed to an arm by the ?code= the link carried. 2. Does it drive the sale? -> PAID orders from compliance_orders, attributed to an arm by discount_code (discount arms) or, for the control arm (no code), by re-hashing customer_email the SAME way the builder bucketed it (sha256(email) % len(arms)). Both DBs live on the same Postgres. Reads: - umami DB: website_event (event_name='campaign-click', has url_query/event_data) - performancewest DB: discount_codes (campaign-daily:: marker -> arm), compliance_orders (paid conversions + customer_email) Usage (run on the host, or anywhere with psql access to the containers): python3 scripts/coupon_ab_scoreboard.py # last 14 days python3 scripts/coupon_ab_scoreboard.py --days 30 python3 scripts/coupon_ab_scoreboard.py --arms 20,30,40,0 The arm set MUST match CAMPAIGN_COUPON_AB_PCTS so the control re-hash lines up. """ from __future__ import annotations import argparse import hashlib import json import os import subprocess import sys from collections import defaultdict # How to reach the two databases. On the host we exec into the postgres # container; override with env if running elsewhere. PG_CONTAINER = os.getenv("PW_PG_CONTAINER", "performancewest-api-postgres-1") UMAMI_CONTAINER = os.getenv("PW_UMAMI_CONTAINER", "performancewest-umami-postgres-1") PW_DB = os.getenv("PW_DB", "performancewest") UMAMI_DB = os.getenv("PW_UMAMI_DB", "umami") PW_USER = os.getenv("PW_DB_USER", "pw") UMAMI_USER = os.getenv("PW_UMAMI_USER", "umami") def psql(container: str, db: str, user: str, sql: str) -> list[list[str]]: """Run SQL via `docker exec ... psql -tAF$'\\t'` and return rows of fields.""" out = subprocess.run( ["docker", "exec", "-i", container, "psql", "-U", user, "-d", db, "-tAF", "\t", "-c", sql], capture_output=True, text=True, timeout=60, ) if out.returncode != 0: raise RuntimeError(f"psql {db} failed: {out.stderr[:300]}") return [line.split("\t") for line in out.stdout.splitlines() if line.strip()] def arm_for_email(email: str, arms: list[int]) -> int: """Reproduce the builder's deterministic bucketing (pick_coupon_for_email). pcts are sorted; bucket = sha256(lower(email)) % len(pcts). """ pcts = sorted(arms) if len(pcts) == 1: return pcts[0] h = hashlib.sha256((email or "").strip().lower().encode()).hexdigest() return pcts[int(h, 16) % len(pcts)] def main() -> int: ap = argparse.ArgumentParser() ap.add_argument("--days", type=int, default=14) ap.add_argument("--arms", default=os.getenv("CAMPAIGN_COUPON_AB_PCTS", "20,30,40,0"), help="comma list matching CAMPAIGN_COUPON_AB_PCTS, e.g. 20,30,40,0") args = ap.parse_args() arms = [int(a) for a in args.arms.split(",") if a.strip().lstrip("-").isdigit()] if not arms: print("no arms parsed", file=sys.stderr) return 1 days = args.days # ── 1. Map every daily A/B code -> its arm pct (from the description marker) ── code_rows = psql(PG_CONTAINER, PW_DB, PW_USER, f""" SELECT upper(code), discount_value FROM discount_codes WHERE description LIKE 'campaign-daily:%' AND created_at > now() - interval '{days} days' """) code_to_pct = {c: int(v) for c, v in code_rows} # ── 2. HUMAN clicks per arm (Umami campaign-click, bot-filtered already) ────── # The arm rides in event_data 'code' (new) OR the url_query ?code= (older). We # read both and map code->pct. Clicks with no code = control arm (pct 0) IF the # control arm exists, but a no-code click could also just be a non-coupon link; # we only count no-code clicks toward control when they hit an /order/ page. click_rows = psql(UMAMI_CONTAINER, UMAMI_DB, UMAMI_USER, f""" SELECT lower(coalesce(url_path,'')) AS path, upper(coalesce(url_query,'')) AS q FROM website_event WHERE event_name = 'campaign-click' AND created_at > now() - interval '{days} days' """) clicks = defaultdict(int) for path, q in click_rows: code = "" if "CODE=" in q: # extract CODE=XXXXX from the query string seg = q.split("CODE=", 1)[1] for sep in ("&", "@", " "): seg = seg.split(sep, 1)[0] code = seg if code and code in code_to_pct: clicks[code_to_pct[code]] += 1 elif not code and "/ORDER/" in path.upper() and 0 in arms: clicks[0] += 1 # no-code order click = full-price control arm # ── 3. PAID conversions per arm ────────────────────────────────────────────── order_rows = psql(PG_CONTAINER, PW_DB, PW_USER, f""" SELECT upper(coalesce(discount_code,'')) AS code, lower(coalesce(customer_email,'')) AS email, service_fee_cents FROM compliance_orders WHERE payment_status = 'paid' AND created_at > now() - interval '{days} days' AND customer_email !~* 'performancewest|carrierone|e2e|test|@example|synthetic|pipeline' """) paid = defaultdict(int) revenue = defaultdict(int) for code, email, fee in order_rows: fee_cents = int(fee) if fee and fee.isdigit() else 0 if code and code in code_to_pct: arm = code_to_pct[code] elif code and code.isalpha() and len(code) == 5: # an A/B code from outside the window; skip rather than misattribute continue else: # control arm (no code) -> recover the arm by re-hashing the email arm = arm_for_email(email, arms) paid[arm] += 1 revenue[arm] += fee_cents # ── 4. Print the scoreboard ────────────────────────────────────────────────── print(f"\n=== Trucking price A/B/C scoreboard (last {days} days) ===") print(f"arms: {arms} (control = full price, no code)\n") hdr = f"{'arm':>8} {'human_clicks':>13} {'paid_orders':>12} {'revenue':>10} {'click→order':>12}" print(hdr) print("-" * len(hdr)) for pct in sorted(arms): label = f"{pct}% off" if pct > 0 else "control" c = clicks.get(pct, 0) o = paid.get(pct, 0) rev = revenue.get(pct, 0) / 100.0 cvr = f"{(o / c * 100):.1f}%" if c else "-" print(f"{label:>8} {c:>13} {o:>12} {('$%.0f' % rev):>10} {cvr:>12}") tot_c = sum(clicks.values()); tot_o = sum(paid.values()); tot_r = sum(revenue.values()) / 100.0 print("-" * len(hdr)) print(f"{'TOTAL':>8} {tot_c:>13} {tot_o:>12} {('$%.0f' % tot_r):>10} " f"{((tot_o/tot_c*100) if tot_c else 0):>11.1f}%") print("\nClicks are HUMAN (bot-filtered via Umami before-send). Conversions are") print("PAID Stripe orders. Control conversions recovered by email re-hash.") if tot_o < 30: print(f"\n NOTE: only {tot_o} paid conversions in window — not yet enough for a") print(" statistically meaningful price comparison (want ~hundreds/arm). Clicks") print(" are the early CTA signal; conversions are the real answer once volume builds.") return 0 if __name__ == "__main__": raise SystemExit(main())