new-site/scripts/coupon_ab_scoreboard.py

#!/usr/bin/env python3
"""
coupon_ab_scoreboard.py — The trucking price A/B/C scoreboard.

Answers two questions per discount arm (20% / 30% / 40% / full-price control):
  1. Does the discount drive the CTA click?  -> HUMAN clicks from Umami
     (campaign-click events, already bot-filtered by pw-bot-filter.js), attributed
     to an arm by the ?code= the link carried.
  2. Does it drive the sale?                 -> PAID orders from compliance_orders,
     attributed to an arm by discount_code (discount arms) or, for the control
     arm (no code), by re-hashing customer_email the SAME way the builder bucketed
     it (sha256(email) % len(arms)).

Both DBs live on the same Postgres. Reads:
  - umami DB:           website_event  (event_name='campaign-click', has url_query/event_data)
  - performancewest DB: discount_codes (campaign-daily:<date>:<pct> marker -> arm),
                        compliance_orders (paid conversions + customer_email)

Usage (run on the host, or anywhere with psql access to the containers):
  python3 scripts/coupon_ab_scoreboard.py                 # last 14 days
  python3 scripts/coupon_ab_scoreboard.py --days 30
  python3 scripts/coupon_ab_scoreboard.py --arms 20,30,40,0

The arm set MUST match CAMPAIGN_COUPON_AB_PCTS so the control re-hash lines up.
"""
from __future__ import annotations

import argparse
import hashlib
import json
import os
import subprocess
import sys
from collections import defaultdict

# How to reach the two databases. On the host we exec into the postgres
# container; override with env if running elsewhere.
PG_CONTAINER = os.getenv("PW_PG_CONTAINER", "performancewest-api-postgres-1")
UMAMI_CONTAINER = os.getenv("PW_UMAMI_CONTAINER", "performancewest-umami-postgres-1")
PW_DB = os.getenv("PW_DB", "performancewest")
UMAMI_DB = os.getenv("PW_UMAMI_DB", "umami")
PW_USER = os.getenv("PW_DB_USER", "pw")
UMAMI_USER = os.getenv("PW_UMAMI_USER", "umami")


def psql(container: str, db: str, user: str, sql: str) -> list[list[str]]:
    """Run SQL via `docker exec ... psql -tAF$'\\t'` and return rows of fields."""
    out = subprocess.run(
        ["docker", "exec", "-i", container, "psql", "-U", user, "-d", db,
         "-tAF", "\t", "-c", sql],
        capture_output=True, text=True, timeout=60,
    )
    if out.returncode != 0:
        raise RuntimeError(f"psql {db} failed: {out.stderr[:300]}")
    return [line.split("\t") for line in out.stdout.splitlines() if line.strip()]


def arm_for_email(email: str, arms: list[int]) -> int:
    """Reproduce the builder's deterministic bucketing (pick_coupon_for_email).

    pcts are sorted; bucket = sha256(lower(email)) % len(pcts).
    """
    pcts = sorted(arms)
    if len(pcts) == 1:
        return pcts[0]
    h = hashlib.sha256((email or "").strip().lower().encode()).hexdigest()
    return pcts[int(h, 16) % len(pcts)]


def main() -> int:
    ap = argparse.ArgumentParser()
    ap.add_argument("--days", type=int, default=14)
    ap.add_argument("--arms", default=os.getenv("CAMPAIGN_COUPON_AB_PCTS", "20,30,40,0"),
                    help="comma list matching CAMPAIGN_COUPON_AB_PCTS, e.g. 20,30,40,0")
    args = ap.parse_args()

    arms = [int(a) for a in args.arms.split(",") if a.strip().lstrip("-").isdigit()]
    if not arms:
        print("no arms parsed", file=sys.stderr)
        return 1
    days = args.days

    # ── 1. Map every daily A/B code -> its arm pct (from the description marker) ──
    code_rows = psql(PG_CONTAINER, PW_DB, PW_USER, f"""
        SELECT upper(code), discount_value
        FROM discount_codes
        WHERE description LIKE 'campaign-daily:%'
          AND created_at > now() - interval '{days} days'
    """)
    code_to_pct = {c: int(v) for c, v in code_rows}

    # ── 2. HUMAN clicks per arm (Umami campaign-click, bot-filtered already) ──────
    # The arm rides in event_data 'code' (new) OR the url_query ?code= (older). We
    # read both and map code->pct. Clicks with no code = control arm (pct 0) IF the
    # control arm exists, but a no-code click could also just be a non-coupon link;
    # we only count no-code clicks toward control when they hit an /order/ page.
    click_rows = psql(UMAMI_CONTAINER, UMAMI_DB, UMAMI_USER, f"""
        SELECT lower(coalesce(url_path,'')) AS path,
               upper(coalesce(url_query,'')) AS q
        FROM website_event
        WHERE event_name = 'campaign-click'
          AND created_at > now() - interval '{days} days'
    """)
    clicks = defaultdict(int)
    for path, q in click_rows:
        code = ""
        if "CODE=" in q:
            # extract CODE=XXXXX from the query string
            seg = q.split("CODE=", 1)[1]
            for sep in ("&", "@", " "):
                seg = seg.split(sep, 1)[0]
            code = seg
        if code and code in code_to_pct:
            clicks[code_to_pct[code]] += 1
        elif not code and "/ORDER/" in path.upper() and 0 in arms:
            clicks[0] += 1  # no-code order click = full-price control arm

    # ── 3. PAID conversions per arm ──────────────────────────────────────────────
    order_rows = psql(PG_CONTAINER, PW_DB, PW_USER, f"""
        SELECT upper(coalesce(discount_code,'')) AS code,
               lower(coalesce(customer_email,'')) AS email,
               service_fee_cents
        FROM compliance_orders
        WHERE payment_status = 'paid'
          AND created_at > now() - interval '{days} days'
          AND customer_email !~* 'performancewest|carrierone|e2e|test|@example|synthetic|pipeline'
    """)
    paid = defaultdict(int)
    revenue = defaultdict(int)
    for code, email, fee in order_rows:
        fee_cents = int(fee) if fee and fee.isdigit() else 0
        if code and code in code_to_pct:
            arm = code_to_pct[code]
        elif code and code.isalpha() and len(code) == 5:
            # an A/B code from outside the window; skip rather than misattribute
            continue
        else:
            # control arm (no code) -> recover the arm by re-hashing the email
            arm = arm_for_email(email, arms)
        paid[arm] += 1
        revenue[arm] += fee_cents

    # ── 4. Print the scoreboard ──────────────────────────────────────────────────
    print(f"\n=== Trucking price A/B/C scoreboard (last {days} days) ===")
    print(f"arms: {arms}   (control = full price, no code)\n")
    hdr = f"{'arm':>8} {'human_clicks':>13} {'paid_orders':>12} {'revenue':>10} {'click→order':>12}"
    print(hdr)
    print("-" * len(hdr))
    for pct in sorted(arms):
        label = f"{pct}% off" if pct > 0 else "control"
        c = clicks.get(pct, 0)
        o = paid.get(pct, 0)
        rev = revenue.get(pct, 0) / 100.0
        cvr = f"{(o / c * 100):.1f}%" if c else "-"
        print(f"{label:>8} {c:>13} {o:>12} {('$%.0f' % rev):>10} {cvr:>12}")
    tot_c = sum(clicks.values()); tot_o = sum(paid.values()); tot_r = sum(revenue.values()) / 100.0
    print("-" * len(hdr))
    print(f"{'TOTAL':>8} {tot_c:>13} {tot_o:>12} {('$%.0f' % tot_r):>10} "
          f"{((tot_o/tot_c*100) if tot_c else 0):>11.1f}%")
    print("\nClicks are HUMAN (bot-filtered via Umami before-send). Conversions are")
    print("PAID Stripe orders. Control conversions recovered by email re-hash.")
    if tot_o < 30:
        print(f"\n  NOTE: only {tot_o} paid conversions in window — not yet enough for a")
        print("  statistically meaningful price comparison (want ~hundreds/arm). Clicks")
        print("  are the early CTA signal; conversions are the real answer once volume builds.")
    return 0


if __name__ == "__main__":
    raise SystemExit(main())