From 60d5d3b9d883f601afcb0f40948c74291e506d4f Mon Sep 17 00:00:00 2001 From: justin Date: Tue, 30 Jun 2026 16:39:26 -0500 Subject: [PATCH] analytics: attribute price-test arm per click + A/B/C scoreboard So we can see whether a discount helps the CTA (clicks) AND the sale (paid orders), per arm: 1. pw-analytics.js: include the ?code= (the price-test arm) on the campaign-click Umami event, and fire it when EITHER a utm OR a code is present (coupon links often carry code with no utm, so those human clicks were being dropped). Also strip a stray @TrackLink suffix Listmonk glues onto the last query value. 2. coupon_ab_scoreboard.py: one report combining HUMAN clicks (Umami, already bot-filtered) and PAID conversions (compliance_orders) per arm. Discount arms map by code->pct (campaign-daily:: marker); the no-code control arm is recovered by re-hashing customer_email the same way the builder bucketed it. Prints clicks, paid orders, revenue, and click->order per arm. --- scripts/coupon_ab_scoreboard.py | 170 ++++++++++++++++++++++++++++++++ site/public/js/pw-analytics.js | 18 +++- 2 files changed, 184 insertions(+), 4 deletions(-) create mode 100644 scripts/coupon_ab_scoreboard.py diff --git a/scripts/coupon_ab_scoreboard.py b/scripts/coupon_ab_scoreboard.py new file mode 100644 index 0000000..3133aa3 --- /dev/null +++ b/scripts/coupon_ab_scoreboard.py @@ -0,0 +1,170 @@ +#!/usr/bin/env python3 +""" +coupon_ab_scoreboard.py — The trucking price A/B/C scoreboard. + +Answers two questions per discount arm (20% / 30% / 40% / full-price control): + 1. Does the discount drive the CTA click? -> HUMAN clicks from Umami + (campaign-click events, already bot-filtered by pw-bot-filter.js), attributed + to an arm by the ?code= the link carried. + 2. Does it drive the sale? -> PAID orders from compliance_orders, + attributed to an arm by discount_code (discount arms) or, for the control + arm (no code), by re-hashing customer_email the SAME way the builder bucketed + it (sha256(email) % len(arms)). + +Both DBs live on the same Postgres. Reads: + - umami DB: website_event (event_name='campaign-click', has url_query/event_data) + - performancewest DB: discount_codes (campaign-daily:: marker -> arm), + compliance_orders (paid conversions + customer_email) + +Usage (run on the host, or anywhere with psql access to the containers): + python3 scripts/coupon_ab_scoreboard.py # last 14 days + python3 scripts/coupon_ab_scoreboard.py --days 30 + python3 scripts/coupon_ab_scoreboard.py --arms 20,30,40,0 + +The arm set MUST match CAMPAIGN_COUPON_AB_PCTS so the control re-hash lines up. +""" +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import subprocess +import sys +from collections import defaultdict + +# How to reach the two databases. On the host we exec into the postgres +# container; override with env if running elsewhere. +PG_CONTAINER = os.getenv("PW_PG_CONTAINER", "performancewest-api-postgres-1") +UMAMI_CONTAINER = os.getenv("PW_UMAMI_CONTAINER", "performancewest-umami-postgres-1") +PW_DB = os.getenv("PW_DB", "performancewest") +UMAMI_DB = os.getenv("PW_UMAMI_DB", "umami") +PW_USER = os.getenv("PW_DB_USER", "pw") +UMAMI_USER = os.getenv("PW_UMAMI_USER", "umami") + + +def psql(container: str, db: str, user: str, sql: str) -> list[list[str]]: + """Run SQL via `docker exec ... psql -tAF$'\\t'` and return rows of fields.""" + out = subprocess.run( + ["docker", "exec", "-i", container, "psql", "-U", user, "-d", db, + "-tAF", "\t", "-c", sql], + capture_output=True, text=True, timeout=60, + ) + if out.returncode != 0: + raise RuntimeError(f"psql {db} failed: {out.stderr[:300]}") + return [line.split("\t") for line in out.stdout.splitlines() if line.strip()] + + +def arm_for_email(email: str, arms: list[int]) -> int: + """Reproduce the builder's deterministic bucketing (pick_coupon_for_email). + + pcts are sorted; bucket = sha256(lower(email)) % len(pcts). + """ + pcts = sorted(arms) + if len(pcts) == 1: + return pcts[0] + h = hashlib.sha256((email or "").strip().lower().encode()).hexdigest() + return pcts[int(h, 16) % len(pcts)] + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--days", type=int, default=14) + ap.add_argument("--arms", default=os.getenv("CAMPAIGN_COUPON_AB_PCTS", "20,30,40,0"), + help="comma list matching CAMPAIGN_COUPON_AB_PCTS, e.g. 20,30,40,0") + args = ap.parse_args() + + arms = [int(a) for a in args.arms.split(",") if a.strip().lstrip("-").isdigit()] + if not arms: + print("no arms parsed", file=sys.stderr) + return 1 + days = args.days + + # ── 1. Map every daily A/B code -> its arm pct (from the description marker) ── + code_rows = psql(PG_CONTAINER, PW_DB, PW_USER, f""" + SELECT upper(code), discount_value + FROM discount_codes + WHERE description LIKE 'campaign-daily:%' + AND created_at > now() - interval '{days} days' + """) + code_to_pct = {c: int(v) for c, v in code_rows} + + # ── 2. HUMAN clicks per arm (Umami campaign-click, bot-filtered already) ────── + # The arm rides in event_data 'code' (new) OR the url_query ?code= (older). We + # read both and map code->pct. Clicks with no code = control arm (pct 0) IF the + # control arm exists, but a no-code click could also just be a non-coupon link; + # we only count no-code clicks toward control when they hit an /order/ page. + click_rows = psql(UMAMI_CONTAINER, UMAMI_DB, UMAMI_USER, f""" + SELECT lower(coalesce(url_path,'')) AS path, + upper(coalesce(url_query,'')) AS q + FROM website_event + WHERE event_name = 'campaign-click' + AND created_at > now() - interval '{days} days' + """) + clicks = defaultdict(int) + for path, q in click_rows: + code = "" + if "CODE=" in q: + # extract CODE=XXXXX from the query string + seg = q.split("CODE=", 1)[1] + for sep in ("&", "@", " "): + seg = seg.split(sep, 1)[0] + code = seg + if code and code in code_to_pct: + clicks[code_to_pct[code]] += 1 + elif not code and "/ORDER/" in path.upper() and 0 in arms: + clicks[0] += 1 # no-code order click = full-price control arm + + # ── 3. PAID conversions per arm ────────────────────────────────────────────── + order_rows = psql(PG_CONTAINER, PW_DB, PW_USER, f""" + SELECT upper(coalesce(discount_code,'')) AS code, + lower(coalesce(customer_email,'')) AS email, + service_fee_cents + FROM compliance_orders + WHERE payment_status = 'paid' + AND created_at > now() - interval '{days} days' + AND customer_email !~* 'performancewest|carrierone|e2e|test|@example|synthetic|pipeline' + """) + paid = defaultdict(int) + revenue = defaultdict(int) + for code, email, fee in order_rows: + fee_cents = int(fee) if fee and fee.isdigit() else 0 + if code and code in code_to_pct: + arm = code_to_pct[code] + elif code and code.isalpha() and len(code) == 5: + # an A/B code from outside the window; skip rather than misattribute + continue + else: + # control arm (no code) -> recover the arm by re-hashing the email + arm = arm_for_email(email, arms) + paid[arm] += 1 + revenue[arm] += fee_cents + + # ── 4. Print the scoreboard ────────────────────────────────────────────────── + print(f"\n=== Trucking price A/B/C scoreboard (last {days} days) ===") + print(f"arms: {arms} (control = full price, no code)\n") + hdr = f"{'arm':>8} {'human_clicks':>13} {'paid_orders':>12} {'revenue':>10} {'click→order':>12}" + print(hdr) + print("-" * len(hdr)) + for pct in sorted(arms): + label = f"{pct}% off" if pct > 0 else "control" + c = clicks.get(pct, 0) + o = paid.get(pct, 0) + rev = revenue.get(pct, 0) / 100.0 + cvr = f"{(o / c * 100):.1f}%" if c else "-" + print(f"{label:>8} {c:>13} {o:>12} {('$%.0f' % rev):>10} {cvr:>12}") + tot_c = sum(clicks.values()); tot_o = sum(paid.values()); tot_r = sum(revenue.values()) / 100.0 + print("-" * len(hdr)) + print(f"{'TOTAL':>8} {tot_c:>13} {tot_o:>12} {('$%.0f' % tot_r):>10} " + f"{((tot_o/tot_c*100) if tot_c else 0):>11.1f}%") + print("\nClicks are HUMAN (bot-filtered via Umami before-send). Conversions are") + print("PAID Stripe orders. Control conversions recovered by email re-hash.") + if tot_o < 30: + print(f"\n NOTE: only {tot_o} paid conversions in window — not yet enough for a") + print(" statistically meaningful price comparison (want ~hundreds/arm). Clicks") + print(" are the early CTA signal; conversions are the real answer once volume builds.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/site/public/js/pw-analytics.js b/site/public/js/pw-analytics.js index e5cd8b6..313ac54 100644 --- a/site/public/js/pw-analytics.js +++ b/site/public/js/pw-analytics.js @@ -107,11 +107,21 @@ var params = new URLSearchParams(location.search); var utmSource = params.get("utm_source"); var utmCampaign = params.get("utm_campaign"); - if (utmSource || utmCampaign) { + // Strip a stray "@TrackLink" suffix Listmonk can leave glued onto the last + // query value (e.g. utm_campaign=deficiency@TrackLink), so the arm/campaign + // reads cleanly in analytics. + var clean = function (v) { return (v || "").split("@TrackLink")[0]; }; + // The price-test arm rides in ?code= (BGQAX=20%, MLTGA=30%, the 40% code; the + // full-price control arm has no code). Coupon links frequently carry code with + // NO utm, so fire campaign-click whenever EITHER a utm OR a code is present, + // and always include the code so each A/B/C arm is attributable per click. + var couponCode = clean(params.get("code")); + if (utmSource || utmCampaign || couponCode) { track("campaign-click", { - source: utmSource || "", - campaign: utmCampaign || "", - medium: params.get("utm_medium") || "", + source: clean(utmSource), + campaign: clean(utmCampaign), + medium: clean(params.get("utm_medium")), + code: couponCode, page: location.pathname, }); }