analytics: attribute price-test arm per click + A/B/C scoreboard
So we can see whether a discount helps the CTA (clicks) AND the sale (paid orders), per arm: 1. pw-analytics.js: include the ?code= (the price-test arm) on the campaign-click Umami event, and fire it when EITHER a utm OR a code is present (coupon links often carry code with no utm, so those human clicks were being dropped). Also strip a stray @TrackLink suffix Listmonk glues onto the last query value. 2. coupon_ab_scoreboard.py: one report combining HUMAN clicks (Umami, already bot-filtered) and PAID conversions (compliance_orders) per arm. Discount arms map by code->pct (campaign-daily:<date>:<pct> marker); the no-code control arm is recovered by re-hashing customer_email the same way the builder bucketed it. Prints clicks, paid orders, revenue, and click->order per arm.
This commit is contained in:
justin
parent
ac10b4bd22
commit
60d5d3b9d8
2 changed files with 184 additions and 4 deletions
170
scripts/coupon_ab_scoreboard.py
Normal file
170
scripts/coupon_ab_scoreboard.py
Normal file
|
|
@ -0,0 +1,170 @@
|
|||
#!/usr/bin/env python3
|
||||
"""
|
||||
coupon_ab_scoreboard.py — The trucking price A/B/C scoreboard.
|
||||
|
||||
Answers two questions per discount arm (20% / 30% / 40% / full-price control):
|
||||
1. Does the discount drive the CTA click? -> HUMAN clicks from Umami
|
||||
(campaign-click events, already bot-filtered by pw-bot-filter.js), attributed
|
||||
to an arm by the ?code= the link carried.
|
||||
2. Does it drive the sale? -> PAID orders from compliance_orders,
|
||||
attributed to an arm by discount_code (discount arms) or, for the control
|
||||
arm (no code), by re-hashing customer_email the SAME way the builder bucketed
|
||||
it (sha256(email) % len(arms)).
|
||||
|
||||
Both DBs live on the same Postgres. Reads:
|
||||
- umami DB: website_event (event_name='campaign-click', has url_query/event_data)
|
||||
- performancewest DB: discount_codes (campaign-daily:<date>:<pct> marker -> arm),
|
||||
compliance_orders (paid conversions + customer_email)
|
||||
|
||||
Usage (run on the host, or anywhere with psql access to the containers):
|
||||
python3 scripts/coupon_ab_scoreboard.py # last 14 days
|
||||
python3 scripts/coupon_ab_scoreboard.py --days 30
|
||||
python3 scripts/coupon_ab_scoreboard.py --arms 20,30,40,0
|
||||
|
||||
The arm set MUST match CAMPAIGN_COUPON_AB_PCTS so the control re-hash lines up.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from collections import defaultdict
|
||||
|
||||
# How to reach the two databases. On the host we exec into the postgres
|
||||
# container; override with env if running elsewhere.
|
||||
PG_CONTAINER = os.getenv("PW_PG_CONTAINER", "performancewest-api-postgres-1")
|
||||
UMAMI_CONTAINER = os.getenv("PW_UMAMI_CONTAINER", "performancewest-umami-postgres-1")
|
||||
PW_DB = os.getenv("PW_DB", "performancewest")
|
||||
UMAMI_DB = os.getenv("PW_UMAMI_DB", "umami")
|
||||
PW_USER = os.getenv("PW_DB_USER", "pw")
|
||||
UMAMI_USER = os.getenv("PW_UMAMI_USER", "umami")
|
||||
|
||||
|
||||
def psql(container: str, db: str, user: str, sql: str) -> list[list[str]]:
|
||||
"""Run SQL via `docker exec ... psql -tAF$'\\t'` and return rows of fields."""
|
||||
out = subprocess.run(
|
||||
["docker", "exec", "-i", container, "psql", "-U", user, "-d", db,
|
||||
"-tAF", "\t", "-c", sql],
|
||||
capture_output=True, text=True, timeout=60,
|
||||
)
|
||||
if out.returncode != 0:
|
||||
raise RuntimeError(f"psql {db} failed: {out.stderr[:300]}")
|
||||
return [line.split("\t") for line in out.stdout.splitlines() if line.strip()]
|
||||
|
||||
|
||||
def arm_for_email(email: str, arms: list[int]) -> int:
|
||||
"""Reproduce the builder's deterministic bucketing (pick_coupon_for_email).
|
||||
|
||||
pcts are sorted; bucket = sha256(lower(email)) % len(pcts).
|
||||
"""
|
||||
pcts = sorted(arms)
|
||||
if len(pcts) == 1:
|
||||
return pcts[0]
|
||||
h = hashlib.sha256((email or "").strip().lower().encode()).hexdigest()
|
||||
return pcts[int(h, 16) % len(pcts)]
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--days", type=int, default=14)
|
||||
ap.add_argument("--arms", default=os.getenv("CAMPAIGN_COUPON_AB_PCTS", "20,30,40,0"),
|
||||
help="comma list matching CAMPAIGN_COUPON_AB_PCTS, e.g. 20,30,40,0")
|
||||
args = ap.parse_args()
|
||||
|
||||
arms = [int(a) for a in args.arms.split(",") if a.strip().lstrip("-").isdigit()]
|
||||
if not arms:
|
||||
print("no arms parsed", file=sys.stderr)
|
||||
return 1
|
||||
days = args.days
|
||||
|
||||
# ── 1. Map every daily A/B code -> its arm pct (from the description marker) ──
|
||||
code_rows = psql(PG_CONTAINER, PW_DB, PW_USER, f"""
|
||||
SELECT upper(code), discount_value
|
||||
FROM discount_codes
|
||||
WHERE description LIKE 'campaign-daily:%'
|
||||
AND created_at > now() - interval '{days} days'
|
||||
""")
|
||||
code_to_pct = {c: int(v) for c, v in code_rows}
|
||||
|
||||
# ── 2. HUMAN clicks per arm (Umami campaign-click, bot-filtered already) ──────
|
||||
# The arm rides in event_data 'code' (new) OR the url_query ?code= (older). We
|
||||
# read both and map code->pct. Clicks with no code = control arm (pct 0) IF the
|
||||
# control arm exists, but a no-code click could also just be a non-coupon link;
|
||||
# we only count no-code clicks toward control when they hit an /order/ page.
|
||||
click_rows = psql(UMAMI_CONTAINER, UMAMI_DB, UMAMI_USER, f"""
|
||||
SELECT lower(coalesce(url_path,'')) AS path,
|
||||
upper(coalesce(url_query,'')) AS q
|
||||
FROM website_event
|
||||
WHERE event_name = 'campaign-click'
|
||||
AND created_at > now() - interval '{days} days'
|
||||
""")
|
||||
clicks = defaultdict(int)
|
||||
for path, q in click_rows:
|
||||
code = ""
|
||||
if "CODE=" in q:
|
||||
# extract CODE=XXXXX from the query string
|
||||
seg = q.split("CODE=", 1)[1]
|
||||
for sep in ("&", "@", " "):
|
||||
seg = seg.split(sep, 1)[0]
|
||||
code = seg
|
||||
if code and code in code_to_pct:
|
||||
clicks[code_to_pct[code]] += 1
|
||||
elif not code and "/ORDER/" in path.upper() and 0 in arms:
|
||||
clicks[0] += 1 # no-code order click = full-price control arm
|
||||
|
||||
# ── 3. PAID conversions per arm ──────────────────────────────────────────────
|
||||
order_rows = psql(PG_CONTAINER, PW_DB, PW_USER, f"""
|
||||
SELECT upper(coalesce(discount_code,'')) AS code,
|
||||
lower(coalesce(customer_email,'')) AS email,
|
||||
service_fee_cents
|
||||
FROM compliance_orders
|
||||
WHERE payment_status = 'paid'
|
||||
AND created_at > now() - interval '{days} days'
|
||||
AND customer_email !~* 'performancewest|carrierone|e2e|test|@example|synthetic|pipeline'
|
||||
""")
|
||||
paid = defaultdict(int)
|
||||
revenue = defaultdict(int)
|
||||
for code, email, fee in order_rows:
|
||||
fee_cents = int(fee) if fee and fee.isdigit() else 0
|
||||
if code and code in code_to_pct:
|
||||
arm = code_to_pct[code]
|
||||
elif code and code.isalpha() and len(code) == 5:
|
||||
# an A/B code from outside the window; skip rather than misattribute
|
||||
continue
|
||||
else:
|
||||
# control arm (no code) -> recover the arm by re-hashing the email
|
||||
arm = arm_for_email(email, arms)
|
||||
paid[arm] += 1
|
||||
revenue[arm] += fee_cents
|
||||
|
||||
# ── 4. Print the scoreboard ──────────────────────────────────────────────────
|
||||
print(f"\n=== Trucking price A/B/C scoreboard (last {days} days) ===")
|
||||
print(f"arms: {arms} (control = full price, no code)\n")
|
||||
hdr = f"{'arm':>8} {'human_clicks':>13} {'paid_orders':>12} {'revenue':>10} {'click→order':>12}"
|
||||
print(hdr)
|
||||
print("-" * len(hdr))
|
||||
for pct in sorted(arms):
|
||||
label = f"{pct}% off" if pct > 0 else "control"
|
||||
c = clicks.get(pct, 0)
|
||||
o = paid.get(pct, 0)
|
||||
rev = revenue.get(pct, 0) / 100.0
|
||||
cvr = f"{(o / c * 100):.1f}%" if c else "-"
|
||||
print(f"{label:>8} {c:>13} {o:>12} {('$%.0f' % rev):>10} {cvr:>12}")
|
||||
tot_c = sum(clicks.values()); tot_o = sum(paid.values()); tot_r = sum(revenue.values()) / 100.0
|
||||
print("-" * len(hdr))
|
||||
print(f"{'TOTAL':>8} {tot_c:>13} {tot_o:>12} {('$%.0f' % tot_r):>10} "
|
||||
f"{((tot_o/tot_c*100) if tot_c else 0):>11.1f}%")
|
||||
print("\nClicks are HUMAN (bot-filtered via Umami before-send). Conversions are")
|
||||
print("PAID Stripe orders. Control conversions recovered by email re-hash.")
|
||||
if tot_o < 30:
|
||||
print(f"\n NOTE: only {tot_o} paid conversions in window — not yet enough for a")
|
||||
print(" statistically meaningful price comparison (want ~hundreds/arm). Clicks")
|
||||
print(" are the early CTA signal; conversions are the real answer once volume builds.")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
|
|
@ -107,11 +107,21 @@
|
|||
var params = new URLSearchParams(location.search);
|
||||
var utmSource = params.get("utm_source");
|
||||
var utmCampaign = params.get("utm_campaign");
|
||||
if (utmSource || utmCampaign) {
|
||||
// Strip a stray "@TrackLink" suffix Listmonk can leave glued onto the last
|
||||
// query value (e.g. utm_campaign=deficiency@TrackLink), so the arm/campaign
|
||||
// reads cleanly in analytics.
|
||||
var clean = function (v) { return (v || "").split("@TrackLink")[0]; };
|
||||
// The price-test arm rides in ?code= (BGQAX=20%, MLTGA=30%, the 40% code; the
|
||||
// full-price control arm has no code). Coupon links frequently carry code with
|
||||
// NO utm, so fire campaign-click whenever EITHER a utm OR a code is present,
|
||||
// and always include the code so each A/B/C arm is attributable per click.
|
||||
var couponCode = clean(params.get("code"));
|
||||
if (utmSource || utmCampaign || couponCode) {
|
||||
track("campaign-click", {
|
||||
source: utmSource || "",
|
||||
campaign: utmCampaign || "",
|
||||
medium: params.get("utm_medium") || "",
|
||||
source: clean(utmSource),
|
||||
campaign: clean(utmCampaign),
|
||||
medium: clean(params.get("utm_medium")),
|
||||
code: couponCode,
|
||||
page: location.pathname,
|
||||
});
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue