From 42a7beb65569dc6609f5bc6935f6d9ac920a322c Mon Sep 17 00:00:00 2001 From: justin Date: Wed, 1 Jul 2026 08:39:53 -0500 Subject: [PATCH] corp-status: surface annual-report-overdue signal (backfilled for CT) entity_cache had CT status but no annual-report clock, so the corporation-check tool told overdue-but-active CT businesses they were fine. Now: - backfill_ct_annual_report_due.py: pulls annual_report_due_date from the live CT Socrata registry into entity_cache.annual_report_due_date (keyed by accountnumber=entity_number). Ran: 577,562 CT rows populated, 235,875 overdue. - corp-status API: reads annual_report_due_date, returns annual_report_overdue + days_overdue, and bumps years_behind to >=1 for an overdue-but-active entity so the remediation cost isn't $0. This makes the tool match the CT campaign hook. --- api/src/routes/corp-status.ts | 35 ++++++- scripts/backfill_ct_annual_report_due.py | 126 +++++++++++++++++++++++ 2 files changed, 160 insertions(+), 1 deletion(-) create mode 100644 scripts/backfill_ct_annual_report_due.py diff --git a/api/src/routes/corp-status.ts b/api/src/routes/corp-status.ts index d003811..37978e3 100644 --- a/api/src/routes/corp-status.ts +++ b/api/src/routes/corp-status.ts @@ -43,6 +43,11 @@ interface CorpStatusResult { state: string | null; // Computed fields years_behind: number; + // Annual-report deadline signal (from entity_cache.annual_report_due_date, + // currently backfilled for CT). null when we have no due date for this entity. + annual_report_due_date: string | null; + annual_report_overdue: boolean; + annual_report_days_overdue: number | null; annual_report_fee_cents: number; annual_report_frequency: string | null; cost_per_year_cents: number; @@ -71,7 +76,8 @@ export async function lookupCorpStatus( try { const exact = await pool.query( `SELECT entity_name, entity_number, entity_type, status, - formation_date, dissolution_date, registered_agent, state + formation_date, dissolution_date, registered_agent, state, + annual_report_due_date FROM entity_cache WHERE state = $1 AND LOWER(entity_name) = LOWER($2) LIMIT 1`, @@ -86,6 +92,7 @@ export async function lookupCorpStatus( const fuzzy = await pool.query( `SELECT entity_name, entity_number, entity_type, status, formation_date, dissolution_date, registered_agent, state, + annual_report_due_date, similarity(entity_name, $2) AS sim FROM entity_cache WHERE state = $1 AND similarity(entity_name, $2) > 0.8 @@ -129,6 +136,7 @@ export async function lookupCorpStatus( dissolution_date: null, registered_agent: liveData.registered_agent || null, state, + annual_report_due_date: null, }; } } catch { @@ -188,6 +196,28 @@ export async function lookupCorpStatus( } } + // Annual-report deadline signal (from entity_cache.annual_report_due_date, + // backfilled for CT). This catches the common case the status field misses: a + // business still "ACTIVE" whose annual report is nonetheless past due. When + // overdue, ensure yearsBehind reflects at least the current lapse so the cost + // estimate isn't $0 for a genuinely-overdue (but not-yet-forfeited) entity. + const dueRaw = row.annual_report_due_date + ? new Date(row.annual_report_due_date as string) + : null; + const annualReportDueDate = dueRaw ? dueRaw.toISOString().slice(0, 10) : null; + let annualReportOverdue = false; + let annualReportDaysOverdue: number | null = null; + if (dueRaw && !Number.isNaN(dueRaw.getTime())) { + const days = Math.floor((Date.now() - dueRaw.getTime()) / 86400000); + annualReportDaysOverdue = days; + annualReportOverdue = days > 0; + if (annualReportOverdue && yearsBehind === 0) { + // Overdue-but-active: at least the current year's report is owed. Add a + // second year only once it's more than ~a year late. + yearsBehind = days > 400 ? 2 : 1; + } + } + // Cost calculations const costPerYear = annualFee + FILING_MARKUP_CENTS; const totalCatchup = yearsBehind * costPerYear; @@ -226,6 +256,9 @@ export async function lookupCorpStatus( principal_address: (row.principal_address as string) || null, state, years_behind: yearsBehind, + annual_report_due_date: annualReportDueDate, + annual_report_overdue: annualReportOverdue, + annual_report_days_overdue: annualReportDaysOverdue, annual_report_fee_cents: annualFee, annual_report_frequency: frequency, cost_per_year_cents: costPerYear, diff --git a/scripts/backfill_ct_annual_report_due.py b/scripts/backfill_ct_annual_report_due.py new file mode 100644 index 0000000..e5450a9 --- /dev/null +++ b/scripts/backfill_ct_annual_report_due.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python3 +""" +backfill_ct_annual_report_due.py — Populate entity_cache.annual_report_due_date +for Connecticut, so the /tools/corporation-check tool can show the +annual-report-overdue signal (the CT campaign's hook). + +entity_cache has CT status but no annual-report clock. This pulls +annual_report_due_date from the live CT Socrata registry (data.ct.gov/n7gp-d28j), +keyed by accountnumber == entity_cache.entity_number, and writes it back. + +Adds the column if missing. Idempotent: re-runnable. + + python3 scripts/backfill_ct_annual_report_due.py # full CT backfill + python3 scripts/backfill_ct_annual_report_due.py --limit 5000 +""" +import argparse +import os +import sys +import time + +import psycopg2 +import psycopg2.extras +import requests + +DB_URL = os.getenv("DATABASE_URL", "postgresql://pw@localhost/performancewest") +SOCRATA = "https://data.ct.gov/resource/n7gp-d28j.json" +PAGE = 50000 + + +def fetch_ct_due_dates(limit=None): + """Yield (accountnumber, annual_report_due_date) for CT entities that have both.""" + off = 0 + got = 0 + s = requests.Session() + while True: + params = { + "$select": "accountnumber,annual_report_due_date", + "$where": "annual_report_due_date IS NOT NULL AND accountnumber IS NOT NULL", + "$limit": PAGE, + "$offset": off, + "$order": "accountnumber", + } + r = s.get(SOCRATA, params=params, timeout=60) + r.raise_for_status() + rows = r.json() + if not rows: + break + for row in rows: + acct = (row.get("accountnumber") or "").strip() + due = (row.get("annual_report_due_date") or "")[:10] # YYYY-MM-DD + if acct and due: + yield acct, due + got += 1 + if limit and got >= limit: + return + off += len(rows) + print(f" fetched {off:,} CT rows...", file=sys.stderr) + if len(rows) < PAGE: + break + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--limit", type=int, default=None) + args = ap.parse_args() + + conn = psycopg2.connect(DB_URL) + conn.autocommit = False + cur = conn.cursor() + + # 1. add column if missing + cur.execute(""" + ALTER TABLE entity_cache + ADD COLUMN IF NOT EXISTS annual_report_due_date date + """) + conn.commit() + print("column ensured: entity_cache.annual_report_due_date") + + # 2. stage the (accountnumber, due) pairs, then bulk-update by join + cur.execute("DROP TABLE IF EXISTS _ct_due_stage") + cur.execute("CREATE TEMP TABLE _ct_due_stage(entity_number text, due date)") + + batch = [] + total = 0 + t0 = time.time() + for acct, due in fetch_ct_due_dates(args.limit): + batch.append((acct, due)) + if len(batch) >= 10000: + psycopg2.extras.execute_values( + cur, "INSERT INTO _ct_due_stage(entity_number, due) VALUES %s", batch) + total += len(batch); batch = [] + if batch: + psycopg2.extras.execute_values( + cur, "INSERT INTO _ct_due_stage(entity_number, due) VALUES %s", batch) + total += len(batch) + conn.commit() + print(f"staged {total:,} CT due-date rows in {time.time()-t0:.0f}s") + + # 3. index + update entity_cache + cur.execute("CREATE INDEX ON _ct_due_stage(entity_number)") + cur.execute(""" + UPDATE entity_cache e + SET annual_report_due_date = s.due + FROM _ct_due_stage s + WHERE e.state = 'CT' AND e.entity_number = s.entity_number + AND (e.annual_report_due_date IS DISTINCT FROM s.due) + """) + updated = cur.rowcount + conn.commit() + print(f"updated {updated:,} CT entity_cache rows with annual_report_due_date") + + # 4. verify + cur.execute(""" + SELECT count(*) FILTER (WHERE annual_report_due_date IS NOT NULL), + count(*) FILTER (WHERE annual_report_due_date < now()), + count(*) + FROM entity_cache WHERE state = 'CT' + """) + have, overdue, tot = cur.fetchone() + print(f"CT entity_cache: {have:,} with due date, {overdue:,} overdue, {tot:,} total") + cur.close(); conn.close() + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())