corp-status: surface annual-report-overdue signal (backfilled for CT)
entity_cache had CT status but no annual-report clock, so the corporation-check tool told overdue-but-active CT businesses they were fine. Now: - backfill_ct_annual_report_due.py: pulls annual_report_due_date from the live CT Socrata registry into entity_cache.annual_report_due_date (keyed by accountnumber=entity_number). Ran: 577,562 CT rows populated, 235,875 overdue. - corp-status API: reads annual_report_due_date, returns annual_report_overdue + days_overdue, and bumps years_behind to >=1 for an overdue-but-active entity so the remediation cost isn't $0. This makes the tool match the CT campaign hook.
This commit is contained in:
justin
parent
b016caecea
commit
42a7beb655
2 changed files with 160 additions and 1 deletions
|
|
@ -43,6 +43,11 @@ interface CorpStatusResult {
|
|||
state: string | null;
|
||||
// Computed fields
|
||||
years_behind: number;
|
||||
// Annual-report deadline signal (from entity_cache.annual_report_due_date,
|
||||
// currently backfilled for CT). null when we have no due date for this entity.
|
||||
annual_report_due_date: string | null;
|
||||
annual_report_overdue: boolean;
|
||||
annual_report_days_overdue: number | null;
|
||||
annual_report_fee_cents: number;
|
||||
annual_report_frequency: string | null;
|
||||
cost_per_year_cents: number;
|
||||
|
|
@ -71,7 +76,8 @@ export async function lookupCorpStatus(
|
|||
try {
|
||||
const exact = await pool.query(
|
||||
`SELECT entity_name, entity_number, entity_type, status,
|
||||
formation_date, dissolution_date, registered_agent, state
|
||||
formation_date, dissolution_date, registered_agent, state,
|
||||
annual_report_due_date
|
||||
FROM entity_cache
|
||||
WHERE state = $1 AND LOWER(entity_name) = LOWER($2)
|
||||
LIMIT 1`,
|
||||
|
|
@ -86,6 +92,7 @@ export async function lookupCorpStatus(
|
|||
const fuzzy = await pool.query(
|
||||
`SELECT entity_name, entity_number, entity_type, status,
|
||||
formation_date, dissolution_date, registered_agent, state,
|
||||
annual_report_due_date,
|
||||
similarity(entity_name, $2) AS sim
|
||||
FROM entity_cache
|
||||
WHERE state = $1 AND similarity(entity_name, $2) > 0.8
|
||||
|
|
@ -129,6 +136,7 @@ export async function lookupCorpStatus(
|
|||
dissolution_date: null,
|
||||
registered_agent: liveData.registered_agent || null,
|
||||
state,
|
||||
annual_report_due_date: null,
|
||||
};
|
||||
}
|
||||
} catch {
|
||||
|
|
@ -188,6 +196,28 @@ export async function lookupCorpStatus(
|
|||
}
|
||||
}
|
||||
|
||||
// Annual-report deadline signal (from entity_cache.annual_report_due_date,
|
||||
// backfilled for CT). This catches the common case the status field misses: a
|
||||
// business still "ACTIVE" whose annual report is nonetheless past due. When
|
||||
// overdue, ensure yearsBehind reflects at least the current lapse so the cost
|
||||
// estimate isn't $0 for a genuinely-overdue (but not-yet-forfeited) entity.
|
||||
const dueRaw = row.annual_report_due_date
|
||||
? new Date(row.annual_report_due_date as string)
|
||||
: null;
|
||||
const annualReportDueDate = dueRaw ? dueRaw.toISOString().slice(0, 10) : null;
|
||||
let annualReportOverdue = false;
|
||||
let annualReportDaysOverdue: number | null = null;
|
||||
if (dueRaw && !Number.isNaN(dueRaw.getTime())) {
|
||||
const days = Math.floor((Date.now() - dueRaw.getTime()) / 86400000);
|
||||
annualReportDaysOverdue = days;
|
||||
annualReportOverdue = days > 0;
|
||||
if (annualReportOverdue && yearsBehind === 0) {
|
||||
// Overdue-but-active: at least the current year's report is owed. Add a
|
||||
// second year only once it's more than ~a year late.
|
||||
yearsBehind = days > 400 ? 2 : 1;
|
||||
}
|
||||
}
|
||||
|
||||
// Cost calculations
|
||||
const costPerYear = annualFee + FILING_MARKUP_CENTS;
|
||||
const totalCatchup = yearsBehind * costPerYear;
|
||||
|
|
@ -226,6 +256,9 @@ export async function lookupCorpStatus(
|
|||
principal_address: (row.principal_address as string) || null,
|
||||
state,
|
||||
years_behind: yearsBehind,
|
||||
annual_report_due_date: annualReportDueDate,
|
||||
annual_report_overdue: annualReportOverdue,
|
||||
annual_report_days_overdue: annualReportDaysOverdue,
|
||||
annual_report_fee_cents: annualFee,
|
||||
annual_report_frequency: frequency,
|
||||
cost_per_year_cents: costPerYear,
|
||||
|
|
|
|||
126
scripts/backfill_ct_annual_report_due.py
Normal file
126
scripts/backfill_ct_annual_report_due.py
Normal file
|
|
@ -0,0 +1,126 @@
|
|||
#!/usr/bin/env python3
|
||||
"""
|
||||
backfill_ct_annual_report_due.py — Populate entity_cache.annual_report_due_date
|
||||
for Connecticut, so the /tools/corporation-check tool can show the
|
||||
annual-report-overdue signal (the CT campaign's hook).
|
||||
|
||||
entity_cache has CT status but no annual-report clock. This pulls
|
||||
annual_report_due_date from the live CT Socrata registry (data.ct.gov/n7gp-d28j),
|
||||
keyed by accountnumber == entity_cache.entity_number, and writes it back.
|
||||
|
||||
Adds the column if missing. Idempotent: re-runnable.
|
||||
|
||||
python3 scripts/backfill_ct_annual_report_due.py # full CT backfill
|
||||
python3 scripts/backfill_ct_annual_report_due.py --limit 5000
|
||||
"""
|
||||
import argparse
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
|
||||
import psycopg2
|
||||
import psycopg2.extras
|
||||
import requests
|
||||
|
||||
DB_URL = os.getenv("DATABASE_URL", "postgresql://pw@localhost/performancewest")
|
||||
SOCRATA = "https://data.ct.gov/resource/n7gp-d28j.json"
|
||||
PAGE = 50000
|
||||
|
||||
|
||||
def fetch_ct_due_dates(limit=None):
|
||||
"""Yield (accountnumber, annual_report_due_date) for CT entities that have both."""
|
||||
off = 0
|
||||
got = 0
|
||||
s = requests.Session()
|
||||
while True:
|
||||
params = {
|
||||
"$select": "accountnumber,annual_report_due_date",
|
||||
"$where": "annual_report_due_date IS NOT NULL AND accountnumber IS NOT NULL",
|
||||
"$limit": PAGE,
|
||||
"$offset": off,
|
||||
"$order": "accountnumber",
|
||||
}
|
||||
r = s.get(SOCRATA, params=params, timeout=60)
|
||||
r.raise_for_status()
|
||||
rows = r.json()
|
||||
if not rows:
|
||||
break
|
||||
for row in rows:
|
||||
acct = (row.get("accountnumber") or "").strip()
|
||||
due = (row.get("annual_report_due_date") or "")[:10] # YYYY-MM-DD
|
||||
if acct and due:
|
||||
yield acct, due
|
||||
got += 1
|
||||
if limit and got >= limit:
|
||||
return
|
||||
off += len(rows)
|
||||
print(f" fetched {off:,} CT rows...", file=sys.stderr)
|
||||
if len(rows) < PAGE:
|
||||
break
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--limit", type=int, default=None)
|
||||
args = ap.parse_args()
|
||||
|
||||
conn = psycopg2.connect(DB_URL)
|
||||
conn.autocommit = False
|
||||
cur = conn.cursor()
|
||||
|
||||
# 1. add column if missing
|
||||
cur.execute("""
|
||||
ALTER TABLE entity_cache
|
||||
ADD COLUMN IF NOT EXISTS annual_report_due_date date
|
||||
""")
|
||||
conn.commit()
|
||||
print("column ensured: entity_cache.annual_report_due_date")
|
||||
|
||||
# 2. stage the (accountnumber, due) pairs, then bulk-update by join
|
||||
cur.execute("DROP TABLE IF EXISTS _ct_due_stage")
|
||||
cur.execute("CREATE TEMP TABLE _ct_due_stage(entity_number text, due date)")
|
||||
|
||||
batch = []
|
||||
total = 0
|
||||
t0 = time.time()
|
||||
for acct, due in fetch_ct_due_dates(args.limit):
|
||||
batch.append((acct, due))
|
||||
if len(batch) >= 10000:
|
||||
psycopg2.extras.execute_values(
|
||||
cur, "INSERT INTO _ct_due_stage(entity_number, due) VALUES %s", batch)
|
||||
total += len(batch); batch = []
|
||||
if batch:
|
||||
psycopg2.extras.execute_values(
|
||||
cur, "INSERT INTO _ct_due_stage(entity_number, due) VALUES %s", batch)
|
||||
total += len(batch)
|
||||
conn.commit()
|
||||
print(f"staged {total:,} CT due-date rows in {time.time()-t0:.0f}s")
|
||||
|
||||
# 3. index + update entity_cache
|
||||
cur.execute("CREATE INDEX ON _ct_due_stage(entity_number)")
|
||||
cur.execute("""
|
||||
UPDATE entity_cache e
|
||||
SET annual_report_due_date = s.due
|
||||
FROM _ct_due_stage s
|
||||
WHERE e.state = 'CT' AND e.entity_number = s.entity_number
|
||||
AND (e.annual_report_due_date IS DISTINCT FROM s.due)
|
||||
""")
|
||||
updated = cur.rowcount
|
||||
conn.commit()
|
||||
print(f"updated {updated:,} CT entity_cache rows with annual_report_due_date")
|
||||
|
||||
# 4. verify
|
||||
cur.execute("""
|
||||
SELECT count(*) FILTER (WHERE annual_report_due_date IS NOT NULL),
|
||||
count(*) FILTER (WHERE annual_report_due_date < now()),
|
||||
count(*)
|
||||
FROM entity_cache WHERE state = 'CT'
|
||||
""")
|
||||
have, overdue, tot = cur.fetchone()
|
||||
print(f"CT entity_cache: {have:,} with due date, {overdue:,} overdue, {tot:,} total")
|
||||
cur.close(); conn.close()
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Loading…
Add table
Add a link
Reference in a new issue