Add 499 detail address fallback when CORES returns empty

CORES scraper sometimes returns empty address depending on server IP.
Now falls back to Headquarters Address from the FCC 499 filer detail
page, which is more reliable.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
justin 2026-04-27 22:07:55 -05:00
parent 783eeeb645
commit 6865da2004

View file

@ -118,7 +118,7 @@ router.get("/api/v1/fcc/lookup", async (req, res) => {
p2[k] = phase2Results[i].status === "fulfilled" ? (phase2Results[i] as PromiseFulfilledResult<unknown>).value : null; p2[k] = phase2Results[i].status === "fulfilled" ? (phase2Results[i] as PromiseFulfilledResult<unknown>).value : null;
}); });
let filerDetail = (p2.filerDetail as { current_as_of: string | null; comm_type: string | null; contributor: boolean | null; error: string | null }) || null; let filerDetail = (p2.filerDetail as { current_as_of: string | null; comm_type: string | null; contributor: boolean | null; hq_address: string | null; hq_city: string | null; hq_state: string | null; hq_zip: string | null; error: string | null }) || null;
let cpniResult = (p2.cpniResult as { filed: boolean; cert_year: number | null; date_filed: string | null; error: string | null }) || null; let cpniResult = (p2.cpniResult as { filed: boolean; cert_year: number | null; date_filed: string | null; error: string | null }) || null;
const coresResult = (p2.cores as CoresData) || { frn, entity_name: null, address: null, city: null, state: null, zip: null, status: null, red_light: null, error: "CORES lookup failed" } as CoresData; const coresResult = (p2.cores as CoresData) || { frn, entity_name: null, address: null, city: null, state: null, zip: null, status: null, red_light: null, error: "CORES lookup failed" } as CoresData;
const rmdResult = (p2.rmd as RmdData) || { found: false, business_name: null, frn: null, rmd_number: null, certification_date: null, implementation_type: null, contact_name: null, contact_email: null, removed: false, removal_reason: null, error: "RMD lookup failed" } as RmdData; const rmdResult = (p2.rmd as RmdData) || { found: false, business_name: null, frn: null, rmd_number: null, certification_date: null, implementation_type: null, contact_name: null, contact_email: null, removed: false, removal_reason: null, error: "RMD lookup failed" } as RmdData;
@ -782,10 +782,10 @@ router.get("/api/v1/fcc/lookup", async (req, res) => {
entity_name: entityName, entity_name: entityName,
cores: { cores: {
entity_name: coresResult.entity_name, entity_name: coresResult.entity_name,
address: coresResult.address, address: coresResult.address || filerDetail?.hq_address || null,
city: coresResult.city, city: coresResult.city || filerDetail?.hq_city || null,
state: coresResult.state, state: coresResult.state || filerDetail?.hq_state || null,
zip: coresResult.zip, zip: coresResult.zip || filerDetail?.hq_zip || null,
red_light: coresResult.red_light, red_light: coresResult.red_light,
error: coresResult.error, error: coresResult.error,
}, },
@ -1010,34 +1010,47 @@ async function fetchLocal499Filer(frn: string): Promise<any | null> {
} }
} }
async function fetch499Detail(filerId: string): Promise<{ current_as_of: string | null; comm_type: string | null; contributor: boolean | null; error: string | null }> { async function fetch499Detail(filerId: string): Promise<{
// Scrape the FCC 499 filer detail page to get "Registration Current as of" date current_as_of: string | null; comm_type: string | null; contributor: boolean | null;
// This tells us whether the carrier has filed their most recent 499-A hq_address: string | null; hq_city: string | null; hq_state: string | null; hq_zip: string | null;
error: string | null;
}> {
// Scrape the FCC 499 filer detail page for filing status + address
const url = `https://apps.fcc.gov/cgb/form499/499detail.cfm?FilerNum=${filerId}`; const url = `https://apps.fcc.gov/cgb/form499/499detail.cfm?FilerNum=${filerId}`;
try { try {
const resp = await fetch(url, { const resp = await fetch(url, {
signal: AbortSignal.timeout(10000), signal: AbortSignal.timeout(10000),
headers: { "Accept": "text/html", "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36" }, headers: { "Accept": "text/html", "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36" },
}); });
if (!resp.ok) return { current_as_of: null, comm_type: null, contributor: null, error: `499 detail returned ${resp.status}` }; if (!resp.ok) return { current_as_of: null, comm_type: null, contributor: null, hq_address: null, hq_city: null, hq_state: null, hq_zip: null, error: `499 detail returned ${resp.status}` };
const html = await resp.text(); const html = await resp.text();
// Extract "Registration Current as of: <b>4/1/2025</b>"
const currentMatch = html.match(/Registration Current as of:\s*<b>([^<]+)<\/b>/i); const currentMatch = html.match(/Registration Current as of:\s*<b>([^<]+)<\/b>/i);
const current_as_of = currentMatch ? currentMatch[1].trim() : null; const current_as_of = currentMatch ? currentMatch[1].trim() : null;
// Extract "Principal Communications Type: <b>Interconnected VoIP</b>"
const commMatch = html.match(/Principal Communications Type:\s*<b>([^<]+)<\/b>/i); const commMatch = html.match(/Principal Communications Type:\s*<b>([^<]+)<\/b>/i);
const comm_type = commMatch ? commMatch[1].trim() || null : null; const comm_type = commMatch ? commMatch[1].trim() || null : null;
// Extract "Universal Service Fund Contributor: <b>No</b>"
const contribMatch = html.match(/Universal Service Fund Contributor:\s*<b>([^<]+)<\/b>/i); const contribMatch = html.match(/Universal Service Fund Contributor:\s*<b>([^<]+)<\/b>/i);
const contributor = contribMatch ? contribMatch[1].trim().toLowerCase() === "yes" : null; const contributor = contribMatch ? contribMatch[1].trim().toLowerCase() === "yes" : null;
return { current_as_of, comm_type, contributor, error: null }; // Extract headquarters address (more reliable than CORES for some FRNs)
const addrMatch = html.match(/Headquarters Address:\s*<b>([^<]*)<\/b>/i);
const cityMatch = html.match(/Headquarters Address:[\s\S]*?City:\s*<b>([^<]*)<\/b>/i);
const stateMatch = html.match(/Headquarters Address:[\s\S]*?State:\s*<b>([^<]*)<\/b>/i);
const zipMatch = html.match(/Headquarters Address:[\s\S]*?ZIP Code:\s*<b>([^<]*)<\/b>/i);
return {
current_as_of, comm_type, contributor,
hq_address: addrMatch ? addrMatch[1].trim() || null : null,
hq_city: cityMatch ? cityMatch[1].trim() || null : null,
hq_state: stateMatch ? stateMatch[1].trim() || null : null,
hq_zip: zipMatch ? zipMatch[1].trim() || null : null,
error: null,
};
} catch (err: any) { } catch (err: any) {
return { current_as_of: null, comm_type: null, contributor: null, error: err.message }; return { current_as_of: null, comm_type: null, contributor: null, hq_address: null, hq_city: null, hq_state: null, hq_zip: null, error: err.message };
} }
} }