justin/new-site
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Initial commit — Performance West telecom compliance platform

Browse source 
Includes: API (Express/TypeScript), Astro site, Python workers,
document generators, FCC compliance tools, Canada CRTC formation,
Ansible infrastructure, and deployment scripts.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
justin 2026-04-27 06:54:22 -05:00
commit f8cd37ac8c
1823 changed files with 145167 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

60
scripts/workers/icc_adapters/__init__.py Normal file
View file

@ -0,0 +1,60 @@
"""Inter-Carrier Compensation (ICC) revenue adapters.
Each adapter parses one carrier-specific interconnection or settlement
artifact format into a normalized ``IccRevenueLine`` stream consumed by
``scripts.workers.icc_ingester``.
Dispatch is driven by ``icc_ingestion_uploads.source_format`` (the CHECK
constraint on that column enumerates the supported slugs; this module's
``ADAPTERS`` dict keys must stay in sync).
Contract — see ``common.py``:
BaseICCAdapter.iter_rows(local_path) -> Iterator[IccRevenueLine]
"""
from __future__ import annotations
from typing import Dict, Optional, Type
from .common import BaseICCAdapter, IccRevenueLine, ValidationError
from .cabs_bos_adapter import CABSBOSAdapter
from .edi_810_adapter import EDI810Adapter
from .iconectiv_8yy_adapter import IconectivQRYAdapter
from .international_settlement import InternationalSettlementAdapter
from .wholesale_sip_csv import WholesaleSIPCSVAdapter
from .carrier_invoice_pdf import CarrierInvoicePDFAdapter
ADAPTERS: Dict[str, Type[BaseICCAdapter]] = {
"cabs_bos": CABSBOSAdapter,
"edi_810": EDI810Adapter,
"8yy_qry": IconectivQRYAdapter,
"itu_tas": InternationalSettlementAdapter,
"icss": InternationalSettlementAdapter,
"wholesale_sip_csv": WholesaleSIPCSVAdapter,
"carrier_invoice_pdf": CarrierInvoicePDFAdapter,
}
def get_adapter(source_format: str) -> Optional[Type[BaseICCAdapter]]:
"""Resolve an ``icc_ingestion_uploads.source_format`` slug into its adapter class.
Returns ``None`` when the slug is unknown; callers should mark the
upload ``failed`` with an explanatory error in that case.
"""
return ADAPTERS.get(source_format)
__all__ = [
"ADAPTERS",
"get_adapter",
"BaseICCAdapter",
"IccRevenueLine",
"ValidationError",
"CABSBOSAdapter",
"EDI810Adapter",
"IconectivQRYAdapter",
"InternationalSettlementAdapter",
"WholesaleSIPCSVAdapter",
"CarrierInvoicePDFAdapter",
]

179
scripts/workers/icc_adapters/cabs_bos_adapter.py Normal file
View file

@ -0,0 +1,179 @@
"""CABS BOS (Billing Output Specification) adapter.
Parses the fixed-width record-oriented CABS BOS format used by ILECs and
interexchange carriers to bill switched- and special-access revenue to
interconnecting carriers. The CABS BOS (Telcordia / Bellcore BR-902-900-
000) carries many record types; this adapter is a **best-effort**
interpreter of the most common ones.
Supported record types
----------------------
``01`` Header — billing company + bill period (context only)
``10`` Switched-Access Revenue detail
positions 5..14 OCN of interconnecting carrier (10 chars)
positions 15..20 Billing period YYMMDD (period-end)
positions 50..59 Minutes of use (right-justified, 10 digits)
positions 60..69 Revenue, implied 2 decimals (10 digits)
``20`` Special-Access Revenue detail
positions 5..14 OCN
positions 15..20 Billing period YYMMDD
positions 40..59 Circuit identifier (free-form)
positions 60..69 Revenue, implied 2 decimals
``90`` Trailer — record counts (context only)
Deferred / not parsed
---------------------
* USOC / rate-element detail lines (record 15, 25)
* Adjustments (record 40) and taxes (record 50)
* End-Office-vs-Tandem jurisdictional apportionment on record 10 —
handled downstream by ``icc_499a_line_mapping.jurisdiction_split``
* Multi-line circuit detail continuation records (11, 21)
Real CABS BOS files have many carrier-specific variants; downstream jobs
that need rate-element granularity should fall back to the raw PDF/EDI
or request an ASR-formatted companion file.
"""
from __future__ import annotations
import logging
from typing import Iterator
from .common import BaseICCAdapter, IccRevenueLine, ValidationError
logger = logging.getLogger(__name__)
class CABSBOSAdapter(BaseICCAdapter):
SOURCE_FORMAT = "cabs_bos"
# 1-based column slices drawn from the Telcordia BR-902-900-000 layout.
# Subclass tuples are (start, end_inclusive) in human positions; we
# convert to 0-based half-open slices via ``_slice()``.
_RECTYPE = (1, 2)
_OCN = (5, 14)
_BILL_PERIOD = (15, 20)
_MOU_R10 = (50, 59)
_REVENUE_R10 = (60, 69)
_CIRCUIT_R20 = (40, 59)
_REVENUE_R20 = (60, 69)
@staticmethod
def _slice(line: str, span: tuple[int, int]) -> str:
start, end = span
return line[start - 1:end]
@staticmethod
def _period_to_quarter(period_yymmdd: str) -> int | None:
"""Convert YYMMDD billing-period-end into a 1..4 quarter index."""
if not period_yymmdd or len(period_yymmdd) < 4:
return None
try:
month = int(period_yymmdd[2:4])
except ValueError:
return None
if 1 <= month <= 3:
return 1
if 4 <= month <= 6:
return 2
if 7 <= month <= 9:
return 3
if 10 <= month <= 12:
return 4
return None
@classmethod
def _implied_cents(cls, raw: str) -> int:
"""CABS BOS revenue fields are zero-padded 10-digit integers with
an implied 2-decimal scale, so '0000123456' → $1,234.56 → 123456 cents.
"""
cleaned = raw.strip()
if not cleaned:
return 0
# Handle explicit sign in leading position (some carriers use it)
negative = False
if cleaned[0] in "+-":
negative = cleaned[0] == "-"
cleaned = cleaned[1:]
if not cleaned.isdigit():
raise ValidationError("bad_revenue", f"CABS revenue non-numeric: {raw!r}")
cents = int(cleaned)
return -cents if negative else cents
def iter_rows(self, local_path: str) -> Iterator[IccRevenueLine]:
with open(local_path, "r", encoding="latin-1", errors="replace") as fh:
for lineno, line in enumerate(fh, start=1):
# Strip CRLF but keep trailing spaces that the fixed-width
# layout relies on.
stripped = line.rstrip("\r\n")
if len(stripped) < 4:
continue
rectype = self._slice(stripped, self._RECTYPE).strip()
try:
if rectype == "10":
yield self._parse_record_10(stripped, lineno)
elif rectype == "20":
yield self._parse_record_20(stripped, lineno)
# 01 / 90 / others → context-only, skipped
except ValidationError as ve:
logger.warning(
"CABS BOS row %d rejected (%s): %s",
lineno, ve.reason_code, ve.detail,
)
raise
def _parse_record_10(self, line: str, lineno: int) -> IccRevenueLine:
ocn = self._slice(line, self._OCN).strip() or None
period = self._slice(line, self._BILL_PERIOD).strip()
mou_raw = self._slice(line, self._MOU_R10).strip()
rev_raw = self._slice(line, self._REVENUE_R10)
try:
mou = self.parse_int(mou_raw) if mou_raw else None
except ValidationError:
mou = None
return IccRevenueLine(
profile_id=self.profile_id,
reporting_year=self.reporting_year,
reporting_quarter=self._period_to_quarter(period),
icc_category="term_switched_access",
counterparty_legal_name=ocn or "UNKNOWN",
counterparty_ocn=ocn,
counterparty_country="US",
revenue_cents=self._implied_cents(rev_raw),
minutes_of_use=mou,
source_line_no=lineno,
raw_row={
"rectype": "10",
"ocn": ocn,
"period": period,
"mou_raw": mou_raw,
"revenue_raw": rev_raw,
"line": line,
},
)
def _parse_record_20(self, line: str, lineno: int) -> IccRevenueLine:
ocn = self._slice(line, self._OCN).strip() or None
period = self._slice(line, self._BILL_PERIOD).strip()
circuit = self._slice(line, self._CIRCUIT_R20).strip()
rev_raw = self._slice(line, self._REVENUE_R20)
return IccRevenueLine(
profile_id=self.profile_id,
reporting_year=self.reporting_year,
reporting_quarter=self._period_to_quarter(period),
icc_category="special_access",
counterparty_legal_name=ocn or "UNKNOWN",
counterparty_ocn=ocn,
counterparty_country="US",
revenue_cents=self._implied_cents(rev_raw),
minutes_of_use=None, # special-access is not MOU-billed
source_line_no=lineno,
raw_row={
"rectype": "20",
"ocn": ocn,
"period": period,
"circuit": circuit,
"revenue_raw": rev_raw,
"line": line,
},
)

46
scripts/workers/icc_adapters/carrier_invoice_pdf.py Normal file
View file

@ -0,0 +1,46 @@
"""Carrier invoice PDF adapter (v2 stub).
Real-world carrier invoices frequently arrive only as PDF — either as
structured text or scanned images. Extracting line-item revenue from
arbitrary carrier PDFs (CenturyLink, Verizon, AT&T, Lumen, Frontier,
Windstream, etc.) requires an OCR+layout-aware pipeline that is out of
scope for the v1 ICC ingester.
This class is a **registered stub** so that the upload endpoint can
accept the ``carrier_invoice_pdf`` source format and persist the blob
into MinIO. When the ingester dispatches to this adapter it raises
``NotImplementedError``; the ingester catches that in its outer
try/except and marks the upload ``status='failed'`` with an explanatory
error message, letting customers see "awaiting v2 PDF parser" in the
portal without losing the file.
v2 plan
-------
* ``pdfplumber`` for text extraction + table reconstruction
* Vendor-specific anchor templates (e.g. "Total Switched Access Charges")
* Fall back to Anthropic Claude vision on scanned-image pages
* OCR output manually QA'd by the Accounting-Advisor role before
``rows_accepted`` is trusted
"""
from __future__ import annotations
import logging
from typing import Iterator
from .common import BaseICCAdapter, IccRevenueLine
logger = logging.getLogger(__name__)
class CarrierInvoicePDFAdapter(BaseICCAdapter):
SOURCE_FORMAT = "carrier_invoice_pdf"
def iter_rows(self, local_path: str) -> Iterator[IccRevenueLine]:
raise NotImplementedError(
"carrier_invoice_pdf parsing is deferred to v2 — upload accepted "
"and stored, but no line items have been extracted. See the "
"module docstring for the planned OCR pipeline."
)
# Unreachable but makes this a generator-returning function:
yield # pragma: no cover

175
scripts/workers/icc_adapters/common.py Normal file
View file

@ -0,0 +1,175 @@
"""Shared base + normalized row type for ICC revenue adapters.
Each concrete adapter parses a carrier-specific interconnection or
settlement artifact (CABS BOS, X12 EDI 810, iconectiv 8YY query report,
international settlement TAS, wholesale SIP CSV, etc.) into a stream of
``IccRevenueLine`` dataclass instances, which the ingester then bulk-
inserts into ``icc_revenue_lines``.
The contract mirrors ``scripts/workers/cdr_adapters/base.py``:
* ``BaseICCAdapter.iter_rows(local_path) -> Iterator[IccRevenueLine]``
* ``ValidationError`` is raised for a single malformed row; the ingester
catches it and increments ``rows_rejected`` without halting iteration.
* ``natural_key_hash`` produces a stable SHA-256 identity used as the
dedup key in ``icc_revenue_lines``.
"""
from __future__ import annotations
import hashlib
import logging
import re
from dataclasses import dataclass, field
from typing import Any, Dict, Iterator, Optional
logger = logging.getLogger(__name__)
class ValidationError(Exception):
"""Raised by an ICC adapter when a single row fails structural validation.
The ingester catches this per-row and increments ``rows_rejected`` on the
upload. ``reason_code`` is a short machine-friendly tag (e.g.
``"bad_revenue"``, ``"missing_ocn"``) used in logs + evidence payloads.
"""
def __init__(self, reason_code: str, detail: str = ""):
super().__init__(f"{reason_code}: {detail}")
self.reason_code = reason_code
self.detail = detail
@dataclass
class IccRevenueLine:
"""Normalized Inter-Carrier-Compensation revenue line, pre-classification.
The ``icc_category`` must be one of the enum values enforced by the
``icc_revenue_lines.icc_category`` CHECK constraint:
``term_switched_access`` | ``orig_switched_access`` | ``8yy_orig_access`` |
``transit`` | ``special_access`` | ``intl_settlement`` |
``wholesale_sip`` | ``access_stim`` | ``other``.
"""
profile_id: int
reporting_year: int
icc_category: str
counterparty_legal_name: str
revenue_cents: int # signed integer cents
reporting_quarter: Optional[int] = None # 1..4 or None for annual
counterparty_ocn: Optional[str] = None
counterparty_country: str = "US" # ISO-2; "US" covers domestic
minutes_of_use: Optional[int] = None
source_line_no: Optional[int] = None
raw_row: Dict[str, Any] = field(default_factory=dict)
def natural_key_hash(self) -> str:
"""Stable SHA-256 hex digest used as the per-row dedup key.
Built from the tuple of fields that together uniquely identify a
revenue line within a (profile, year, quarter) scope:
``icc_category | counterparty_ocn-or-name | year | quarter |
revenue_cents | minutes_of_use``.
"""
counterparty_tag = (self.counterparty_ocn or self.counterparty_legal_name or "").strip().upper()
basis = "|".join(
[
self.icc_category,
counterparty_tag,
str(self.reporting_year),
str(self.reporting_quarter if self.reporting_quarter is not None else ""),
str(self.revenue_cents),
str(self.minutes_of_use if self.minutes_of_use is not None else ""),
]
)
return hashlib.sha256(basis.encode("utf-8")).hexdigest()
class BaseICCAdapter:
"""Abstract ICC adapter. Subclasses implement ``iter_rows()``."""
SOURCE_FORMAT: str = "" # matches icc_ingestion_uploads.source_format
def __init__(self, profile_id: int, reporting_year: int):
self.profile_id = profile_id
self.reporting_year = reporting_year
# ------------------------------------------------------------------
# Abstract
# ------------------------------------------------------------------
def iter_rows(self, local_path: str) -> Iterator[IccRevenueLine]:
"""Yield one ``IccRevenueLine`` per revenue line in the artifact."""
raise NotImplementedError
# ------------------------------------------------------------------
# Helpers for subclasses
# ------------------------------------------------------------------
_CENTS_CLEAN_RE = re.compile(r"[\s,$]")
@classmethod
def parse_cents(cls, val: Any) -> int:
"""Convert a currency/number input into signed integer cents.
Accepts ``1234``, ``"1,234.56"``, ``"$1,234.56"``, ``" $-97.10 "``,
``"(123.45)"`` (accounting-negative parens), floats, Decimals.
Returns ``0`` on empty inputs. Raises :class:`ValidationError`
(``reason_code='bad_revenue'``) for garbage that can't be parsed.
"""
if val is None:
return 0
if isinstance(val, bool): # avoid treating True as 1
return 0
if isinstance(val, (int, float)):
return int(round(float(val) * 100))
s = str(val).strip()
if not s:
return 0
negative = False
if s.startswith("(") and s.endswith(")"):
negative = True
s = s[1:-1]
cleaned = cls._CENTS_CLEAN_RE.sub("", s)
# Strip trailing "CR" (credit) / "DR" (debit) suffixes seen in invoices
if cleaned.upper().endswith("CR"):
negative = True
cleaned = cleaned[:-2]
elif cleaned.upper().endswith("DR"):
cleaned = cleaned[:-2]
if cleaned in ("", "-", "+"):
return 0
try:
f = float(cleaned)
except ValueError as exc:
raise ValidationError("bad_revenue", f"unparseable revenue: {val!r}") from exc
if negative:
f = -abs(f)
return int(round(f * 100))
@staticmethod
def parse_int(val: Any, default: int = 0) -> int:
"""Convert a number-like value into an int with robust coercion.
Accepts ints, floats, numeric strings with commas/whitespace, and
empty/``None`` (→ ``default``). Raises :class:`ValidationError`
(``reason_code='bad_integer'``) for non-numeric garbage.
"""
if val is None:
return default
if isinstance(val, bool):
return default
if isinstance(val, int):
return val
if isinstance(val, float):
return int(val)
s = str(val).strip()
if not s:
return default
cleaned = s.replace(",", "").replace(" ", "")
if cleaned in ("-", "+"):
return default
try:
return int(float(cleaned))
except ValueError as exc:
raise ValidationError("bad_integer", f"unparseable int: {val!r}") from exc

242
scripts/workers/icc_adapters/edi_810_adapter.py Normal file
View file

@ -0,0 +1,242 @@
"""X12 EDI 810 (Invoice) adapter for inter-carrier settlement invoices.
EDI 810 is the ANSI ASC X12 standard transaction for invoices. This
adapter walks the envelope (ISA/GS/ST) and extracts invoice- and line-
level totals, mapping them into ``IccRevenueLine`` records.
Segment coverage
----------------
``ISA`` Interchange control header — captures ``isa_control_num`` for hashing
``GS`` Functional group header — context only
``ST`` Transaction set header — one 810 per ST/SE pair
``BIG`` Beginning of invoice (invoice number, invoice date)
``N1`` Name segment — qualifier ``IC`` (Intermediate Consignee) is treated
as the interconnecting carrier; ``RE`` / ``BT`` used as fallback
``IT1`` Baseline item detail
IT103 = unit price
IT104 = unit of measurement qualifier
IT107 = quantity
IT109 = ``"MG"`` → quantity represents minutes
``TDS`` Total monetary value (TDS01 in implied-2-decimal cents)
Category heuristics
-------------------
``icc_category`` defaults to ``transit``. Free-text in PID / MSG / DTM
segments is scanned case-insensitively for keywords:
* "8yy" / "toll free" → ``8yy_orig_access``
* "term" / "terminat" → ``term_switched_access``
* "orig" / "originat" → ``orig_switched_access``
* "special access" → ``special_access``
* "international" → ``intl_settlement``
Parsing strategy
----------------
Uses ``pyx12`` if importable; otherwise falls back to a lightweight
segment-splitter driven by the ISA segment's declared separators (element
separator = ISA[1][3], segment terminator = last char of ISA). This
tolerates both LF-padded and tight single-line variants.
Deferred
--------
* Full X12 005010 syntactic validation
* 997 / 999 acknowledgment emission
* Sub-element composites beyond position-0 use
* Multi-ST interchanges (first transaction set is parsed; subsequent are
yielded but share the interchange header)
"""
from __future__ import annotations
import logging
from typing import Iterator, List, Optional
from .common import BaseICCAdapter, IccRevenueLine, ValidationError
logger = logging.getLogger(__name__)
try: # pragma: no cover — optional dependency
import pyx12 # noqa: F401
_HAS_PYX12 = True
except ImportError:
_HAS_PYX12 = False
class EDI810Adapter(BaseICCAdapter):
SOURCE_FORMAT = "edi_810"
_CATEGORY_KEYWORDS = [
("8yy_orig_access", ("8yy", "toll free", "toll-free", "tollfree")),
("intl_settlement", ("international", "intl ", "settlement")),
("special_access", ("special access", "spec access")),
("term_switched_access", ("terminating", "term access", "term switched")),
("orig_switched_access", ("originating", "orig access", "orig switched")),
]
def iter_rows(self, local_path: str) -> Iterator[IccRevenueLine]:
with open(local_path, "r", encoding="latin-1", errors="replace") as fh:
raw = fh.read()
if not raw.strip():
return
segments, elem_sep = self._split_segments(raw)
yield from self._walk(segments, elem_sep)
# ------------------------------------------------------------------
# Envelope splitter — derives separators from ISA header
# ------------------------------------------------------------------
@staticmethod
def _split_segments(raw: str) -> tuple[List[List[str]], str]:
if not raw.startswith("ISA"):
# Some exporters prepend a BOM or whitespace; search
idx = raw.find("ISA")
if idx < 0:
raise ValidationError("bad_envelope", "no ISA header found")
raw = raw[idx:]
if len(raw) < 106:
raise ValidationError("bad_envelope", "truncated ISA header")
# ISA is fixed-width 106 bytes; element separator is byte 3,
# segment terminator is byte 105.
elem_sep = raw[3]
seg_term = raw[105]
# Segment terminator may or may not be followed by \n; strip both
chunks = [s for s in raw.split(seg_term) if s.strip()]
segments = [[e for e in seg.strip("\r\n").split(elem_sep)] for seg in chunks]
return segments, elem_sep
# ------------------------------------------------------------------
# Transaction-set walker
# ------------------------------------------------------------------
def _walk(self, segments: List[List[str]], elem_sep: str) -> Iterator[IccRevenueLine]:
isa_control_num = ""
current_counterparty_name = ""
current_counterparty_id: Optional[str] = None
invoice_number = ""
line_no = 0
category_hint = "transit"
pending_line: Optional[dict] = None
tds_emitted_for_st = False
lineno_counter = 0 # file segment counter for hashing
for seg in segments:
lineno_counter += 1
if not seg:
continue
tag = seg[0]
if tag == "ISA" and len(seg) >= 14:
isa_control_num = seg[13].strip()
elif tag == "ST":
# Reset per-ST state
current_counterparty_name = ""
current_counterparty_id = None
invoice_number = ""
line_no = 0
category_hint = "transit"
pending_line = None
tds_emitted_for_st = False
elif tag == "BIG" and len(seg) >= 3:
invoice_number = seg[2].strip()
elif tag == "N1" and len(seg) >= 2:
qual = seg[1].strip().upper()
if qual in ("IC", "RE", "BT"):
current_counterparty_name = (seg[2].strip() if len(seg) > 2 else "") or current_counterparty_name
if len(seg) > 4:
current_counterparty_id = seg[4].strip() or current_counterparty_id
elif tag in ("PID", "MSG", "DTM"):
payload = " ".join(seg[1:]).lower()
for cat, keywords in self._CATEGORY_KEYWORDS:
if any(k in payload for k in keywords):
category_hint = cat
break
elif tag == "IT1":
line_no += 1
# IT103 = unit price, IT107 = qty, IT109 = UoM
unit_price = seg[3] if len(seg) > 3 else ""
quantity = seg[2] if len(seg) > 2 else ""
uom = seg[4] if len(seg) > 4 else ""
is_minutes = False
# Scan remaining elements for "MG" UoM flag
for e in seg[4:]:
if e.strip().upper() == "MG":
is_minutes = True
break
try:
qty_int = self.parse_int(quantity) if quantity else 0
except ValidationError:
qty_int = 0
try:
extended_cents = self.parse_cents(unit_price) * qty_int if unit_price else 0
except ValidationError:
extended_cents = 0
pending_line = {
"line_no": line_no,
"quantity": qty_int,
"unit_price_raw": unit_price,
"uom": uom,
"is_minutes": is_minutes,
"extended_cents": extended_cents,
}
yield self._build_line(
pending_line,
isa_control_num=isa_control_num,
invoice_number=invoice_number,
counterparty_name=current_counterparty_name,
counterparty_id=current_counterparty_id,
category=category_hint,
file_lineno=lineno_counter,
)
elif tag == "TDS" and len(seg) >= 2 and not tds_emitted_for_st and line_no == 0:
# Invoice-total fallback when no IT1 lines were present
try:
total_cents = self.parse_cents(seg[1])
except ValidationError:
continue
tds_emitted_for_st = True
yield IccRevenueLine(
profile_id=self.profile_id,
reporting_year=self.reporting_year,
icc_category=category_hint,
counterparty_legal_name=current_counterparty_name or "UNKNOWN",
counterparty_ocn=current_counterparty_id,
revenue_cents=total_cents,
minutes_of_use=None,
source_line_no=lineno_counter,
raw_row={
"segment": "TDS",
"isa_control_num": isa_control_num,
"invoice_number": invoice_number,
"total_raw": seg[1],
},
)
def _build_line(
self,
pending: dict,
*,
isa_control_num: str,
invoice_number: str,
counterparty_name: str,
counterparty_id: Optional[str],
category: str,
file_lineno: int,
) -> IccRevenueLine:
return IccRevenueLine(
profile_id=self.profile_id,
reporting_year=self.reporting_year,
icc_category=category,
counterparty_legal_name=counterparty_name or "UNKNOWN",
counterparty_ocn=counterparty_id,
revenue_cents=pending["extended_cents"],
minutes_of_use=pending["quantity"] if pending["is_minutes"] else None,
source_line_no=file_lineno,
raw_row={
"segment": "IT1",
"isa_control_num": isa_control_num,
"invoice_number": invoice_number,
"line_no": pending["line_no"],
"uom": pending["uom"],
"unit_price_raw": pending["unit_price_raw"],
"quantity": pending["quantity"],
},
)

193
scripts/workers/icc_adapters/iconectiv_8yy_adapter.py Normal file
View file

@ -0,0 +1,193 @@
"""iconectiv 8YY Query Report adapter.
Parses the monthly 8YY query-counts-and-revenue report produced by
iconectiv (formerly Ericsson / Somos toll-free number administration).
The report drives the database-query component of 8YY originating access
charges — each query to the 8YY routing database is billable at a
tariff-specified per-query rate.
Supported layouts
-----------------
* **XML** (default, `.xml`): ``<QueryReport>`` root with a single
``<Period year="YYYY" month="MM"/>`` followed by N ``<Query>`` elements,
each containing ``<OCN>``, ``<CarrierName>`` (optional), ``<Queries>``
(integer count), and ``<Revenue>`` (decimal USD).
* **QRY pipe-delimited** (`.qry`): one record per line, fields separated
by ``|``: ``OCN|QUERIES|REVENUE|PERIOD`` where PERIOD is ``YYYYMM``.
Header row ``OCN|QUERIES|REVENUE|PERIOD`` is tolerated and skipped.
Output
------
Every record yields ``icc_category='8yy_orig_access'``,
``minutes_of_use=None`` (8YY database queries are discrete events, not
duration), ``counterparty_country='US'``. Reporting quarter is derived
from the record's period month.
Deferred
--------
* XML schema validation / multiple ``<Period>`` blocks per file
* Multi-currency revenue (assumes USD)
* Query-type breakdown (POTS Dip vs LRN Dip) — treated as one category
"""
from __future__ import annotations
import csv
import logging
import os
from typing import Iterator
from .common import BaseICCAdapter, IccRevenueLine, ValidationError
logger = logging.getLogger(__name__)
try:
from lxml import etree as _etree # pragma: no cover
_USE_LXML = True
except ImportError:
import xml.etree.ElementTree as _etree
_USE_LXML = False
class IconectivQRYAdapter(BaseICCAdapter):
SOURCE_FORMAT = "8yy_qry"
@staticmethod
def _month_to_quarter(month: int) -> int | None:
if 1 <= month <= 3:
return 1
if 4 <= month <= 6:
return 2
if 7 <= month <= 9:
return 3
if 10 <= month <= 12:
return 4
return None
def iter_rows(self, local_path: str) -> Iterator[IccRevenueLine]:
ext = os.path.splitext(local_path)[1].lower()
if ext in (".xml",):
yield from self._iter_xml(local_path)
return
# Sniff by content when extension is unknown
with open(local_path, "rb") as fh:
head = fh.read(256).lstrip()
if head.startswith(b"<"):
yield from self._iter_xml(local_path)
else:
yield from self._iter_pipe(local_path)
# ------------------------------------------------------------------
# XML layout
# ------------------------------------------------------------------
def _iter_xml(self, local_path: str) -> Iterator[IccRevenueLine]:
try:
tree = _etree.parse(local_path)
except Exception as exc:
raise ValidationError("bad_xml", f"could not parse XML: {exc}") from exc
root = tree.getroot()
period_el = root.find(".//Period")
period_year = self.reporting_year
period_month: int | None = None
if period_el is not None:
try:
period_year = int(period_el.get("year") or self.reporting_year)
except ValueError:
period_year = self.reporting_year
try:
pm = period_el.get("month")
period_month = int(pm) if pm else None
except ValueError:
period_month = None
quarter = self._month_to_quarter(period_month) if period_month else None
lineno = 0
for q in root.iter("Query"):
lineno += 1
ocn_el = q.find("OCN")
rev_el = q.find("Revenue")
queries_el = q.find("Queries")
name_el = q.find("CarrierName")
ocn = (ocn_el.text or "").strip() if ocn_el is not None else ""
revenue_text = (rev_el.text or "").strip() if rev_el is not None else ""
queries_text = (queries_el.text or "").strip() if queries_el is not None else ""
name = (name_el.text or "").strip() if name_el is not None else ""
if not ocn and not name:
raise ValidationError("missing_ocn", f"Query row {lineno} has no OCN or CarrierName")
try:
revenue_cents = self.parse_cents(revenue_text)
query_count = self.parse_int(queries_text) if queries_text else None
except ValidationError:
raise
yield IccRevenueLine(
profile_id=self.profile_id,
reporting_year=period_year,
reporting_quarter=quarter,
icc_category="8yy_orig_access",
counterparty_legal_name=name or ocn,
counterparty_ocn=ocn or None,
counterparty_country="US",
revenue_cents=revenue_cents,
minutes_of_use=None,
source_line_no=lineno,
raw_row={
"ocn": ocn,
"name": name,
"queries": query_count,
"revenue_raw": revenue_text,
"period_year": period_year,
"period_month": period_month,
},
)
# ------------------------------------------------------------------
# Pipe-delimited layout
# ------------------------------------------------------------------
def _iter_pipe(self, local_path: str) -> Iterator[IccRevenueLine]:
with open(local_path, "r", encoding="utf-8", errors="replace", newline="") as fh:
reader = csv.reader(fh, delimiter="|")
for i, row in enumerate(reader, start=1):
if not row or not any(cell.strip() for cell in row):
continue
# Skip the header row
if i == 1 and row[0].strip().upper() == "OCN":
continue
if len(row) < 4:
raise ValidationError(
"bad_row",
f"8yy QRY row {i} has {len(row)} fields, expected 4",
)
ocn = row[0].strip()
queries_raw = row[1].strip()
revenue_raw = row[2].strip()
period_raw = row[3].strip()
period_year = self.reporting_year
period_month: int | None = None
if len(period_raw) >= 6:
try:
period_year = int(period_raw[:4])
period_month = int(period_raw[4:6])
except ValueError:
pass
quarter = self._month_to_quarter(period_month) if period_month else None
yield IccRevenueLine(
profile_id=self.profile_id,
reporting_year=period_year,
reporting_quarter=quarter,
icc_category="8yy_orig_access",
counterparty_legal_name=ocn,
counterparty_ocn=ocn or None,
counterparty_country="US",
revenue_cents=self.parse_cents(revenue_raw),
minutes_of_use=None,
source_line_no=i,
raw_row={
"ocn": ocn,
"queries_raw": queries_raw,
"revenue_raw": revenue_raw,
"period_raw": period_raw,
},
)

300
scripts/workers/icc_adapters/international_settlement.py Normal file
View file

@ -0,0 +1,300 @@
"""International carrier settlement adapter (ITU TAS + IC&SS).
Parses international inter-carrier settlement statements from either
the ITU-T D.150-compliant Traffic and Accounting Statement (TAS) tab-
delimited export, or the legacy IC&SS (International Carrier Settlement
& Statement) fixed-width format used by older clearinghouses.
ITU TAS — tab-delimited
-----------------------
First row is a header; required columns (case-insensitive, leading/trailing
whitespace ignored):
destination_country ISO-2 or country name
outbound_minutes integer
outbound_revenue_usd decimal
inbound_minutes integer
inbound_revenue_usd decimal
Each row yields **two** ``IccRevenueLine`` records — one for outbound
(revenue we *paid* the partner, which reduces net intl_settlement) and
one for inbound (revenue we *received*). Both are stored with
``icc_category='intl_settlement'`` and ``counterparty_country=<ISO-2>``;
the ``raw_row.direction`` tag distinguishes them for downstream analytics.
Outbound revenue is stored as a **negative** signed integer so that sum()
on the category yields net settlement.
IC&SS — fixed-width
-------------------
Record layout (positions 1-based):
01..02 record type ("SR" = settlement row)
03..05 destination country (ISO-3)
06..15 outbound minutes (10 digits)
16..27 outbound revenue, implied 2 decimals (12 digits)
28..37 inbound minutes
38..49 inbound revenue, implied 2 decimals
50..55 period YYMMDD
Detection
---------
Layout is sniffed from the first non-blank line: presence of a literal
``\\t`` tab character and an alphabetic header → TAS; otherwise IC&SS.
Deferred
--------
* Traffic-termination rate reconciliation (expected-vs-actual)
* ISO-3 to ISO-2 mapping table beyond the small built-in set — unknown
ISO-3 codes are stored raw in ``raw_row.iso3`` and ``counterparty_country``
is set to the first two characters as a best-effort fallback
"""
from __future__ import annotations
import csv
import logging
from typing import Iterator, Optional
from .common import BaseICCAdapter, IccRevenueLine, ValidationError
logger = logging.getLogger(__name__)
# Minimal ISO-3 → ISO-2 mapping for the most common settlement partners.
# Extend via an external table when the list outgrows readability.
_ISO3_TO_ISO2 = {
"USA": "US", "CAN": "CA", "MEX": "MX", "GBR": "GB", "DEU": "DE",
"FRA": "FR", "ITA": "IT", "ESP": "ES", "NLD": "NL", "BEL": "BE",
"CHE": "CH", "AUT": "AT", "SWE": "SE", "NOR": "NO", "DNK": "DK",
"FIN": "FI", "IRL": "IE", "POL": "PL", "PRT": "PT", "JPN": "JP",
"KOR": "KR", "CHN": "CN", "HKG": "HK", "TWN": "TW", "SGP": "SG",
"AUS": "AU", "NZL": "NZ", "IND": "IN", "BRA": "BR", "ARG": "AR",
"CHL": "CL", "COL": "CO", "PER": "PE", "ZAF": "ZA", "EGY": "EG",
"ISR": "IL", "ARE": "AE", "SAU": "SA", "TUR": "TR", "RUS": "RU",
"UKR": "UA", "PHL": "PH", "IDN": "ID", "THA": "TH", "VNM": "VN",
}
class InternationalSettlementAdapter(BaseICCAdapter):
SOURCE_FORMAT_TAS = "itu_tas"
SOURCE_FORMAT_ICSS = "icss"
# Generic SOURCE_FORMAT left blank; the adapter is registered under
# both slugs in ``__init__.py`` and sniffs per-file.
SOURCE_FORMAT = "itu_tas"
def iter_rows(self, local_path: str) -> Iterator[IccRevenueLine]:
layout = self._sniff_layout(local_path)
if layout == "tas":
yield from self._iter_tas(local_path)
else:
yield from self._iter_icss(local_path)
@staticmethod
def _sniff_layout(path: str) -> str:
with open(path, "r", encoding="latin-1", errors="replace") as fh:
for line in fh:
stripped = line.strip()
if not stripped:
continue
if "\t" in line:
return "tas"
return "icss"
return "tas" # empty file → treat as TAS no-rows
# ------------------------------------------------------------------
# TAS — tab-delimited
# ------------------------------------------------------------------
_TAS_REQUIRED = (
"destination_country",
"outbound_minutes",
"outbound_revenue_usd",
"inbound_minutes",
"inbound_revenue_usd",
)
def _iter_tas(self, local_path: str) -> Iterator[IccRevenueLine]:
with open(local_path, "r", encoding="utf-8", errors="replace", newline="") as fh:
reader = csv.DictReader(fh, delimiter="\t")
# Normalize header keys to lowercase/stripped
if not reader.fieldnames:
return
header_map = {name: name.strip().lower() for name in reader.fieldnames}
missing = [
req for req in self._TAS_REQUIRED
if req not in header_map.values()
]
if missing:
raise ValidationError(
"bad_header",
f"ITU TAS file missing required columns: {missing}",
)
# Build lookup: canonical_name → raw_column_name
canon = {v: k for k, v in header_map.items()}
for i, raw in enumerate(reader, start=2): # +1 for 1-index, +1 for header
country = (raw.get(canon["destination_country"]) or "").strip()
if not country:
raise ValidationError("missing_country", f"TAS row {i} has no country")
iso2 = self._country_to_iso2(country)
try:
out_min = self.parse_int(raw.get(canon["outbound_minutes"]))
out_rev = self.parse_cents(raw.get(canon["outbound_revenue_usd"]))
in_min = self.parse_int(raw.get(canon["inbound_minutes"]))
in_rev = self.parse_cents(raw.get(canon["inbound_revenue_usd"]))
except ValidationError:
raise
# Outbound = we pay partner → negative net
yield IccRevenueLine(
profile_id=self.profile_id,
reporting_year=self.reporting_year,
icc_category="intl_settlement",
counterparty_legal_name=country,
counterparty_ocn=None,
counterparty_country=iso2,
revenue_cents=-out_rev,
minutes_of_use=out_min or None,
source_line_no=i,
raw_row={
"direction": "outbound",
"country_raw": country,
"iso2": iso2,
"outbound_minutes": out_min,
"outbound_revenue_usd": raw.get(canon["outbound_revenue_usd"]),
},
)
# Inbound = we receive revenue → positive
yield IccRevenueLine(
profile_id=self.profile_id,
reporting_year=self.reporting_year,
icc_category="intl_settlement",
counterparty_legal_name=country,
counterparty_ocn=None,
counterparty_country=iso2,
revenue_cents=in_rev,
minutes_of_use=in_min or None,
source_line_no=i,
raw_row={
"direction": "inbound",
"country_raw": country,
"iso2": iso2,
"inbound_minutes": in_min,
"inbound_revenue_usd": raw.get(canon["inbound_revenue_usd"]),
},
)
# ------------------------------------------------------------------
# IC&SS — fixed-width
# ------------------------------------------------------------------
_ICSS_REC = (1, 2)
_ICSS_COUNTRY = (3, 5)
_ICSS_OUT_MIN = (6, 15)
_ICSS_OUT_REV = (16, 27)
_ICSS_IN_MIN = (28, 37)
_ICSS_IN_REV = (38, 49)
_ICSS_PERIOD = (50, 55)
@staticmethod
def _slice(line: str, span: tuple[int, int]) -> str:
start, end = span
return line[start - 1:end]
@staticmethod
def _implied_cents(raw: str) -> int:
cleaned = raw.strip()
if not cleaned:
return 0
negative = False
if cleaned[0] in "+-":
negative = cleaned[0] == "-"
cleaned = cleaned[1:]
if not cleaned.isdigit():
raise ValidationError("bad_revenue", f"ICSS revenue non-numeric: {raw!r}")
cents = int(cleaned)
return -cents if negative else cents
@staticmethod
def _month_to_quarter(month: int) -> Optional[int]:
if 1 <= month <= 3:
return 1
if 4 <= month <= 6:
return 2
if 7 <= month <= 9:
return 3
if 10 <= month <= 12:
return 4
return None
def _iter_icss(self, local_path: str) -> Iterator[IccRevenueLine]:
with open(local_path, "r", encoding="latin-1", errors="replace") as fh:
for lineno, line in enumerate(fh, start=1):
stripped = line.rstrip("\r\n")
if len(stripped) < 55:
continue
rec = self._slice(stripped, self._ICSS_REC).strip().upper()
if rec != "SR":
continue
iso3 = self._slice(stripped, self._ICSS_COUNTRY).strip().upper()
out_min = self.parse_int(self._slice(stripped, self._ICSS_OUT_MIN))
out_rev = self._implied_cents(self._slice(stripped, self._ICSS_OUT_REV))
in_min = self.parse_int(self._slice(stripped, self._ICSS_IN_MIN))
in_rev = self._implied_cents(self._slice(stripped, self._ICSS_IN_REV))
period = self._slice(stripped, self._ICSS_PERIOD).strip()
quarter = None
if len(period) >= 4:
try:
quarter = self._month_to_quarter(int(period[2:4]))
except ValueError:
quarter = None
iso2 = _ISO3_TO_ISO2.get(iso3, iso3[:2] if iso3 else "US")
yield IccRevenueLine(
profile_id=self.profile_id,
reporting_year=self.reporting_year,
reporting_quarter=quarter,
icc_category="intl_settlement",
counterparty_legal_name=iso3,
counterparty_ocn=None,
counterparty_country=iso2,
revenue_cents=-out_rev,
minutes_of_use=out_min or None,
source_line_no=lineno,
raw_row={
"direction": "outbound",
"iso3": iso3,
"iso2": iso2,
"period": period,
},
)
yield IccRevenueLine(
profile_id=self.profile_id,
reporting_year=self.reporting_year,
reporting_quarter=quarter,
icc_category="intl_settlement",
counterparty_legal_name=iso3,
counterparty_ocn=None,
counterparty_country=iso2,
revenue_cents=in_rev,
minutes_of_use=in_min or None,
source_line_no=lineno,
raw_row={
"direction": "inbound",
"iso3": iso3,
"iso2": iso2,
"period": period,
},
)
# ------------------------------------------------------------------
# Country normalization
# ------------------------------------------------------------------
@staticmethod
def _country_to_iso2(value: str) -> str:
v = value.strip().upper()
if len(v) == 2 and v.isalpha():
return v
if len(v) == 3 and v in _ISO3_TO_ISO2:
return _ISO3_TO_ISO2[v]
# Fall back to first two alphabetic characters
letters = "".join(ch for ch in v if ch.isalpha())[:2]
return letters or "US"

221
scripts/workers/icc_adapters/wholesale_sip_csv.py Normal file
View file

@ -0,0 +1,221 @@
"""Wholesale SIP CSV adapter (Sangoma, Bandwidth, Flowroute).
Vendor wholesale-SIP invoices arrive as CSVs with per-vendor column
layouts. This adapter sniffs the header row and dispatches to a
vendor-specific column map, normalizing into ``IccRevenueLine`` records
tagged ``icc_category='wholesale_sip'``.
Supported vendors
-----------------
* **Sangoma** (``SIPStation`` / ``Sangoma Wholesale``):
``invoice_number, account_id, billing_period, mou, revenue,
vendor_legal_name``
* **Bandwidth.com**:
``invoice, accountId, billingPeriod, totalMinutes, totalAmount``
* **Flowroute**:
``Invoice Number, Account, Period, Minutes, Total``
Generic fallback
----------------
If the header doesn't match any known vendor but contains the canonical
columns ``invoice_number, account_id, billing_period, mou, revenue`` the
adapter still parses the file. Vendor name then falls back to a
``Vendor`` / ``vendor`` / ``CarrierName`` column if present, else
``"UNKNOWN"``.
Deferred
--------
* Taxed-line reconciliation (separate tax columns are summed into
``revenue_cents`` only if the header implies "amount" rather than
"subtotal"; reviewer should confirm per invoice)
* Multi-currency (assumes USD; non-USD rows raise ValidationError)
"""
from __future__ import annotations
import csv
import logging
from typing import Iterator, Optional
from .common import BaseICCAdapter, IccRevenueLine, ValidationError
logger = logging.getLogger(__name__)
# ---------------------------------------------------------------------------
# Vendor column maps. Each entry: signature columns → canonical map.
# ---------------------------------------------------------------------------
_VENDOR_MAPS = [
{
"name": "sangoma",
"signature": {"invoice_number", "account_id", "billing_period", "mou", "revenue"},
"map": {
"invoice_number": "invoice_number",
"account_id": "account_id",
"billing_period": "billing_period",
"mou": "mou",
"revenue": "revenue",
"vendor": "vendor_legal_name",
},
"default_vendor": "Sangoma",
},
{
"name": "bandwidth",
"signature": {"invoice", "accountid", "billingperiod", "totalminutes", "totalamount"},
"map": {
"invoice_number": "invoice",
"account_id": "accountId",
"billing_period": "billingPeriod",
"mou": "totalMinutes",
"revenue": "totalAmount",
},
"default_vendor": "Bandwidth.com",
},
{
"name": "flowroute",
"signature": {"invoice number", "account", "period", "minutes", "total"},
"map": {
"invoice_number": "Invoice Number",
"account_id": "Account",
"billing_period": "Period",
"mou": "Minutes",
"revenue": "Total",
},
"default_vendor": "Flowroute",
},
]
class WholesaleSIPCSVAdapter(BaseICCAdapter):
SOURCE_FORMAT = "wholesale_sip_csv"
@staticmethod
def _period_to_quarter(period: str) -> Optional[int]:
"""Extract a quarter index from a billing-period free-text field.
Supports ``YYYY-MM``, ``YYYYMM``, ``MM/YYYY``, ``Mon YYYY``.
Returns None if unparseable.
"""
if not period:
return None
p = period.strip()
month: Optional[int] = None
if len(p) >= 7 and p[4] in "-/." and p[:4].isdigit():
try:
month = int(p[5:7])
except ValueError:
month = None
elif len(p) >= 6 and p[:6].isdigit():
try:
month = int(p[4:6])
except ValueError:
month = None
elif "/" in p:
parts = p.split("/")
try:
month = int(parts[0])
except ValueError:
month = None
else:
months = {
"jan": 1, "feb": 2, "mar": 3, "apr": 4, "may": 5, "jun": 6,
"jul": 7, "aug": 8, "sep": 9, "oct": 10, "nov": 11, "dec": 12,
}
token = p[:3].lower()
month = months.get(token)
if month is None:
return None
if 1 <= month <= 3:
return 1
if 4 <= month <= 6:
return 2
if 7 <= month <= 9:
return 3
if 10 <= month <= 12:
return 4
return None
def _detect_vendor(self, headers: list[str]) -> tuple[dict, str]:
"""Return (vendor_map_entry, default_vendor_name).
Raises ``ValidationError('bad_header')`` when no vendor matches.
"""
normalized = {h.strip().lower() for h in headers if h}
for entry in _VENDOR_MAPS:
if entry["signature"].issubset(normalized):
return entry, entry["default_vendor"]
raise ValidationError(
"bad_header",
f"wholesale_sip_csv header doesn't match any vendor: {sorted(normalized)}",
)
def iter_rows(self, local_path: str) -> Iterator[IccRevenueLine]:
with open(local_path, "r", encoding="utf-8-sig", errors="replace", newline="") as fh:
reader = csv.DictReader(fh)
if not reader.fieldnames:
return
entry, default_vendor = self._detect_vendor(reader.fieldnames)
# Build case-insensitive header → raw-name lookup
header_lookup = {h.strip().lower(): h for h in reader.fieldnames if h}
colmap = {
canon: header_lookup.get(raw.strip().lower(), raw)
for canon, raw in entry["map"].items()
}
# Vendor column may be None when the vendor is fixed
vendor_col = colmap.get("vendor")
for i, raw in enumerate(reader, start=2):
# Detect non-USD rows if a currency column is present
for candidate in ("currency", "Currency", "CCY"):
if candidate in raw and raw[candidate] and raw[candidate].strip().upper() not in ("USD", ""):
raise ValidationError(
"non_usd",
f"row {i} currency={raw[candidate]!r}, only USD supported",
)
billing_period = (raw.get(colmap["billing_period"]) or "").strip()
quarter = self._period_to_quarter(billing_period)
vendor = default_vendor
if vendor_col and raw.get(vendor_col):
vendor = raw[vendor_col].strip() or default_vendor
else:
# Generic fallback columns
for k in ("Vendor", "vendor", "CarrierName", "carrier_name"):
if raw.get(k):
vendor = raw[k].strip() or default_vendor
break
account_id = (raw.get(colmap["account_id"]) or "").strip()
invoice_number = (raw.get(colmap["invoice_number"]) or "").strip()
mou_raw = raw.get(colmap["mou"])
rev_raw = raw.get(colmap["revenue"])
try:
mou = self.parse_int(mou_raw) if mou_raw else None
except ValidationError:
mou = None
revenue_cents = self.parse_cents(rev_raw)
yield IccRevenueLine(
profile_id=self.profile_id,
reporting_year=self.reporting_year,
reporting_quarter=quarter,
icc_category="wholesale_sip",
counterparty_legal_name=vendor,
counterparty_ocn=None,
counterparty_country="US",
revenue_cents=revenue_cents,
minutes_of_use=mou,
source_line_no=i,
raw_row={
"vendor": vendor,
"vendor_detected": entry["name"],
"invoice_number": invoice_number,
"account_id": account_id,
"billing_period": billing_period,
"mou_raw": mou_raw,
"revenue_raw": rev_raw,
},
)