Initial commit — Performance West telecom compliance platform

Includes: API (Express/TypeScript), Astro site, Python workers,
document generators, FCC compliance tools, Canada CRTC formation,
Ansible infrastructure, and deployment scripts.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

scripts/workers/cdr_presets/__init__.py

@@ -0,0 +1,35 @@
+"""Switch preset registry.
+
+A preset captures "everything we know about how to pull CDRs from
+Switch X" — transport method, credential field set, CDR format adapter,
+sensible default cron. The portal dropdown is driven off this registry.
+"""
+
+from .base import BasePreset, CredentialField
+from .asterisk import AsteriskPreset
+from .freeswitch import FreeSWITCHPreset
+from .grandstream import GrandstreamPreset
+from .netsapiens import NetSapiensPreset
+from .kazoo import KazooPreset
+from .ribbon import RibbonPreset
+from .fortysix_labs import FortySixLabsPreset
+from .metaswitch import MetaswitchPreset
+from .sansay import SansayPreset
+from .broadworks import BroadWorksPreset
+from .sip_navigator import SIPNavigatorPreset
+
+PRESETS: dict[str, type[BasePreset]] = {
+    "asterisk":        AsteriskPreset,
+    "freeswitch":      FreeSWITCHPreset,
+    "grandstream":     GrandstreamPreset,
+    "netsapiens":      NetSapiensPreset,
+    "kazoo":           KazooPreset,
+    "ribbon":          RibbonPreset,
+    "fortysix_labs":   FortySixLabsPreset,
+    "metaswitch":      MetaswitchPreset,
+    "sansay":          SansayPreset,
+    "broadworks":      BroadWorksPreset,
+    "sip_navigator":   SIPNavigatorPreset,
+}
+
+__all__ = ["BasePreset", "CredentialField", "PRESETS"]

scripts/workers/cdr_presets/_scrape_base.py

@@ -0,0 +1,86 @@
+"""Shared skeleton for Playwright-scrape presets.
+
+Switches that don't expose an API for CDR export (Metaswitch iCM, Sansay
+SSM, BroadWorks OCS web, some Cataleya SIP Navigator deployments) need a
+login + download flow driven by Playwright. The selectors vary per
+deployment version, so each subclass locks its own login URL and
+post-login navigation; the base here handles the undetected browser
+launch + download capture.
+
+Until live recon finalizes the selectors, each concrete preset's
+``fetch()`` raises NotImplementedError with a clear instruction; the
+cdr_puller catches that, creates an admin ToDo, and the admin runs the
+download manually (or a PW engineer extends the preset against the
+specific deployment).
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+from datetime import datetime
+from typing import Iterable, Optional
+
+from .base import BasePreset, CredentialField, FetchedFile
+
+logger = logging.getLogger(__name__)
+
+
+class ScrapePreset(BasePreset):
+    """Skeleton for Playwright-driven presets."""
+
+    TRANSPORT_METHOD = "scrape"
+    LOGIN_URL: str = ""              # subclass sets (e.g. https://icm.example/admin)
+    CDR_DOWNLOAD_URL: str = ""       # subclass sets
+
+    CREDENTIAL_FIELDS = (
+        CredentialField("admin_url", "Web admin URL", "text",
+                        help="Root URL of the management UI."),
+        CredentialField("username", "Admin username", "text"),
+        CredentialField("password", "Admin password", "password", sensitive=True),
+    )
+
+    async def _run_scrape(self, cfg: dict, secrets: dict, since: Optional[datetime]) -> Iterable[FetchedFile]:
+        """Subclasses implement the Playwright flow here."""
+        raise NotImplementedError(
+            f"{self.__class__.__name__} requires live-session recon before it can "
+            "automate CDR download. Until selectors are locked, the puller will "
+            "file an admin ToDo instructing a human to export + upload manually."
+        )
+
+    def validate(self, profile_config: dict, secrets: dict) -> tuple[bool, str]:
+        # Without a concrete flow, we at least confirm the admin URL is reachable.
+        import urllib.request
+        url = profile_config.get("admin_url") or self.LOGIN_URL
+        if not url:
+            return False, "admin_url not configured"
+        try:
+            req = urllib.request.Request(url, method="HEAD")
+            with urllib.request.urlopen(req, timeout=15) as resp:
+                return True, f"admin URL reachable (HTTP {resp.status})"
+        except Exception as exc:
+            return False, f"admin URL unreachable: {exc}"
+
+    def fetch(
+        self,
+        profile_config: dict,
+        secrets: dict,
+        since: Optional[datetime],
+    ) -> Iterable[FetchedFile]:
+        try:
+            loop = asyncio.new_event_loop()
+            try:
+                return list(loop.run_until_complete(
+                    self._to_async_list(profile_config, secrets, since)
+                ))
+            finally:
+                loop.close()
+        except NotImplementedError as exc:
+            logger.warning("%s: %s", self.__class__.__name__, exc)
+            raise
+
+    async def _to_async_list(self, cfg, secrets, since):
+        out: list[FetchedFile] = []
+        async for f in self._run_scrape(cfg, secrets, since):
+            out.append(f)
+        return out

scripts/workers/cdr_presets/asterisk.py

@@ -0,0 +1,66 @@
+"""Asterisk preset — SFTP pull from the Asterisk Master.csv location."""
+
+from __future__ import annotations
+
+from datetime import datetime
+from typing import Iterable, Optional
+
+from .base import BasePreset, CredentialField, FetchedFile
+from ..cdr_transports.sftp_transport import SFTPTransport
+
+
+class AsteriskPreset(BasePreset):
+    PRESET_SLUG = "asterisk"
+    LABEL = "Asterisk / AsteriskNOW / FreePBX"
+    CDR_FORMAT = "asterisk"
+    TRANSPORT_METHOD = "sftp"
+    DEFAULT_CRON = "0 2 * * *"
+
+    CREDENTIAL_FIELDS = (
+        CredentialField("host", "Asterisk host / IP", "text"),
+        CredentialField("port", "SSH port", "number", required=False),
+        CredentialField("username", "SSH username", "text"),
+        CredentialField("password", "SSH password", "password",
+                        required=False, sensitive=True,
+                        help="Leave blank if using SSH key auth."),
+        CredentialField("private_key", "SSH private key", "ssh_key",
+                        required=False, sensitive=True),
+        CredentialField("remote_dir", "Asterisk CDR directory", "text",
+                        help="Typically /var/log/asterisk/cdr-csv/"),
+        CredentialField("file_glob", "File pattern", "text",
+                        required=False,
+                        help="Default Master.csv; or *.csv for rotating logs."),
+    )
+
+    def _transport(self, cfg: dict, secrets: dict) -> SFTPTransport:
+        remote_dir = cfg.get("remote_dir", "/var/log/asterisk/cdr-csv")
+        glob = cfg.get("file_glob", "Master.csv")
+        return SFTPTransport(
+            host=cfg["host"],
+            port=int(cfg.get("port") or 22),
+            username=cfg["username"],
+            password=secrets.get("password"),
+            private_key=secrets.get("private_key"),
+            remote_glob=f"{remote_dir.rstrip('/')}/{glob}",
+        )
+
+    def validate(self, profile_config: dict, secrets: dict) -> tuple[bool, str]:
+        try:
+            return self._transport(profile_config, secrets).validate()
+        except Exception as exc:
+            return False, f"{exc}"
+
+    def fetch(
+        self,
+        profile_config: dict,
+        secrets: dict,
+        since: Optional[datetime],
+    ) -> Iterable[FetchedFile]:
+        transport = self._transport(profile_config, secrets)
+        for remote in transport.list_since(since):
+            yield FetchedFile(
+                remote_path=remote.path,
+                mtime=remote.mtime,
+                content=transport.fetch(remote.path),
+                size_bytes=remote.size_bytes,
+            )

scripts/workers/cdr_presets/base.py

@@ -0,0 +1,68 @@
+"""Switch-preset contract.
+
+A preset tells the portal:
+  * what human-readable label to show in the dropdown
+  * which credential fields to render
+  * what CDR format adapter to hook the output through
+  * what cron cadence is appropriate by default
+
+And provides two methods to the cdr_puller worker:
+  * validate(profile) → (ok, detail)  — "test connection" button
+  * fetch(profile, since) → Iterator[(remote_path, bytes)]  — pull new files
+
+Credentials on ``profile`` are loaded at the puller layer from the
+ERPNext Sensitive ID record linked by ``profile.pull_sensitive_id``.
+"""
+
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass
+from datetime import datetime
+from typing import Iterable, Optional
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class CredentialField:
+    """One field the portal should render in the preset credential form."""
+    key: str                  # JSON key stored under profile.preset_config / Sensitive ID
+    label: str                # shown on the form
+    kind: str                 # 'text' | 'password' | 'ssh_key' | 'select' | 'number'
+    required: bool = True
+    sensitive: bool = False   # if True, stored encrypted in Sensitive ID not profile_config
+    help: str = ""
+    options: Optional[list[str]] = None  # for kind='select'
+
+
+@dataclass
+class FetchedFile:
+    remote_path: str
+    mtime: datetime
+    content: bytes
+    size_bytes: int
+
+
+class BasePreset:
+    """Abstract preset. Subclasses set class attributes + implement methods."""
+
+    PRESET_SLUG: str = ""               # matches cdr_ingestion_profiles.switch_preset
+    LABEL: str = ""                     # shown in the portal dropdown
+    CDR_FORMAT: str = "generic_csv"     # matches cdr_ingestion_profiles.format
+    TRANSPORT_METHOD: str = ""          # 'api' | 'scrape' | 'sftp' | 'ftps'
+    DEFAULT_CRON: str = "0 2 * * *"
+    CREDENTIAL_FIELDS: tuple[CredentialField, ...] = ()
+
+    def validate(self, profile_config: dict, secrets: dict) -> tuple[bool, str]:
+        """Test-connection hook for the portal. Returns (ok, detail)."""
+        raise NotImplementedError
+
+    def fetch(
+        self,
+        profile_config: dict,
+        secrets: dict,
+        since: Optional[datetime],
+    ) -> Iterable[FetchedFile]:
+        """Yield new CDR files since `since`. Puller streams them to MinIO."""
+        raise NotImplementedError

scripts/workers/cdr_presets/broadworks.py

@@ -0,0 +1,30 @@
+"""Cisco BroadWorks preset — OCS web UI / SOAP hybrid.
+
+BroadWorks exposes CDRs via several mechanisms — SOAP OCI-P, OSS push to
+an SFTP dropoff, or manual CSV download from the OCS web portal. The
+simplest cross-deployment path is the OCS web UI scrape (CSV export
+button on the "CDR File Delivery" page). SOAP / OCI-P can be added later
+if a deployment only allows programmatic access.
+"""
+
+from __future__ import annotations
+
+from ._scrape_base import ScrapePreset
+
+
+class BroadWorksPreset(ScrapePreset):
+    PRESET_SLUG = "broadworks"
+    LABEL = "Cisco BroadWorks (OCS)"
+    CDR_FORMAT = "generic_csv"
+    DEFAULT_CRON = "0 3 * * *"
+
+    FORMAT_CONFIG = {
+        "start_time":    "startTime",
+        "caller_number": "callingNumber",
+        "called_number": "calledNumber",
+        "duration_sec":  "releaseTime",     # computed (release - answer) in custom map
+        "call_id":       "correlationId",
+        "trunk_group":   "trunkGroup",
+        "disposition":   "releaseCauseKey",
+        "ts_format":     "%Y-%m-%dT%H:%M:%S",
+    }

scripts/workers/cdr_presets/fortysix_labs.py

@@ -0,0 +1,97 @@
+"""46Labs preset — Peering / NOVA platform REST API."""
+
+from __future__ import annotations
+
+import json
+import urllib.parse
+import urllib.request
+from datetime import datetime, timedelta
+from typing import Iterable, Optional
+
+from .base import BasePreset, CredentialField, FetchedFile
+
+
+class FortySixLabsPreset(BasePreset):
+    PRESET_SLUG = "fortysix_labs"
+    LABEL = "46Labs (Peering / NOVA)"
+    CDR_FORMAT = "generic_csv"
+    TRANSPORT_METHOD = "api"
+    DEFAULT_CRON = "0 2 * * *"
+
+    CREDENTIAL_FIELDS = (
+        CredentialField("api_host", "46Labs API host", "text",
+                        help="e.g. https://api.46labs.com"),
+        CredentialField("account_id", "Account ID", "text"),
+        CredentialField("api_key", "API key", "password", sensitive=True),
+    )
+
+    FORMAT_CONFIG = {
+        "start_time":    "start_time",
+        "caller_number": "ani",
+        "called_number": "dnis",
+        "duration_sec":  "billable_duration",
+        "billed_amount": "call_cost",
+        "call_id":       "cdr_id",
+        "trunk_group":   "ingress_trunk",
+        "disposition":   "release_cause",
+        "ts_format":     "%Y-%m-%dT%H:%M:%SZ",
+    }
+
+    def _request(self, url: str, api_key: str) -> bytes:
+        req = urllib.request.Request(
+            url, headers={"Authorization": f"Bearer {api_key}",
+                          "Accept": "application/json"},
+        )
+        with urllib.request.urlopen(req, timeout=60) as resp:
+            return resp.read()
+
+    def validate(self, profile_config: dict, secrets: dict) -> tuple[bool, str]:
+        try:
+            base = profile_config["api_host"].rstrip("/")
+            acct = profile_config["account_id"]
+            self._request(
+                f"{base}/v1/accounts/{acct}", api_key=secrets["api_key"],
+            )
+            return True, "46Labs account reachable"
+        except Exception as exc:
+            return False, f"46Labs validate failed: {exc}"
+
+    def fetch(
+        self,
+        profile_config: dict,
+        secrets: dict,
+        since: Optional[datetime],
+    ) -> Iterable[FetchedFile]:
+        base = profile_config["api_host"].rstrip("/")
+        acct = profile_config["account_id"]
+        since = since or (datetime.utcnow() - timedelta(days=1))
+        end = datetime.utcnow()
+
+        records: list[dict] = []
+        cursor: Optional[str] = None
+        while True:
+            qs = urllib.parse.urlencode({
+                "start_time": since.strftime("%Y-%m-%dT%H:%M:%SZ"),
+                "end_time":   end.strftime("%Y-%m-%dT%H:%M:%SZ"),
+                "limit":      1000,
+                **({"cursor": cursor} if cursor else {}),
+            })
+            body = self._request(
+                f"{base}/v1/accounts/{acct}/cdr/search?{qs}",
+                api_key=secrets["api_key"],
+            )
+            payload = json.loads(body)
+            items = payload.get("cdrs", []) or payload.get("data", [])
+            records.extend(items)
+            cursor = payload.get("next_cursor")
+            if not cursor or not items:
+                break
+
+        if not records:
+            return
+
+        ndjson = "\n".join(json.dumps(r) for r in records).encode("utf-8")
+        yield FetchedFile(
+            remote_path=f"fortysix_labs_cdrs_{end:%Y%m%dT%H%M%SZ}.ndjson",
+            mtime=end, content=ndjson, size_bytes=len(ndjson),
+        )

scripts/workers/cdr_presets/freeswitch.py

@@ -0,0 +1,65 @@
+"""FreeSWITCH preset — SFTP pull from mod_cdr_csv output directory."""
+
+from __future__ import annotations
+
+from datetime import datetime
+from typing import Iterable, Optional
+
+from .base import BasePreset, CredentialField, FetchedFile
+from ..cdr_transports.sftp_transport import SFTPTransport
+
+
+class FreeSWITCHPreset(BasePreset):
+    PRESET_SLUG = "freeswitch"
+    LABEL = "FreeSWITCH (mod_cdr_csv)"
+    CDR_FORMAT = "freeswitch"
+    TRANSPORT_METHOD = "sftp"
+    DEFAULT_CRON = "0 2 * * *"
+
+    CREDENTIAL_FIELDS = (
+        CredentialField("host", "FreeSWITCH host / IP", "text",
+                        help="Hostname or IP reachable via SSH."),
+        CredentialField("port", "SSH port", "number", required=False,
+                        help="Default 22."),
+        CredentialField("username", "SSH username", "text"),
+        CredentialField("password", "SSH password", "password",
+                        required=False, sensitive=True,
+                        help="Leave blank if using SSH key auth."),
+        CredentialField("private_key", "SSH private key", "ssh_key",
+                        required=False, sensitive=True,
+                        help="PEM-format private key. Preferred over password."),
+        CredentialField("remote_dir", "CDR directory on the server", "text",
+                        help="Typically /var/log/freeswitch/cdr-csv/"),
+    )
+
+    def _transport(self, cfg: dict, secrets: dict) -> SFTPTransport:
+        remote_dir = cfg.get("remote_dir", "/var/log/freeswitch/cdr-csv")
+        return SFTPTransport(
+            host=cfg["host"],
+            port=int(cfg.get("port") or 22),
+            username=cfg["username"],
+            password=secrets.get("password"),
+            private_key=secrets.get("private_key"),
+            remote_glob=f"{remote_dir.rstrip('/')}/*.csv",
+        )
+
+    def validate(self, profile_config: dict, secrets: dict) -> tuple[bool, str]:
+        try:
+            return self._transport(profile_config, secrets).validate()
+        except Exception as exc:
+            return False, f"{exc}"
+
+    def fetch(
+        self,
+        profile_config: dict,
+        secrets: dict,
+        since: Optional[datetime],
+    ) -> Iterable[FetchedFile]:
+        transport = self._transport(profile_config, secrets)
+        for remote in transport.list_since(since):
+            yield FetchedFile(
+                remote_path=remote.path,
+                mtime=remote.mtime,
+                content=transport.fetch(remote.path),
+                size_bytes=remote.size_bytes,
+            )

scripts/workers/cdr_presets/grandstream.py

@@ -0,0 +1,74 @@
+"""Grandstream UCM preset — web-UI CDR download via the local API.
+
+Grandstream UCM62xx/63xx exposes a REST-style admin API at
+  ``https://<host>/api`` with cookie-based auth. The CDR export endpoint
+returns CSV; we reuse it via HTTPS transport.
+"""
+
+from __future__ import annotations
+
+from datetime import datetime
+from typing import Iterable, Optional
+
+from .base import BasePreset, CredentialField, FetchedFile
+from ..cdr_transports.https_transport import HTTPSTransport
+
+
+class GrandstreamPreset(BasePreset):
+    PRESET_SLUG = "grandstream"
+    LABEL = "Grandstream UCM (62xx / 63xx)"
+    CDR_FORMAT = "generic_csv"
+    TRANSPORT_METHOD = "api"
+    DEFAULT_CRON = "0 2 * * *"
+
+    CREDENTIAL_FIELDS = (
+        CredentialField("host", "UCM host / IP", "text"),
+        CredentialField("port", "HTTPS port", "number", required=False,
+                        help="Default 8089 for Grandstream admin API."),
+        CredentialField("username", "Admin username", "text"),
+        CredentialField("password", "Admin password", "password", sensitive=True),
+    )
+
+    FORMAT_CONFIG = {
+        "start_time":    "AcctStartTime",
+        "caller_number": "src",
+        "called_number": "dst",
+        "duration_sec":  "billsec",
+        "call_id":       "uniqueid",
+        "trunk_group":   "channel",
+        "disposition":   "disposition",
+        "ts_format":     "%Y-%m-%d %H:%M:%S",
+    }
+
+    def _transport(self, cfg: dict, secrets: dict) -> HTTPSTransport:
+        return HTTPSTransport(
+            host=cfg["host"],
+            port=int(cfg.get("port") or 8089),
+            username=cfg.get("username"),
+            password=secrets.get("password"),
+            remote_glob="cgi-bin/api/cdrapi?format=csv",
+            extra={"scheme": "https"},
+        )
+
+    def validate(self, profile_config: dict, secrets: dict) -> tuple[bool, str]:
+        try:
+            return self._transport(profile_config, secrets).validate()
+        except Exception as exc:
+            return False, f"{exc}"
+
+    def fetch(
+        self,
+        profile_config: dict,
+        secrets: dict,
+        since: Optional[datetime],
+    ) -> Iterable[FetchedFile]:
+        # Grandstream's CDR endpoint returns the full export as one CSV
+        # per request — dedup is handled later via natural_key_hash.
+        transport = self._transport(profile_config, secrets)
+        content = transport.fetch(transport.remote_glob)
+        yield FetchedFile(
+            remote_path=f"grandstream_cdr_{datetime.utcnow():%Y%m%dT%H%M%SZ}.csv",
+            mtime=datetime.utcnow(),
+            content=content,
+            size_bytes=len(content),
+        )

scripts/workers/cdr_presets/kazoo.py

@@ -0,0 +1,100 @@
+"""2600Hz Kazoo preset — REST API pull.
+
+Kazoo exposes ``/v2/accounts/{account_id}/cdrs`` with bearer-token auth.
+We page through and materialize as NDJSON; the generic_csv adapter with
+a Kazoo-specific column map reads it back.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import urllib.request
+from datetime import datetime, timedelta
+from typing import Iterable, Optional
+
+from .base import BasePreset, CredentialField, FetchedFile
+
+logger = logging.getLogger(__name__)
+
+
+class KazooPreset(BasePreset):
+    PRESET_SLUG = "kazoo"
+    LABEL = "2600Hz Kazoo"
+    CDR_FORMAT = "generic_csv"
+    TRANSPORT_METHOD = "api"
+    DEFAULT_CRON = "0 2 * * *"
+
+    CREDENTIAL_FIELDS = (
+        CredentialField("api_host", "Kazoo API host", "text",
+                        help="e.g. https://api.example.com:8443"),
+        CredentialField("account_id", "Kazoo account ID", "text"),
+        CredentialField("auth_token", "Kazoo auth token", "password", sensitive=True,
+                        help="Generated via the Kazoo user-auth API."),
+    )
+
+    FORMAT_CONFIG = {
+        "start_time":    "timestamp",
+        "caller_number": "caller_id_number",
+        "called_number": "callee_id_number",
+        "duration_sec":  "duration_seconds",
+        "billed_amount": "billing_seconds",   # Kazoo doesn't bill here by default
+        "call_id":       "call_id",
+        "trunk_group":   "request",
+    }
+
+    def _request(self, url: str, token: str) -> bytes:
+        req = urllib.request.Request(
+            url, headers={"X-Auth-Token": token, "Accept": "application/json"},
+        )
+        with urllib.request.urlopen(req, timeout=30) as resp:
+            return resp.read()
+
+    def validate(self, profile_config: dict, secrets: dict) -> tuple[bool, str]:
+        try:
+            base = profile_config["api_host"].rstrip("/")
+            acct = profile_config["account_id"]
+            self._request(
+                f"{base}/v2/accounts/{acct}",
+                token=secrets["auth_token"],
+            )
+            return True, "Account endpoint reachable"
+        except Exception as exc:
+            return False, f"Kazoo validate failed: {exc}"
+
+    def fetch(
+        self,
+        profile_config: dict,
+        secrets: dict,
+        since: Optional[datetime],
+    ) -> Iterable[FetchedFile]:
+        base = profile_config["api_host"].rstrip("/")
+        acct = profile_config["account_id"]
+        token = secrets["auth_token"]
+        since = since or (datetime.utcnow() - timedelta(days=1))
+        end = datetime.utcnow()
+        start_epoch = int(since.timestamp()) + 62167219200  # Kazoo uses gregorian epoch
+        end_epoch = int(end.timestamp()) + 62167219200
+
+        records: list[dict] = []
+        page_key: Optional[str] = None
+        while True:
+            qs = f"created_from={start_epoch}&created_to={end_epoch}&paginate=true&page_size=500"
+            if page_key:
+                qs += f"&start_key={page_key}"
+            body = self._request(f"{base}/v2/accounts/{acct}/cdrs?{qs}", token)
+            payload = json.loads(body)
+            items = payload.get("data", [])
+            records.extend(items)
+            page_key = payload.get("next_start_key")
+            if not page_key or not items:
+                break
+
+        if not records:
+            return
+
+        ndjson = "\n".join(json.dumps(r) for r in records).encode("utf-8")
+        yield FetchedFile(
+            remote_path=f"kazoo_cdrs_{end:%Y%m%dT%H%M%SZ}.ndjson",
+            mtime=end, content=ndjson, size_bytes=len(ndjson),
+        )

scripts/workers/cdr_presets/metaswitch.py

@@ -0,0 +1,28 @@
+"""Metaswitch iCM Provisioning Server preset — Playwright scrape.
+
+iCM does not expose a public CDR API; operators download CSV exports
+from the CDR Archive page inside the Provisioning Server admin UI.
+Selectors vary across iCM versions (V11/V12+), so the concrete login
++ download flow is deferred until live recon on a specific deployment.
+"""
+
+from __future__ import annotations
+
+from ._scrape_base import ScrapePreset
+
+
+class MetaswitchPreset(ScrapePreset):
+    PRESET_SLUG = "metaswitch"
+    LABEL = "Metaswitch iCM (Provisioning Server)"
+    CDR_FORMAT = "generic_csv"
+    DEFAULT_CRON = "0 3 * * *"
+
+    FORMAT_CONFIG = {
+        "start_time":    "startTime",
+        "caller_number": "callingNumber",
+        "called_number": "calledNumber",
+        "duration_sec":  "connectionTime",
+        "billed_amount": "totalCharge",
+        "call_id":       "cdr_sequence",
+        "ts_format":     "%Y-%m-%d %H:%M:%S",
+    }

scripts/workers/cdr_presets/netsapiens.py

@@ -0,0 +1,117 @@
+"""NetSapiens preset — REST API pull (CDRv2 endpoint).
+
+NetSapiens exposes ``/ns-api/`` with OAuth2 client-credentials. CDRs are
+paginated at ``/cdrs`` with ``start_datetime`` / ``end_datetime`` filters;
+we paginate and stream the full response as NDJSON into MinIO for the
+netsapiens adapter to consume.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import urllib.parse
+import urllib.request
+from datetime import datetime, timedelta
+from typing import Iterable, Optional
+
+from .base import BasePreset, CredentialField, FetchedFile
+
+logger = logging.getLogger(__name__)
+
+
+class NetSapiensPreset(BasePreset):
+    PRESET_SLUG = "netsapiens"
+    LABEL = "NetSapiens"
+    CDR_FORMAT = "netsapiens"
+    TRANSPORT_METHOD = "api"
+    DEFAULT_CRON = "0 2 * * *"
+
+    CREDENTIAL_FIELDS = (
+        CredentialField("api_host", "NetSapiens API host", "text",
+                        help="e.g. https://core1.example.com"),
+        CredentialField("client_id", "OAuth client ID", "text"),
+        CredentialField("client_secret", "OAuth client secret", "password", sensitive=True),
+        CredentialField("domain", "NetSapiens domain", "text",
+                        help="Your tenant/domain within the switch."),
+    )
+
+    def _request(self, url: str, headers: dict, method: str = "GET",
+                 data: Optional[bytes] = None, timeout: int = 30) -> bytes:
+        req = urllib.request.Request(url, data=data, headers=headers, method=method)
+        with urllib.request.urlopen(req, timeout=timeout) as resp:
+            return resp.read()
+
+    def _oauth_token(self, cfg: dict, secrets: dict) -> str:
+        body = urllib.parse.urlencode({
+            "grant_type": "client_credentials",
+            "client_id": cfg["client_id"],
+            "client_secret": secrets["client_secret"],
+        }).encode("utf-8")
+        token_url = cfg["api_host"].rstrip("/") + "/ns-api/oauth2/token/"
+        resp = self._request(
+            token_url,
+            headers={"Content-Type": "application/x-www-form-urlencoded"},
+            method="POST", data=body,
+        )
+        payload = json.loads(resp)
+        token = payload.get("access_token")
+        if not token:
+            raise RuntimeError(f"NetSapiens OAuth failed: {payload}")
+        return token
+
+    def validate(self, profile_config: dict, secrets: dict) -> tuple[bool, str]:
+        try:
+            self._oauth_token(profile_config, secrets)
+            return True, "OAuth token acquired"
+        except Exception as exc:
+            return False, f"NetSapiens validate failed: {exc}"
+
+    def fetch(
+        self,
+        profile_config: dict,
+        secrets: dict,
+        since: Optional[datetime],
+    ) -> Iterable[FetchedFile]:
+        token = self._oauth_token(profile_config, secrets)
+        since = since or (datetime.utcnow() - timedelta(days=1))
+        end = datetime.utcnow()
+        base = profile_config["api_host"].rstrip("/")
+        headers = {
+            "Authorization": f"Bearer {token}",
+            "Accept": "application/json",
+        }
+        records: list[dict] = []
+        page = 1
+        while True:
+            qs = urllib.parse.urlencode({
+                "domain":         profile_config["domain"],
+                "start_datetime": since.strftime("%Y-%m-%d %H:%M:%S"),
+                "end_datetime":   end.strftime("%Y-%m-%d %H:%M:%S"),
+                "page":           page,
+                "per_page":       500,
+            })
+            url = f"{base}/ns-api/v2/cdrs?{qs}"
+            try:
+                body = self._request(url, headers=headers)
+            except Exception as exc:
+                logger.warning("NetSapiens fetch page %s failed: %s", page, exc)
+                break
+            payload = json.loads(body)
+            items = payload if isinstance(payload, list) else payload.get("data", [])
+            if not items:
+                break
+            records.extend(items)
+            if len(items) < 500:
+                break
+            page += 1
+
+        if not records:
+            return
+
+        # Stream as NDJSON — the netsapiens adapter reads both array + NDJSON
+        ndjson_bytes = "\n".join(json.dumps(r) for r in records).encode("utf-8")
+        yield FetchedFile(
+            remote_path=f"netsapiens_cdrs_{end:%Y%m%dT%H%M%SZ}.ndjson",
+            mtime=end, content=ndjson_bytes, size_bytes=len(ndjson_bytes),
+        )

scripts/workers/cdr_presets/ribbon.py

@@ -0,0 +1,90 @@
+"""Ribbon / Sonus PSX or EMA preset — REST API pull.
+
+Ribbon's Element Management Application (EMA) and PSX both expose CDR
+export APIs. The common path is ``/rest/cdrs`` with basic auth. Output
+format varies: CSV (EMA default) or XML (PSX default); we request CSV
+and feed the generic_csv adapter.
+"""
+
+from __future__ import annotations
+
+import base64
+import urllib.parse
+import urllib.request
+from datetime import datetime, timedelta
+from typing import Iterable, Optional
+
+from .base import BasePreset, CredentialField, FetchedFile
+
+
+class RibbonPreset(BasePreset):
+    PRESET_SLUG = "ribbon"
+    LABEL = "Ribbon / Sonus SBC (EMA / PSX)"
+    CDR_FORMAT = "generic_csv"
+    TRANSPORT_METHOD = "api"
+    DEFAULT_CRON = "0 3 * * *"
+
+    CREDENTIAL_FIELDS = (
+        CredentialField("api_host", "EMA / PSX host", "text",
+                        help="e.g. https://ema.example.com"),
+        CredentialField("username", "API username", "text"),
+        CredentialField("password", "API password", "password", sensitive=True),
+    )
+
+    FORMAT_CONFIG = {
+        "start_time":    "startTime",
+        "caller_number": "callingNumber",
+        "called_number": "calledNumber",
+        "duration_sec":  "callDurationSeconds",
+        "billed_amount": "totalCharge",
+        "call_id":       "globalCallId",
+        "trunk_group":   "ingressTrunkGroup",
+        "disposition":   "releaseCause",
+        "ts_format":     "%Y-%m-%dT%H:%M:%S%z",
+    }
+
+    def _auth_header(self, cfg: dict, secrets: dict) -> str:
+        creds = base64.b64encode(
+            f"{cfg['username']}:{secrets['password']}".encode("utf-8")
+        ).decode("ascii")
+        return f"Basic {creds}"
+
+    def _request(self, url: str, headers: dict) -> bytes:
+        req = urllib.request.Request(url, headers=headers)
+        with urllib.request.urlopen(req, timeout=60) as resp:
+            return resp.read()
+
+    def validate(self, profile_config: dict, secrets: dict) -> tuple[bool, str]:
+        try:
+            base = profile_config["api_host"].rstrip("/")
+            self._request(
+                f"{base}/rest/v1/system/status",
+                headers={"Authorization": self._auth_header(profile_config, secrets)},
+            )
+            return True, "EMA/PSX reachable"
+        except Exception as exc:
+            return False, f"Ribbon validate failed: {exc}"
+
+    def fetch(
+        self,
+        profile_config: dict,
+        secrets: dict,
+        since: Optional[datetime],
+    ) -> Iterable[FetchedFile]:
+        base = profile_config["api_host"].rstrip("/")
+        since = since or (datetime.utcnow() - timedelta(days=1))
+        end = datetime.utcnow()
+        qs = urllib.parse.urlencode({
+            "startTime": since.strftime("%Y-%m-%dT%H:%M:%SZ"),
+            "endTime":   end.strftime("%Y-%m-%dT%H:%M:%SZ"),
+            "format":    "csv",
+        })
+        headers = {
+            "Authorization": self._auth_header(profile_config, secrets),
+            "Accept": "text/csv",
+        }
+        content = self._request(f"{base}/rest/v1/cdrs?{qs}", headers=headers)
+        yield FetchedFile(
+            remote_path=f"ribbon_cdrs_{end:%Y%m%dT%H%M%SZ}.csv",
+            mtime=end, content=content, size_bytes=len(content),
+        )

scripts/workers/cdr_presets/sansay.py

@@ -0,0 +1,24 @@
+"""Sansay SBC — SSM (Sansay System Manager) Playwright scrape preset."""
+
+from __future__ import annotations
+
+from ._scrape_base import ScrapePreset
+
+
+class SansayPreset(ScrapePreset):
+    PRESET_SLUG = "sansay"
+    LABEL = "Sansay SBC (SSM)"
+    CDR_FORMAT = "generic_csv"
+    DEFAULT_CRON = "0 3 * * *"
+
+    FORMAT_CONFIG = {
+        "start_time":    "cdr_start",
+        "caller_number": "calling_number",
+        "called_number": "called_number",
+        "duration_sec":  "duration",
+        "billed_amount": "call_charge",
+        "call_id":       "call_id",
+        "trunk_group":   "ingress_tg",
+        "disposition":   "termination_cause",
+        "ts_format":     "%Y-%m-%d %H:%M:%S",
+    }

scripts/workers/cdr_presets/sip_navigator.py

@@ -0,0 +1,97 @@
+"""SIP Navigator (Cataleya Orchid One) preset.
+
+Orchid One publishes a REST CDR API in newer deployments, but older ones
+only expose the web CDR-export page. The preset tries the API first
+(requires ``api_key``); if the admin doesn't have API access, it falls
+back to the Playwright web-scrape flow.
+"""
+
+from __future__ import annotations
+
+import json
+import urllib.request
+from datetime import datetime, timedelta
+from typing import Iterable, Optional
+
+from .base import BasePreset, CredentialField, FetchedFile
+from ._scrape_base import ScrapePreset
+
+
+class SIPNavigatorPreset(ScrapePreset):
+    PRESET_SLUG = "sip_navigator"
+    LABEL = "SIP Navigator (Cataleya Orchid One)"
+    CDR_FORMAT = "generic_csv"
+    TRANSPORT_METHOD = "api"       # preferred; scrape fallback if api_key absent
+    DEFAULT_CRON = "0 3 * * *"
+
+    CREDENTIAL_FIELDS = (
+        CredentialField("admin_url", "Orchid One admin URL", "text",
+                        help="e.g. https://orchid.example.com"),
+        CredentialField("username", "Admin username", "text"),
+        CredentialField("password", "Admin password", "password", sensitive=True),
+        CredentialField("api_key", "Orchid One API key", "password",
+                        required=False, sensitive=True,
+                        help="Leave blank if the deployment doesn't expose the REST API."),
+    )
+
+    FORMAT_CONFIG = {
+        "start_time":    "setup_time",
+        "caller_number": "calling_party",
+        "called_number": "called_party",
+        "duration_sec":  "answer_duration",
+        "billed_amount": "charge_amount",
+        "call_id":       "global_call_id",
+        "trunk_group":   "ingress_peering",
+        "disposition":   "release_reason",
+        "ts_format":     "%Y-%m-%dT%H:%M:%S",
+    }
+
+    # ── API path (preferred) ────────────────────────────────────────
+
+    def validate(self, profile_config: dict, secrets: dict) -> tuple[bool, str]:
+        if secrets.get("api_key"):
+            try:
+                base = profile_config["admin_url"].rstrip("/")
+                req = urllib.request.Request(
+                    f"{base}/api/v1/system/status",
+                    headers={"Authorization": f"Bearer {secrets['api_key']}"},
+                )
+                with urllib.request.urlopen(req, timeout=15) as resp:
+                    return True, f"API reachable (HTTP {resp.status})"
+            except Exception as exc:
+                return False, f"Orchid API validate failed: {exc}"
+        # No API key — fall back to ScrapePreset.validate() (HEAD the admin URL)
+        return super().validate(profile_config, secrets)
+
+    def fetch(
+        self,
+        profile_config: dict,
+        secrets: dict,
+        since: Optional[datetime],
+    ) -> Iterable[FetchedFile]:
+        if secrets.get("api_key"):
+            return self._fetch_api(profile_config, secrets, since)
+        # No API key → scrape (scaffolded, raises until recon finalized)
+        return super().fetch(profile_config, secrets, since)
+
+    def _fetch_api(
+        self,
+        cfg: dict,
+        secrets: dict,
+        since: Optional[datetime],
+    ) -> Iterable[FetchedFile]:
+        base = cfg["admin_url"].rstrip("/")
+        since = since or (datetime.utcnow() - timedelta(days=1))
+        end = datetime.utcnow()
+        req = urllib.request.Request(
+            f"{base}/api/v1/cdrs?start={since:%Y-%m-%dT%H:%M:%SZ}"
+            f"&end={end:%Y-%m-%dT%H:%M:%SZ}&format=csv",
+            headers={"Authorization": f"Bearer {secrets['api_key']}",
+                     "Accept": "text/csv"},
+        )
+        with urllib.request.urlopen(req, timeout=60) as resp:
+            content = resp.read()
+        yield FetchedFile(
+            remote_path=f"sip_navigator_cdrs_{end:%Y%m%dT%H%M%SZ}.csv",
+            mtime=end, content=content, size_bytes=len(content),
+        )