Initial commit — Performance West telecom compliance platform

Includes: API (Express/TypeScript), Astro site, Python workers,
document generators, FCC compliance tools, Canada CRTC formation,
Ansible infrastructure, and deployment scripts.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

scripts/workers/cdr_transports/__init__.py

@@ -0,0 +1,28 @@
+"""Generic CDR transport adapters (pull side).
+
+Used when the customer's profile has ``switch_preset IS NULL`` — i.e. they
+picked "Other (configure manually)" in the portal. Switch-specific
+pulls go through ``scripts.workers.cdr_presets`` instead; those may
+internally reuse these transport classes for their mechanics.
+"""
+
+from .base import BaseTransport, RemoteFile
+from .sftp_transport import SFTPTransport
+from .ftp_transport import FTPTransport
+from .ftps_transport import FTPSTransport
+from .https_transport import HTTPSTransport
+from .s3_transport import S3Transport
+
+TRANSPORTS: dict[str, type[BaseTransport]] = {
+    "sftp":  SFTPTransport,
+    "ftp":   FTPTransport,
+    "ftps":  FTPSTransport,
+    "https": HTTPSTransport,
+    "s3":    S3Transport,
+}
+
+__all__ = [
+    "BaseTransport", "RemoteFile", "TRANSPORTS",
+    "SFTPTransport", "FTPTransport", "FTPSTransport",
+    "HTTPSTransport", "S3Transport",
+]

scripts/workers/cdr_transports/base.py

@@ -0,0 +1,77 @@
+"""Base class for CDR pull transports.
+
+Every transport implements three methods:
+  * validate()                  — test the credentials + reachability
+  * list_since(mtime)           — list RemoteFile entries newer than mtime
+  * fetch(remote_path) -> bytes — download a single file
+The cdr_puller worker orchestrates: validate on config save, list on each
+poll, fetch + stream to MinIO for each new file.
+"""
+
+from __future__ import annotations
+
+import fnmatch
+from dataclasses import dataclass
+from datetime import datetime
+from typing import Iterable, Optional
+
+
+@dataclass
+class RemoteFile:
+    path: str                  # remote key (e.g. "/var/log/cdr/2026-04-15.csv")
+    mtime: datetime
+    size_bytes: int = 0
+
+
+class TransportError(Exception):
+    """Raised when a transport operation fails non-fatally.
+
+    Callers retry or surface to the admin; they do NOT mark the profile
+    permanently failed unless multiple consecutive attempts raise.
+    """
+
+
+class BaseTransport:
+    """Abstract transport. Subclasses implement the three methods below."""
+
+    TRANSPORT_SLUG = ""
+
+    def __init__(
+        self,
+        *,
+        host: str,
+        port: Optional[int] = None,
+        username: Optional[str] = None,
+        password: Optional[str] = None,
+        private_key: Optional[str] = None,
+        remote_glob: str = "*",
+        timeout: int = 30,
+        extra: Optional[dict] = None,
+    ):
+        self.host = host
+        self.port = port
+        self.username = username
+        self.password = password
+        self.private_key = private_key
+        self.remote_glob = remote_glob
+        self.timeout = timeout
+        self.extra = extra or {}
+
+    # ──────────────────────────────────────────────────────────────────
+
+    def validate(self) -> tuple[bool, str]:
+        """Attempt to connect + list the target directory. Returns (ok, detail)."""
+        raise NotImplementedError
+
+    def list_since(self, since: Optional[datetime]) -> Iterable[RemoteFile]:
+        """Yield RemoteFile entries with mtime > since that match remote_glob."""
+        raise NotImplementedError
+
+    def fetch(self, remote_path: str) -> bytes:
+        """Download and return the file bytes."""
+        raise NotImplementedError
+
+    # ── Helpers ────────────────────────────────────────────────────────
+
+    def _match_glob(self, name: str) -> bool:
+        return fnmatch.fnmatch(name, self.remote_glob)

scripts/workers/cdr_transports/ftp_transport.py

@@ -0,0 +1,119 @@
+"""Plain FTP transport (stdlib ftplib)."""
+
+from __future__ import annotations
+
+import ftplib
+import io
+import logging
+import posixpath
+from datetime import datetime
+from typing import Iterable, Optional
+
+from .base import BaseTransport, RemoteFile, TransportError
+
+logger = logging.getLogger(__name__)
+
+
+class FTPTransport(BaseTransport):
+    TRANSPORT_SLUG = "ftp"
+    _FTP_CLASS = ftplib.FTP
+
+    def _connect(self):
+        conn = self._FTP_CLASS(timeout=self.timeout)
+        conn.connect(self.host, self.port or 21)
+        conn.login(self.username or "", self.password or "")
+        # Some FTP servers need binary mode explicitly
+        conn.sendcmd("TYPE I")
+        return conn
+
+    def _remote_dir(self) -> str:
+        if "/" in self.remote_glob:
+            return posixpath.dirname(self.remote_glob) or "/"
+        return self.extra.get("remote_dir", ".")
+
+    def _glob_pattern(self) -> str:
+        if "/" in self.remote_glob:
+            return posixpath.basename(self.remote_glob)
+        return self.remote_glob
+
+    def validate(self) -> tuple[bool, str]:
+        try:
+            conn = self._connect()
+            try:
+                conn.cwd(self._remote_dir())
+                conn.nlst()
+                return True, "connected + listed remote directory"
+            finally:
+                conn.quit()
+        except Exception as exc:
+            return False, f"FTP validate failed: {exc}"
+
+    def list_since(self, since: Optional[datetime]) -> Iterable[RemoteFile]:
+        import fnmatch
+        conn = self._connect()
+        try:
+            conn.cwd(self._remote_dir())
+            pattern = self._glob_pattern()
+            # MLSD is preferred; fall back to LIST parsing if unsupported
+            try:
+                entries = list(conn.mlsd())
+            except (ftplib.error_perm, AttributeError):
+                entries = _parse_list_fallback(conn)
+            for name, facts in entries:
+                if facts.get("type") not in (None, "file"):
+                    continue
+                if pattern and not fnmatch.fnmatch(name, pattern):
+                    continue
+                modify = facts.get("modify")
+                if modify:
+                    try:
+                        mtime = datetime.strptime(modify, "%Y%m%d%H%M%S")
+                    except ValueError:
+                        mtime = datetime.utcnow()
+                else:
+                    mtime = datetime.utcnow()
+                if since and mtime <= since:
+                    continue
+                size = int(facts.get("size", 0) or 0)
+                yield RemoteFile(
+                    path=posixpath.join(self._remote_dir(), name),
+                    mtime=mtime, size_bytes=size,
+                )
+        finally:
+            conn.quit()
+
+    def fetch(self, remote_path: str) -> bytes:
+        conn = self._connect()
+        try:
+            conn.cwd(posixpath.dirname(remote_path) or "/")
+            buf = io.BytesIO()
+            conn.retrbinary(f"RETR {posixpath.basename(remote_path)}", buf.write)
+            return buf.getvalue()
+        finally:
+            conn.quit()
+
+
+def _parse_list_fallback(conn):
+    """For servers without MLSD, parse LIST output (best-effort)."""
+    lines: list[str] = []
+    conn.retrlines("LIST", lines.append)
+    for line in lines:
+        parts = line.split(maxsplit=8)
+        if len(parts) < 9:
+            continue
+        name = parts[-1]
+        try:
+            size = int(parts[4])
+        except ValueError:
+            size = 0
+        yield name, {"type": "file", "size": str(size)}
+
+
+class FTPSPlain(ftplib.FTP_TLS):
+    """Trivial subclass enabling data-channel protection by default."""
+
+    def __init__(self, *a, **kw):
+        super().__init__(*a, **kw)
+
+
+__all__ = ["FTPTransport"]

scripts/workers/cdr_transports/ftps_transport.py

@@ -0,0 +1,21 @@
+"""FTPS (explicit TLS) transport — inherits FTP transport behavior."""
+
+from __future__ import annotations
+
+import ftplib
+
+from .ftp_transport import FTPTransport
+
+
+class FTPSTransport(FTPTransport):
+    TRANSPORT_SLUG = "ftps"
+    _FTP_CLASS = ftplib.FTP_TLS
+
+    def _connect(self):
+        conn = self._FTP_CLASS(timeout=self.timeout)
+        conn.connect(self.host, self.port or 21)
+        conn.login(self.username or "", self.password or "")
+        # Enable encryption on the data channel too (most servers require this)
+        conn.prot_p()
+        conn.sendcmd("TYPE I")
+        return conn

scripts/workers/cdr_transports/https_transport.py

@@ -0,0 +1,129 @@
+"""HTTPS transport — for switches that expose a REST or file-download API.
+
+The remote endpoint is expected to be one of:
+  (a) a directory listing in JSON (array of objects with name + mtime + size), or
+  (b) a single file endpoint that is polled directly (treat as a named file
+      with current-timestamp mtime and an If-Modified-Since header).
+
+Authentication options:
+  * Bearer token (``extra["bearer_token"]``)
+  * Basic auth (``username``/``password``)
+  * Custom header(s) (``extra["headers"]`` — dict)
+"""
+
+from __future__ import annotations
+
+import base64
+import json
+import logging
+import posixpath
+from datetime import datetime
+from email.utils import parsedate_to_datetime
+from typing import Iterable, Optional
+from urllib.parse import urljoin
+
+import urllib.request
+import urllib.error
+
+from .base import BaseTransport, RemoteFile, TransportError
+
+logger = logging.getLogger(__name__)
+
+
+class HTTPSTransport(BaseTransport):
+    TRANSPORT_SLUG = "https"
+
+    def _headers(self) -> dict:
+        headers = {"User-Agent": "PerformanceWest-CDR-Puller/1.0"}
+        if self.username and self.password:
+            token = base64.b64encode(
+                f"{self.username}:{self.password}".encode("utf-8")
+            ).decode("ascii")
+            headers["Authorization"] = f"Basic {token}"
+        elif self.extra.get("bearer_token"):
+            headers["Authorization"] = f"Bearer {self.extra['bearer_token']}"
+        for k, v in (self.extra.get("headers") or {}).items():
+            headers[k] = v
+        return headers
+
+    def _base_url(self) -> str:
+        scheme = self.extra.get("scheme", "https")
+        port = f":{self.port}" if self.port and self.port not in (80, 443) else ""
+        return f"{scheme}://{self.host}{port}/"
+
+    def _request(self, url: str, timeout: Optional[int] = None) -> bytes:
+        req = urllib.request.Request(url, headers=self._headers())
+        try:
+            with urllib.request.urlopen(req, timeout=timeout or self.timeout) as resp:
+                return resp.read()
+        except urllib.error.HTTPError as exc:
+            raise TransportError(f"HTTP {exc.code}: {exc.reason}") from exc
+        except Exception as exc:
+            raise TransportError(str(exc)) from exc
+
+    def validate(self) -> tuple[bool, str]:
+        try:
+            url = urljoin(self._base_url(), self.remote_glob.lstrip("/"))
+            # Use HEAD if the endpoint supports it, fall back to GET.
+            req = urllib.request.Request(url, headers=self._headers(), method="HEAD")
+            try:
+                with urllib.request.urlopen(req, timeout=self.timeout) as resp:
+                    return True, f"HTTP {resp.status}"
+            except urllib.error.HTTPError as exc:
+                if exc.code == 405:   # method not allowed → fallback to GET
+                    self._request(url)
+                    return True, "GET ok"
+                return False, f"HTTP {exc.code}: {exc.reason}"
+        except Exception as exc:
+            return False, f"HTTPS validate failed: {exc}"
+
+    def list_since(self, since: Optional[datetime]) -> Iterable[RemoteFile]:
+        """Query the JSON listing endpoint (extra['listing_url'] or remote_glob)."""
+        listing_url = urljoin(
+            self._base_url(),
+            (self.extra.get("listing_url") or self.remote_glob).lstrip("/"),
+        )
+        try:
+            body = self._request(listing_url)
+            parsed = json.loads(body.decode("utf-8"))
+        except Exception as exc:
+            logger.warning("HTTPS listing failed: %s", exc)
+            return
+        if isinstance(parsed, dict) and "files" in parsed:
+            parsed = parsed["files"]
+        if not isinstance(parsed, list):
+            logger.warning("HTTPS listing did not return an array")
+            return
+        for entry in parsed:
+            name = entry.get("name") or entry.get("path")
+            if not name:
+                continue
+            mtime_raw = entry.get("modified") or entry.get("mtime") or entry.get("last_modified")
+            mtime = _parse_timestamp(mtime_raw) if mtime_raw else datetime.utcnow()
+            if since and mtime <= since:
+                continue
+            yield RemoteFile(
+                path=name, mtime=mtime,
+                size_bytes=int(entry.get("size", 0) or 0),
+            )
+
+    def fetch(self, remote_path: str) -> bytes:
+        url = urljoin(self._base_url(), remote_path.lstrip("/"))
+        return self._request(url)
+
+
+def _parse_timestamp(value) -> datetime:
+    """Accept ISO-8601, RFC 2822, or Unix epoch."""
+    if isinstance(value, (int, float)):
+        return datetime.utcfromtimestamp(value)
+    s = str(value).strip()
+    if s.isdigit():
+        return datetime.utcfromtimestamp(int(s))
+    try:
+        return datetime.fromisoformat(s.replace("Z", "+00:00")).replace(tzinfo=None)
+    except ValueError:
+        pass
+    try:
+        return parsedate_to_datetime(s).replace(tzinfo=None)
+    except (TypeError, ValueError):
+        return datetime.utcnow()

scripts/workers/cdr_transports/s3_transport.py

@@ -0,0 +1,93 @@
+"""S3 transport — pulls CDRs from a customer-provided S3 (or S3-compatible) bucket.
+
+Credentials:
+  * username / password → AWS access_key / secret_key
+  * extra["endpoint_url"]  (optional — set for S3-compatible backends like
+    Wasabi, MinIO, Backblaze)
+  * extra["region"]  (default "us-east-1")
+  * extra["bucket"]  (required)
+  * remote_glob  — object-key prefix or glob (e.g. "cdrs/2026/*.csv.gz")
+"""
+
+from __future__ import annotations
+
+import fnmatch
+import logging
+import posixpath
+from datetime import datetime, timezone
+from typing import Iterable, Optional
+
+from .base import BaseTransport, RemoteFile, TransportError
+
+logger = logging.getLogger(__name__)
+
+
+class S3Transport(BaseTransport):
+    TRANSPORT_SLUG = "s3"
+
+    def _client(self):
+        try:
+            import boto3
+        except ImportError as exc:
+            raise TransportError("boto3 not installed") from exc
+        kwargs = {
+            "aws_access_key_id": self.username,
+            "aws_secret_access_key": self.password,
+            "region_name": self.extra.get("region", "us-east-1"),
+        }
+        if self.extra.get("endpoint_url"):
+            kwargs["endpoint_url"] = self.extra["endpoint_url"]
+        return boto3.client("s3", **kwargs)
+
+    def _bucket(self) -> str:
+        bucket = self.extra.get("bucket") or self.host
+        if not bucket:
+            raise TransportError("S3 bucket not configured (extra.bucket)")
+        return bucket
+
+    def _prefix_and_pattern(self) -> tuple[str, str]:
+        """Split remote_glob into (key-prefix, glob-pattern)."""
+        if "*" in self.remote_glob or "?" in self.remote_glob:
+            # Pattern — the prefix is everything up to the first wildcard.
+            # e.g. "cdrs/2026/*.csv.gz" -> prefix="cdrs/2026/"
+            idx = min(
+                self.remote_glob.find(c) for c in "*?["
+                if c in self.remote_glob
+            )
+            return self.remote_glob[:idx], self.remote_glob
+        # No wildcards — treat as a directory prefix.
+        prefix = self.remote_glob.rstrip("/") + "/"
+        return prefix, "*"
+
+    def validate(self) -> tuple[bool, str]:
+        try:
+            client = self._client()
+            prefix, _ = self._prefix_and_pattern()
+            resp = client.list_objects_v2(
+                Bucket=self._bucket(), Prefix=prefix, MaxKeys=1,
+            )
+            return True, f"bucket reachable (KeyCount={resp.get('KeyCount', 0)})"
+        except Exception as exc:
+            return False, f"S3 validate failed: {exc}"
+
+    def list_since(self, since: Optional[datetime]) -> Iterable[RemoteFile]:
+        client = self._client()
+        bucket = self._bucket()
+        prefix, pattern = self._prefix_and_pattern()
+        paginator = client.get_paginator("list_objects_v2")
+        for page in paginator.paginate(Bucket=bucket, Prefix=prefix):
+            for obj in page.get("Contents", []) or []:
+                key = obj["Key"]
+                if pattern != "*" and not fnmatch.fnmatch(key, pattern):
+                    continue
+                mtime = obj["LastModified"]
+                if mtime.tzinfo is not None:
+                    mtime = mtime.astimezone(timezone.utc).replace(tzinfo=None)
+                if since and mtime <= since:
+                    continue
+                yield RemoteFile(path=key, mtime=mtime, size_bytes=obj.get("Size", 0))
+
+    def fetch(self, remote_path: str) -> bytes:
+        client = self._client()
+        resp = client.get_object(Bucket=self._bucket(), Key=remote_path)
+        return resp["Body"].read()

scripts/workers/cdr_transports/sftp_transport.py

@@ -0,0 +1,99 @@
+"""SFTP transport (paramiko)."""
+
+from __future__ import annotations
+
+import io
+import logging
+import os
+import posixpath
+from datetime import datetime
+from typing import Iterable, Optional
+
+from .base import BaseTransport, RemoteFile, TransportError
+
+logger = logging.getLogger(__name__)
+
+
+class SFTPTransport(BaseTransport):
+    TRANSPORT_SLUG = "sftp"
+
+    def _connect(self):
+        try:
+            import paramiko
+        except ImportError as exc:
+            raise TransportError("paramiko not installed") from exc
+
+        client = paramiko.SSHClient()
+        client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+        kwargs: dict = {
+            "hostname": self.host,
+            "port": self.port or 22,
+            "username": self.username,
+            "timeout": self.timeout,
+            "allow_agent": False,
+            "look_for_keys": False,
+        }
+        if self.private_key:
+            pkey = paramiko.RSAKey.from_private_key(io.StringIO(self.private_key))
+            kwargs["pkey"] = pkey
+        else:
+            kwargs["password"] = self.password
+        client.connect(**kwargs)
+        return client
+
+    def _remote_dir(self) -> str:
+        # If remote_glob is a path+glob ("/var/log/cdr/*.csv"), split dir off
+        if "/" in self.remote_glob:
+            return posixpath.dirname(self.remote_glob) or "/"
+        return self.extra.get("remote_dir", ".")
+
+    def _glob_pattern(self) -> str:
+        if "/" in self.remote_glob:
+            return posixpath.basename(self.remote_glob)
+        return self.remote_glob
+
+    def validate(self) -> tuple[bool, str]:
+        try:
+            client = self._connect()
+            try:
+                sftp = client.open_sftp()
+                sftp.listdir(self._remote_dir())
+                return True, "connected + listed remote directory"
+            finally:
+                client.close()
+        except Exception as exc:
+            return False, f"SFTP validate failed: {exc}"
+
+    def list_since(self, since: Optional[datetime]) -> Iterable[RemoteFile]:
+        client = self._connect()
+        try:
+            sftp = client.open_sftp()
+            remote_dir = self._remote_dir()
+            pattern = self._glob_pattern()
+            for attr in sftp.listdir_attr(remote_dir):
+                if not self._match_glob_basename(attr.filename, pattern):
+                    continue
+                mtime = datetime.utcfromtimestamp(attr.st_mtime or 0)
+                if since and mtime <= since:
+                    continue
+                yield RemoteFile(
+                    path=posixpath.join(remote_dir, attr.filename),
+                    mtime=mtime,
+                    size_bytes=attr.st_size or 0,
+                )
+        finally:
+            client.close()
+
+    def fetch(self, remote_path: str) -> bytes:
+        client = self._connect()
+        try:
+            sftp = client.open_sftp()
+            with sftp.open(remote_path, "rb") as fh:
+                return fh.read()
+        finally:
+            client.close()
+
+    @staticmethod
+    def _match_glob_basename(name: str, pattern: str) -> bool:
+        import fnmatch
+        return fnmatch.fnmatch(name, pattern) if pattern else True