78c04b8bc3
When any Playwright submission fails (selector not found, timeout, etc.): 1. Full-page screenshot captured and uploaded to MinIO 2. Telegram alert sent immediately with error details + screenshot link 3. Email alert to ops with same info 4. Admin todo includes screenshot MinIO path for debugging 5. Client order stays pending for manual completion Proactive selector health check (daily 7am CT cron): - Navigates to each portal (FCC RMD, USAC E-File, FCC CPNI/ECFS) - Verifies all critical selectors are still present in the DOM - If selectors are missing (UI changed): alerts via Telegram + email BEFORE any real client order fails - Reports which service slugs are affected Integrated into: - RMD filing handler (fccprod.servicenowservices.com) - Form 499-A handler (forms.universalservice.org) - Form 499-Q handler (already had error handling) Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
530 lines
24 KiB
Python
530 lines
24 KiB
Python
"""RMD Registration / Recertification filing handler.
|
|
|
|
Flow:
|
|
1. Generate the RMD certification letter (and Exhibit A if partial STIR/SHAKEN)
|
|
using the existing document generators.
|
|
2. Convert to PDF.
|
|
3. Launch an undetected browser, navigate to the FCC Robocall Mitigation
|
|
Database portal (https://apps.fcc.gov/rmd/), log in using the CORES
|
|
session previously authorized via the chrome-extension/fcc-access-helper
|
|
for this carrier's FRN.
|
|
4. Submit the certification (or recertification) form, attaching the
|
|
signed DOCX. Capture the confirmation page and store it in MinIO as
|
|
``rmd_confirmation_{order}.pdf``.
|
|
5. On success: ``UPDATE telecom_entities SET rmd_last_cert_date = NOW(),
|
|
rmd_confirmation_number = $conf`` so the next checkup reads green.
|
|
|
|
Prerequisite: the carrier's FRN must already have filings@performancewest.net
|
|
authorized as an agent in FCC CORES. The chrome-extension helper handles
|
|
this human-in-the-loop onboarding; it is a one-time-per-carrier step.
|
|
|
|
Idempotency: if ``already_filed(entity_id, "rmd")`` is True, the handler
|
|
skips the portal submission and returns just the packet (the customer
|
|
still gets their documents for their records).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
import os
|
|
from datetime import datetime
|
|
|
|
from .base_handler import BaseServiceHandler
|
|
from .telecom import filing_state
|
|
from .telecom.auto_filing import check_auto_filing, request_admin_review
|
|
from .telecom.undetected_browser import undetected_browser, human_delay
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
FCC_RMD_URL = os.environ.get("FCC_RMD_URL", "https://apps.fcc.gov/rmd/")
|
|
FCC_CORES_STORAGE_STATE = os.environ.get(
|
|
"FCC_CORES_STORAGE_STATE",
|
|
"/app/data/fcc_cores_session.json",
|
|
)
|
|
|
|
|
|
class RMDFilingHandler(BaseServiceHandler):
|
|
SERVICE_SLUG = "rmd-filing"
|
|
SERVICE_NAME = "RMD Registration / Recertification"
|
|
REQUIRES_LLM = False
|
|
|
|
async def process(self, order_data: dict) -> list[str]:
|
|
work_dir = self._make_work_dir()
|
|
order_number = order_data["name"]
|
|
entity = order_data.get("entity", {})
|
|
entity_id = entity.get("id")
|
|
date_str = datetime.now().strftime("%Y%m%d")
|
|
|
|
# ── Guard: require entity data before generating documents ───────
|
|
legal_name = entity.get("legal_name", "").strip()
|
|
if not legal_name:
|
|
logger.warning(
|
|
"RMDFilingHandler: no entity data for %s — pausing for intake",
|
|
order_number,
|
|
)
|
|
self._request_entity_intake(order_data)
|
|
return []
|
|
|
|
generated: list[str] = []
|
|
|
|
# ── 1. Generate the signed-ready packet ─────────────────────────
|
|
generated.extend(self._build_packet(order_number, entity, work_dir, date_str))
|
|
|
|
# ── 2. Idempotency gate ──────────────────────────────────────────
|
|
if entity_id and filing_state.already_filed(entity_id, "rmd"):
|
|
logger.info(
|
|
"RMDFilingHandler: RMD already on file for entity %s — returning packet only",
|
|
entity_id,
|
|
)
|
|
return generated
|
|
|
|
# ── 2b. Client review gate ─────────────────────────────────────
|
|
# The client MUST review and approve the certification before we
|
|
# submit. The perjury declaration (47 CFR § 1.16) is on them.
|
|
client_approved = order_data.get("client_approved", False)
|
|
if not client_approved:
|
|
# Upload documents to MinIO for the review portal
|
|
from scripts.document_gen import MinioStorage
|
|
storage = MinioStorage()
|
|
minio_paths = []
|
|
for path in generated:
|
|
if path.endswith(".pdf"):
|
|
remote = f"compliance/{order_number}/{os.path.basename(path)}"
|
|
try:
|
|
storage.upload_file(path, remote)
|
|
minio_paths.append(remote)
|
|
except Exception as exc:
|
|
logger.warning("MinIO upload failed for %s: %s", path, exc)
|
|
|
|
# Update order with review status + send review link
|
|
try:
|
|
import psycopg2
|
|
conn = psycopg2.connect(os.environ.get("DATABASE_URL", ""))
|
|
cur = conn.cursor()
|
|
cur.execute(
|
|
"""UPDATE compliance_orders
|
|
SET rmd_review_status = 'pending',
|
|
rmd_packet_minio_paths = %s
|
|
WHERE order_number = %s""",
|
|
(minio_paths, order_number),
|
|
)
|
|
conn.commit()
|
|
cur.close()
|
|
conn.close()
|
|
except Exception as exc:
|
|
logger.warning("Could not update review status: %s", exc)
|
|
|
|
# Send review link email to client
|
|
try:
|
|
customer_email = order_data.get("customer_email") or entity.get("contact_email")
|
|
customer_name = order_data.get("customer_name") or entity.get("contact_name", "")
|
|
if customer_email:
|
|
# Build JWT review URL
|
|
try:
|
|
import jwt as pyjwt
|
|
except ImportError:
|
|
import PyJWT as pyjwt # type: ignore
|
|
secret = os.environ.get("CUSTOMER_JWT_SECRET", "changeme")
|
|
domain = os.environ.get("DOMAIN", "performancewest.net")
|
|
token = pyjwt.encode(
|
|
{"order_id": order_number, "order_type": "compliance", "email": customer_email},
|
|
secret, algorithm="HS256",
|
|
)
|
|
review_url = f"https://{domain}/portal/rmd-review?token={token}"
|
|
|
|
if customer_email:
|
|
import smtplib
|
|
from email.mime.text import MIMEText
|
|
from email.mime.multipart import MIMEMultipart
|
|
subject = f"Action Required — Review your RMD certification for {entity.get('legal_name', order_number)}"
|
|
body = (
|
|
f"<h2>Your RMD Certification is Ready for Review</h2>"
|
|
f"<p>Hi {customer_name.split(' ')[0] if customer_name else 'there'},</p>"
|
|
f"<p>We've prepared your Robocall Mitigation Database certification for "
|
|
f"<strong>{entity.get('legal_name', '')}</strong> (FRN: {entity.get('frn', '')}).</p>"
|
|
f"<p>Before we submit this to the FCC, we need you to review the filing details "
|
|
f"and confirm everything is accurate. This is required because the certification "
|
|
f"includes a declaration under penalty of perjury (47 CFR § 1.16).</p>"
|
|
f"<p><a href='{review_url}' style='display:inline-block;background:#1e3a5f;color:#fff;"
|
|
f"padding:12px 28px;border-radius:6px;text-decoration:none;font-weight:600;'>"
|
|
f"Review & Approve Filing →</a></p>"
|
|
f"<p style='font-size:12px;color:#9ca3af;'>This link expires in 72 hours.</p>"
|
|
)
|
|
smtp_host = os.environ.get("SMTP_HOST", "co.carrierone.com")
|
|
smtp_port = int(os.environ.get("SMTP_PORT", "587"))
|
|
smtp_user = os.environ.get("SMTP_USER", "")
|
|
smtp_pass = os.environ.get("SMTP_PASS", "")
|
|
smtp_from = os.environ.get("SMTP_FROM", "Performance West <noreply@performancewest.net>")
|
|
if smtp_user and smtp_pass:
|
|
msg = MIMEMultipart("alternative")
|
|
msg["Subject"] = subject
|
|
msg["From"] = smtp_from
|
|
msg["To"] = customer_email
|
|
msg["Reply-To"] = "info@performancewest.net"
|
|
msg.attach(MIMEText(body, "html"))
|
|
with smtplib.SMTP(smtp_host, smtp_port) as server:
|
|
server.starttls()
|
|
server.login(smtp_user, smtp_pass)
|
|
server.send_message(msg)
|
|
logger.info("RMD review link sent to %s for %s", customer_email, order_number)
|
|
|
|
except Exception as exc:
|
|
logger.warning("Could not send RMD review email: %s", exc)
|
|
|
|
logger.info(
|
|
"RMDFilingHandler: paused for client review — order %s",
|
|
order_number,
|
|
)
|
|
return generated
|
|
|
|
# ── 2a. Auto-filing toggle — admin review when disabled ─────────
|
|
decision = check_auto_filing(order_data)
|
|
if not decision.may_submit:
|
|
logger.info(
|
|
"RMDFilingHandler: %s — staging for admin review (order=%s)",
|
|
decision.reason, order_number,
|
|
)
|
|
request_admin_review(
|
|
order_number=order_number,
|
|
service_slug=self.SERVICE_SLUG,
|
|
service_name=self.SERVICE_NAME,
|
|
entity_name=entity.get("legal_name", ""),
|
|
frn=entity.get("frn", ""),
|
|
packet_minio_paths=[f"compliance/{order_number}/{os.path.basename(p)}" for p in generated],
|
|
admin_email=decision.admin_email,
|
|
summary=(
|
|
f"RMD certification packet ready. STIR/SHAKEN posture: "
|
|
f"{entity.get('stir_shaken_status', 'N/A')}. "
|
|
f"Clicking Approve & File re-runs this handler against the FCC "
|
|
f"RMD portal at {FCC_RMD_URL}."
|
|
),
|
|
)
|
|
return generated
|
|
|
|
# ── 3. Submit to the FCC RMD portal ──────────────────────────────
|
|
confirmation_path, confirmation_number = await self._submit_to_rmd(
|
|
order_number=order_number,
|
|
entity=entity,
|
|
packet_docx=next((p for p in generated if p.endswith(".docx")), None),
|
|
work_dir=work_dir,
|
|
)
|
|
if confirmation_path:
|
|
generated.append(confirmation_path)
|
|
|
|
# ── 4. Persist success + email confirmation to client ──────────
|
|
if entity_id and confirmation_number:
|
|
filing_state.record_rmd_filing(entity_id, confirmation_number)
|
|
|
|
if confirmation_number:
|
|
try:
|
|
from scripts.workers.job_server import _send_filing_confirmation
|
|
from scripts.document_gen import MinioStorage
|
|
customer_email = order_data.get("customer_email") or entity.get("contact_email")
|
|
customer_name = order_data.get("customer_name") or entity.get("contact_name", "")
|
|
if customer_email:
|
|
conf_paths = [p for p in generated if "confirmation" in p.lower()]
|
|
_send_filing_confirmation(
|
|
customer_email=customer_email,
|
|
customer_name=customer_name,
|
|
order_number=order_number,
|
|
service_name=self.SERVICE_NAME,
|
|
confirmation_number=confirmation_number,
|
|
authority="FCC Robocall Mitigation Database (RMD)",
|
|
minio_paths=[f"compliance/{order_number}/{os.path.basename(p)}" for p in conf_paths],
|
|
storage=MinioStorage(),
|
|
)
|
|
except Exception as exc:
|
|
logger.warning("RMD filing confirmation email failed: %s", exc)
|
|
|
|
return generated
|
|
|
|
# ------------------------------------------------------------------ #
|
|
# Packet generation (shared with the checkup handler)
|
|
# ------------------------------------------------------------------ #
|
|
|
|
def _build_packet(
|
|
self,
|
|
order_number: str,
|
|
entity: dict,
|
|
work_dir: str,
|
|
date_str: str,
|
|
) -> list[str]:
|
|
"""Produce the RMD cert letter (+ Exhibit A if partial STIR/SHAKEN)."""
|
|
from scripts.document_gen.templates.rmd_letter_generator import (
|
|
generate_rmd_letter,
|
|
_determine_primary_role,
|
|
)
|
|
|
|
files: list[str] = []
|
|
|
|
rmd_docx = os.path.join(
|
|
work_dir, f"rmd_certification_letter_{order_number}_{date_str}.docx"
|
|
)
|
|
letter = generate_rmd_letter(
|
|
entity_name=entity.get("legal_name", ""),
|
|
dba_name=entity.get("dba_name", ""),
|
|
frn=entity.get("frn", ""),
|
|
rmd_number=entity.get("rmd_number", ""),
|
|
filer_id_499=entity.get("filer_id_499", ""),
|
|
address_street=entity.get("address_street", ""),
|
|
address_city=entity.get("address_city", ""),
|
|
address_state=entity.get("address_state", ""),
|
|
address_zip=entity.get("address_zip", ""),
|
|
contact_name=entity.get("contact_name", ""),
|
|
contact_title=entity.get("contact_title", ""),
|
|
contact_email=entity.get("contact_email", ""),
|
|
contact_phone=entity.get("contact_phone", ""),
|
|
ceo_name=entity.get("ceo_name", ""),
|
|
ceo_title=entity.get("ceo_title", "Chief Executive Officer"),
|
|
carrier_category=entity.get("carrier_category", "interconnected_voip"),
|
|
infra_type=entity.get("infra_type", "facilities"),
|
|
is_wholesale=entity.get("is_wholesale", False),
|
|
is_gateway_provider=entity.get("is_gateway_provider", False),
|
|
is_international_only=entity.get("is_international_only", False),
|
|
uses_ucaas_provider=entity.get("uses_ucaas_provider", False),
|
|
carrier_metadata=entity.get("carrier_metadata", {}),
|
|
stir_shaken_status=entity.get("stir_shaken_status", "complete_implementation"),
|
|
stir_shaken_cert_authority=entity.get("stir_shaken_cert_authority", ""),
|
|
upstream_provider_name=entity.get("upstream_provider_name", ""),
|
|
upstream_provider_frn=entity.get("upstream_provider_frn", ""),
|
|
output_path=rmd_docx,
|
|
)
|
|
if letter:
|
|
files.append(letter)
|
|
try:
|
|
files.append(self._convert_to_pdf(letter))
|
|
except Exception as exc:
|
|
logger.warning("RMD letter PDF conversion failed: %s", exc)
|
|
|
|
stir_status = entity.get("stir_shaken_status", "complete_implementation")
|
|
if stir_status in (
|
|
"partial_implementation",
|
|
"terminate_only",
|
|
"robocall_mitigation_only",
|
|
"exempt_small_carrier",
|
|
):
|
|
from scripts.document_gen.templates.rmd_exhibit_a_generator import (
|
|
generate_exhibit_a,
|
|
)
|
|
|
|
role = _determine_primary_role(
|
|
is_gateway_provider=entity.get("is_gateway_provider", False),
|
|
uses_ucaas_provider=entity.get("uses_ucaas_provider", False),
|
|
is_wholesale=entity.get("is_wholesale", False),
|
|
is_international_only=entity.get("is_international_only", False),
|
|
infra_type=entity.get("infra_type", "facilities"),
|
|
)
|
|
exhibit_docx = os.path.join(
|
|
work_dir,
|
|
f"robocall_mitigation_program_{order_number}_{date_str}.docx",
|
|
)
|
|
# Map stir_shaken_status to Exhibit A rmd_option
|
|
_STATUS_TO_OPTION = {
|
|
"complete_implementation": "option1",
|
|
"partial_implementation": "option2",
|
|
"terminate_only": "terminate_only",
|
|
"robocall_mitigation_only": "option3",
|
|
"exempt_small_carrier": "option3",
|
|
}
|
|
exhibit = generate_exhibit_a(
|
|
entity_name=entity.get("legal_name", ""),
|
|
frn=entity.get("frn", ""),
|
|
ocn=entity.get("ocn", ""),
|
|
carrier_role=role,
|
|
carrier_metadata=entity.get("carrier_metadata", {}),
|
|
upstream_provider_name=entity.get("upstream_provider_name", ""),
|
|
is_wholesale=entity.get("is_wholesale", False),
|
|
is_gateway=entity.get("is_gateway_provider", False),
|
|
foreign_traffic=entity.get("is_international_only", False),
|
|
rmd_option=_STATUS_TO_OPTION.get(stir_status, "option2"),
|
|
contact_name=entity.get("contact_name", ""),
|
|
contact_title=entity.get("contact_title", ""),
|
|
contact_email=entity.get("contact_email", ""),
|
|
contact_phone=entity.get("contact_phone", ""),
|
|
address=entity.get("address", ""),
|
|
llm_generate=self._call_llm,
|
|
output_path=exhibit_docx,
|
|
)
|
|
if exhibit:
|
|
files.append(exhibit)
|
|
try:
|
|
files.append(self._convert_to_pdf(exhibit))
|
|
except Exception as exc:
|
|
logger.warning("Exhibit A PDF conversion failed: %s", exc)
|
|
|
|
return files
|
|
|
|
# ------------------------------------------------------------------ #
|
|
# FCC RMD Playwright submission
|
|
# ------------------------------------------------------------------ #
|
|
|
|
async def _submit_to_rmd(
|
|
self,
|
|
*,
|
|
order_number: str,
|
|
entity: dict,
|
|
packet_docx: str | None,
|
|
work_dir: str,
|
|
) -> tuple[str | None, str]:
|
|
"""Submit the RMD certification and return (confirmation_pdf, conf_number).
|
|
|
|
Returns ``(None, "")`` if the submission could not be attempted
|
|
(e.g. no authorized CORES session available). In that case the
|
|
customer still receives the packet; an admin ToDo is created so a
|
|
human can authorize the FRN and the handler can be replayed.
|
|
"""
|
|
|
|
frn = entity.get("frn", "").strip()
|
|
if not frn:
|
|
logger.warning("RMDFilingHandler: no FRN — skipping portal submission")
|
|
self._create_admin_todo(
|
|
order_number,
|
|
"RMD filing requires an FRN but none was on file for this carrier. "
|
|
"Capture the FRN on the telecom entity and re-dispatch.",
|
|
)
|
|
return None, ""
|
|
|
|
storage_exists = os.path.exists(FCC_CORES_STORAGE_STATE)
|
|
storage_state = FCC_CORES_STORAGE_STATE if storage_exists else None
|
|
|
|
confirmation_path = os.path.join(
|
|
work_dir, f"rmd_confirmation_{order_number}.pdf"
|
|
)
|
|
confirmation_number = ""
|
|
|
|
try:
|
|
async with undetected_browser(
|
|
headless=True,
|
|
storage_state=storage_state,
|
|
) as (ctx, page):
|
|
await page.goto(FCC_RMD_URL, wait_until="domcontentloaded")
|
|
await human_delay(1.5, 3.0)
|
|
|
|
# If we weren't handed a logged-in session, redirect to
|
|
# CORES login will fire — detect that and fall through to
|
|
# the admin-ToDo path.
|
|
if "login" in page.url.lower() or "coresWeb" in page.url:
|
|
logger.warning(
|
|
"RMDFilingHandler: CORES session not authorized for FRN %s; "
|
|
"admin must run chrome-extension FCC access helper first",
|
|
frn,
|
|
)
|
|
self._create_admin_todo(
|
|
order_number,
|
|
f"RMD portal required CORES login for FRN {frn}. "
|
|
"Run the FCC Access Helper Chrome extension to authorize "
|
|
"filings@performancewest.net on this carrier's FRN, export "
|
|
f"the session to {FCC_CORES_STORAGE_STATE}, then re-dispatch "
|
|
f"order {order_number}.",
|
|
)
|
|
return None, ""
|
|
|
|
# Navigate to the certification form. The real selectors are
|
|
# pulled from a live recon session (TODO before production —
|
|
# covered by state-automation-status.md pattern).
|
|
await page.wait_for_selector("text=Certification", timeout=20000)
|
|
await page.click("text=File Certification")
|
|
await human_delay()
|
|
|
|
# Pre-populate the form from entity data.
|
|
await page.fill('input[name="frn"]', frn)
|
|
await page.fill(
|
|
'input[name="company_legal_name"]',
|
|
entity.get("legal_name", ""),
|
|
)
|
|
|
|
# Attach the signed-ready packet DOCX (plus PDF if present).
|
|
if packet_docx:
|
|
await page.set_input_files(
|
|
'input[type="file"][name="certification_doc"]',
|
|
packet_docx,
|
|
)
|
|
|
|
# STIR/SHAKEN implementation status — value must match the
|
|
# RMD form radio options. terminate_only maps to
|
|
# partial_implementation on the FCC form (verification-only).
|
|
stir_status = entity.get("stir_shaken_status", "complete_implementation")
|
|
if stir_status == "terminate_only":
|
|
stir_status = "partial_implementation"
|
|
await page.click(f'input[name="stir_shaken_status"][value="{stir_status}"]')
|
|
await human_delay()
|
|
|
|
# Review + submit.
|
|
await page.click('button[type="submit"]')
|
|
await page.wait_for_selector(
|
|
"text=Confirmation Number", timeout=60000
|
|
)
|
|
await human_delay(2.0, 4.0)
|
|
|
|
# Capture confirmation.
|
|
body = await page.locator("body").inner_text()
|
|
for line in body.splitlines():
|
|
if "Confirmation Number" in line:
|
|
parts = line.split(":", 1)
|
|
if len(parts) == 2:
|
|
confirmation_number = parts[1].strip()
|
|
break
|
|
|
|
# Save the confirmation page as PDF.
|
|
await page.pdf(path=confirmation_path, format="Letter")
|
|
|
|
logger.info(
|
|
"RMDFilingHandler: submitted FRN %s, confirmation %s",
|
|
frn,
|
|
confirmation_number,
|
|
)
|
|
return confirmation_path, confirmation_number
|
|
|
|
except Exception as exc:
|
|
logger.exception("RMDFilingHandler: portal submission failed: %s", exc)
|
|
|
|
# Upload screenshot + alert ops via Telegram
|
|
screenshot_key = None
|
|
try:
|
|
from scripts.workers.services.telecom.playwright_monitor import (
|
|
upload_failure_screenshot_async, alert_playwright_failure,
|
|
)
|
|
screenshot_key = await upload_failure_screenshot_async(
|
|
page, order_number, "rmd-filing", work_dir,
|
|
)
|
|
alert_playwright_failure(
|
|
order_number=order_number,
|
|
service_slug="rmd-filing",
|
|
service_name="RMD Registration",
|
|
entity_name=entity.get("legal_name", ""),
|
|
error=exc,
|
|
screenshot_key=screenshot_key,
|
|
portal_url="https://fccprod.servicenowservices.com/rmd",
|
|
)
|
|
except Exception as alert_exc:
|
|
logger.warning("Playwright failure alert failed: %s", alert_exc)
|
|
|
|
self._create_admin_todo(
|
|
order_number,
|
|
f"Automated RMD submission failed for FRN {frn}: {exc}. "
|
|
f"{'Screenshot: MinIO ' + screenshot_key if screenshot_key else 'No screenshot captured.'}"
|
|
f"\nFile manually at https://fccprod.servicenowservices.com/rmd",
|
|
)
|
|
return None, ""
|
|
|
|
# ------------------------------------------------------------------ #
|
|
# Admin fallback
|
|
# ------------------------------------------------------------------ #
|
|
|
|
def _create_admin_todo(self, order_number: str, description: str) -> None:
|
|
"""Create an ERPNext ToDo so a human can unblock the filing."""
|
|
try:
|
|
from scripts.workers.erpnext_client import ERPNextClient
|
|
|
|
ERPNextClient().create_resource(
|
|
"ToDo",
|
|
{
|
|
"description": (
|
|
f"[{self.SERVICE_SLUG}] {order_number}\n\n{description}"
|
|
),
|
|
"priority": "High",
|
|
"role": "Accounting Advisor",
|
|
},
|
|
)
|
|
except Exception as exc:
|
|
logger.error("Could not create admin ToDo: %s", exc)
|