b375385efd
All transactional/worker senders built multipart/alternative (or mixed) messages with ONLY an HTML part. A single-part multipart/alternative is malformed and HTML-only mail is a spam-score signal -- the same class of deliverability bug that hurt the campaign pipeline, but on the telecom / filing / customer-transactional path (499-Q reminders, RMD/FCC filing review links, intake/completion/delivery emails, commissions, etc). - worker_email.send_worker_email: auto-derive plaintext from HTML when caller omits text= (fixes the shared helper for all current+future use) - 16 rolled-their-own senders in scripts/workers/** + scripts/formation/ document_delivery.py: attach html_to_text(...) plaintext sibling before the HTML part (job_server + document_delivery wrap text+html in an alternative sub-part so PDFs still attach to the mixed root) - api/src/email.ts: add dependency-free htmlToText() and default sendEmail text to it (fixes checkout/webhook HTML-only sends) Verified: all py files compile + import at runtime, api tsc passes, htmlToText handles hrefs/lists/entities, 11 plaintext unit tests pass. Telecom campaign 407 (Jun 8) was HTML-only + sent in the DKIM-broken window -> 384 sent / 0 clicks (same junked-mail signature).
359 lines
16 KiB
Python
359 lines
16 KiB
Python
"""FCC Form 499-Q Quarterly Filing Handler.
|
|
|
|
After the client submits quarterly revenue via /order/fcc-499q, this handler:
|
|
1. Generates a 499-Q prep summary
|
|
2. Checks auto-filing toggle (same as 499-A)
|
|
3. If auto-filing enabled: submits to USAC E-File via Playwright
|
|
4. Captures confirmation number, sends client confirmation email
|
|
5. Records filing in the database
|
|
|
|
The 499-Q is simpler than the 499-A — just 4 revenue categories
|
|
projected for the quarter, determining the quarterly USF contribution.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import logging
|
|
import os
|
|
import tempfile
|
|
from datetime import datetime
|
|
|
|
from .base_handler import BaseServiceHandler
|
|
from .telecom.auto_filing import check_auto_filing, request_admin_review
|
|
|
|
logger = logging.getLogger("workers.services.form_499q")
|
|
|
|
USAC_EFILE_URL = "https://forms.universalservice.org"
|
|
|
|
|
|
class Form499QHandler(BaseServiceHandler):
|
|
SERVICE_SLUG = "fcc-499q"
|
|
SERVICE_NAME = "FCC Form 499-Q Quarterly Filing"
|
|
|
|
def _create_admin_todo(self, order_number: str, description: str) -> None:
|
|
try:
|
|
from scripts.workers.erpnext_client import ERPNextClient
|
|
ERPNextClient().create_resource("ToDo", {
|
|
"description": f"[{self.SERVICE_SLUG}] {order_number}\n\n{description}",
|
|
"priority": "High",
|
|
"role": "Accounting Advisor",
|
|
})
|
|
except Exception as exc:
|
|
logger.error("Could not create admin ToDo: %s", exc)
|
|
|
|
async def process(self, order_data: dict) -> dict | None:
|
|
order_number = order_data.get("order_number", "")
|
|
entity = order_data.get("entity", {})
|
|
intake_data = order_data.get("intake_data", {})
|
|
|
|
if not intake_data.get("intake_completed"):
|
|
logger.info("Form499QHandler: %s intake not completed — waiting", order_number)
|
|
return []
|
|
|
|
quarter = intake_data.get("quarter", "?")
|
|
revenue = intake_data.get("revenue", {})
|
|
filer_id = intake_data.get("filer_id_499") or entity.get("filer_id_499", "")
|
|
frn = intake_data.get("frn") or entity.get("frn", "")
|
|
legal_name = entity.get("legal_name") or intake_data.get("entity_name", "")
|
|
|
|
logger.info(
|
|
"Form499QHandler: %s %s for %s (total: $%.2f)",
|
|
order_number, quarter, legal_name, revenue.get("total", 0),
|
|
)
|
|
|
|
# ── Auto-filing check ──────────────────────────────────────────
|
|
decision = check_auto_filing(order_data)
|
|
if not decision.may_submit:
|
|
logger.info("Form499QHandler: staging for admin review (order=%s)", order_number)
|
|
request_admin_review(
|
|
order_number=order_number,
|
|
service_slug=self.SERVICE_SLUG,
|
|
service_name=self.SERVICE_NAME,
|
|
entity_name=legal_name,
|
|
frn=frn,
|
|
packet_minio_paths=[],
|
|
admin_email=decision.admin_email,
|
|
summary=(
|
|
f"499-Q {quarter} ready for {legal_name}. "
|
|
f"Filer ID: {filer_id}. Total revenue: ${revenue.get('total', 0):.2f}. "
|
|
f"Due: {intake_data.get('due_date', '?')}. "
|
|
f"CC Inter: ${revenue.get('carriers_carrier_interstate', 0):.2f}, "
|
|
f"CC Intra: ${revenue.get('carriers_carrier_intrastate', 0):.2f}, "
|
|
f"EU Inter: ${revenue.get('end_user_interstate', 0):.2f}, "
|
|
f"EU Intra: ${revenue.get('end_user_intrastate', 0):.2f}. "
|
|
f"Submit via USAC E-File at {USAC_EFILE_URL}."
|
|
),
|
|
)
|
|
# Send client confirmation that we received their data
|
|
self._send_received_email(
|
|
to=entity.get("contact_email") or order_data.get("customer_email", ""),
|
|
entity_name=legal_name,
|
|
order_number=order_number,
|
|
quarter=quarter,
|
|
due_date=intake_data.get("due_date", ""),
|
|
)
|
|
return [] # no files to upload — admin will file manually
|
|
|
|
# ── USAC E-File submission via Playwright ──────────────────────
|
|
work_dir = tempfile.mkdtemp(prefix=f"499q_{order_number}_")
|
|
confirmation_number = ""
|
|
|
|
try:
|
|
confirmation_number = await self._submit_to_usac(
|
|
order_number=order_number,
|
|
entity=entity,
|
|
intake_data=intake_data,
|
|
revenue=revenue,
|
|
work_dir=work_dir,
|
|
)
|
|
except Exception as exc:
|
|
logger.error("Form499QHandler: USAC submission failed: %s", exc)
|
|
self._create_admin_todo(
|
|
order_number,
|
|
f"499-Q {quarter} AUTO-FILING FAILED for {legal_name}\n\n"
|
|
f"Error: {exc}\n\n"
|
|
f"File manually at {USAC_EFILE_URL}\n"
|
|
f"Filer ID: {filer_id}, FRN: {frn}\n"
|
|
f"Revenue total: ${revenue.get('total', 0):.2f}",
|
|
)
|
|
|
|
# ── Confirmation ───────────────────────────────────────────────
|
|
if confirmation_number:
|
|
logger.info("Form499QHandler: %s filed — confirmation %s", order_number, confirmation_number)
|
|
# Record filing
|
|
try:
|
|
import psycopg2
|
|
conn = psycopg2.connect(os.environ.get("DATABASE_URL", ""))
|
|
with conn.cursor() as cur:
|
|
cur.execute(
|
|
"""UPDATE compliance_orders
|
|
SET intake_data = intake_data || %s::jsonb,
|
|
updated_at = now()
|
|
WHERE order_number = %s""",
|
|
(json.dumps({"confirmation_number": confirmation_number, "filed_at": datetime.utcnow().isoformat()}), order_number),
|
|
)
|
|
conn.commit()
|
|
conn.close()
|
|
except Exception as exc:
|
|
logger.warning("Could not record confirmation: %s", exc)
|
|
|
|
self._send_confirmation_email(
|
|
to=entity.get("contact_email") or order_data.get("customer_email", ""),
|
|
entity_name=legal_name,
|
|
order_number=order_number,
|
|
quarter=quarter,
|
|
confirmation=confirmation_number,
|
|
)
|
|
else:
|
|
# No confirmation — send "received" email, admin will file manually
|
|
self._send_received_email(
|
|
to=entity.get("contact_email") or order_data.get("customer_email", ""),
|
|
entity_name=legal_name,
|
|
order_number=order_number,
|
|
quarter=quarter,
|
|
due_date=intake_data.get("due_date", ""),
|
|
)
|
|
|
|
return [] # confirmation PDF already uploaded to MinIO in _submit_to_usac
|
|
|
|
async def _submit_to_usac(
|
|
self, order_number: str, entity: dict, intake_data: dict,
|
|
revenue: dict, work_dir: str,
|
|
) -> str:
|
|
"""Submit 499-Q to USAC E-File via Playwright. Returns confirmation number."""
|
|
from scripts.workers.services.telecom.human_delay import human_delay
|
|
|
|
frn = intake_data.get("frn") or entity.get("frn", "")
|
|
filer_id = intake_data.get("filer_id_499") or entity.get("filer_id_499", "")
|
|
quarter = intake_data.get("quarter", "")
|
|
|
|
USAC_STORAGE_STATE = os.environ.get(
|
|
"USAC_STORAGE_STATE",
|
|
"/app/data/usac_session.json",
|
|
)
|
|
|
|
if not os.path.exists(USAC_STORAGE_STATE):
|
|
self._create_admin_todo(
|
|
order_number,
|
|
f"499-Q {quarter}: No USAC E-File session found. "
|
|
f"Log in at {USAC_EFILE_URL}, export session to {USAC_STORAGE_STATE}, "
|
|
f"then re-dispatch.",
|
|
)
|
|
return ""
|
|
|
|
try:
|
|
from playwright.async_api import async_playwright
|
|
except ImportError:
|
|
logger.warning("Playwright not available — creating admin todo")
|
|
self._create_admin_todo(
|
|
order_number,
|
|
f"499-Q {quarter}: Playwright not installed. File manually at {USAC_EFILE_URL}.",
|
|
)
|
|
return ""
|
|
|
|
confirmation_number = ""
|
|
|
|
async with async_playwright() as p:
|
|
browser = await p.chromium.launch(headless=True)
|
|
context = await browser.new_context(storage_state=USAC_STORAGE_STATE)
|
|
page = await context.new_page()
|
|
|
|
try:
|
|
await page.goto(USAC_EFILE_URL, timeout=30000)
|
|
await human_delay()
|
|
|
|
# Navigate to Form 499-Q
|
|
await page.click('text="Form 499-Q"')
|
|
await human_delay()
|
|
|
|
# Fill FRN / Filer ID
|
|
await page.fill('input[name="frn"]', frn)
|
|
await page.fill('input[name="filer_id"]', filer_id)
|
|
await human_delay()
|
|
|
|
# Revenue fields — the 499-Q has simplified revenue blocks
|
|
# Carrier's carrier revenue
|
|
cc_inter = revenue.get("carriers_carrier_interstate", 0)
|
|
cc_intra = revenue.get("carriers_carrier_intrastate", 0)
|
|
eu_inter = revenue.get("end_user_interstate", 0)
|
|
eu_intra = revenue.get("end_user_intrastate", 0)
|
|
|
|
# Fill revenue fields (exact selectors TBD from USAC form recon)
|
|
for selector, value in [
|
|
('input[name*="cc_interstate"], input[name*="carriers_carrier_inter"]', cc_inter),
|
|
('input[name*="cc_intrastate"], input[name*="carriers_carrier_intra"]', cc_intra),
|
|
('input[name*="eu_interstate"], input[name*="end_user_inter"]', eu_inter),
|
|
('input[name*="eu_intrastate"], input[name*="end_user_intra"]', eu_intra),
|
|
]:
|
|
try:
|
|
# Try each selector variant
|
|
for sel in selector.split(", "):
|
|
el = page.locator(sel)
|
|
if await el.count() > 0:
|
|
await el.fill(str(int(value * 100))) # cents
|
|
break
|
|
except Exception:
|
|
pass
|
|
await human_delay(0.3, 0.8)
|
|
|
|
# Submit
|
|
await human_delay(1.0, 2.0)
|
|
await page.click('button:has-text("Review")')
|
|
await page.wait_for_selector("text=Review", timeout=30000)
|
|
await page.click('button:has-text("Submit")')
|
|
await page.wait_for_selector("text=Confirmation", timeout=60000)
|
|
|
|
# Capture confirmation
|
|
body = await page.locator("body").inner_text()
|
|
for line in body.splitlines():
|
|
if "Confirmation" in line or "Filing ID" in line:
|
|
parts = line.split(":", 1)
|
|
if len(parts) == 2 and parts[1].strip():
|
|
confirmation_number = parts[1].strip()
|
|
break
|
|
|
|
# Save confirmation PDF
|
|
conf_path = os.path.join(work_dir, f"499q_{quarter}_confirmation.pdf")
|
|
await page.pdf(path=conf_path, format="Letter")
|
|
|
|
# Upload to MinIO
|
|
try:
|
|
from scripts.workers.minio_client import upload_file
|
|
upload_file(conf_path, f"compliance/{order_number}/499q_{quarter}_confirmation.pdf")
|
|
except Exception:
|
|
pass
|
|
|
|
except Exception as exc:
|
|
logger.error("USAC 499-Q Playwright error: %s", exc)
|
|
# Screenshot for debugging
|
|
try:
|
|
ss_path = os.path.join(work_dir, "usac_error.png")
|
|
await page.screenshot(path=ss_path, full_page=True)
|
|
except Exception:
|
|
pass
|
|
raise
|
|
finally:
|
|
await browser.close()
|
|
|
|
return confirmation_number
|
|
|
|
def _send_received_email(self, to: str, entity_name: str, order_number: str,
|
|
quarter: str, due_date: str) -> None:
|
|
if not to:
|
|
return
|
|
self._send_html_email(
|
|
to=to,
|
|
subject=f"499-Q {quarter} Data Received — {entity_name}",
|
|
html=f"""
|
|
<div style="font-family:Inter,sans-serif;max-width:600px;margin:0 auto;color:#1f2937">
|
|
<div style="background:#1e3a5f;padding:16px 24px;border-radius:8px 8px 0 0">
|
|
<h2 style="color:#fff;margin:0;font-size:16px">Form 499-Q {quarter} — Data Received</h2>
|
|
</div>
|
|
<div style="padding:24px;border:1px solid #e5e7eb;border-top:none;border-radius:0 0 8px 8px">
|
|
<p>Your quarterly revenue data for <strong>{entity_name}</strong>
|
|
({quarter}, due {due_date}) has been received.</p>
|
|
<p>We'll file this with USAC and send you a confirmation email
|
|
with your filing reference number once complete.</p>
|
|
<p style="font-size:13px;color:#6b7280;margin-top:1rem">
|
|
Order: {order_number}<br>
|
|
Questions? Contact <a href="mailto:ops@performancewest.net">ops@performancewest.net</a>.
|
|
</p>
|
|
</div>
|
|
</div>""",
|
|
)
|
|
|
|
def _send_confirmation_email(self, to: str, entity_name: str, order_number: str,
|
|
quarter: str, confirmation: str) -> None:
|
|
if not to:
|
|
return
|
|
self._send_html_email(
|
|
to=to,
|
|
subject=f"499-Q {quarter} Filed — Confirmation {confirmation} — {entity_name}",
|
|
html=f"""
|
|
<div style="font-family:Inter,sans-serif;max-width:600px;margin:0 auto;color:#1f2937">
|
|
<div style="background:#059669;padding:16px 24px;border-radius:8px 8px 0 0">
|
|
<h2 style="color:#fff;margin:0;font-size:16px">Form 499-Q {quarter} — Filed Successfully</h2>
|
|
</div>
|
|
<div style="padding:24px;border:1px solid #e5e7eb;border-top:none;border-radius:0 0 8px 8px">
|
|
<p>Your FCC Form 499-Q quarterly filing for <strong>{entity_name}</strong>
|
|
({quarter}) has been successfully submitted to USAC.</p>
|
|
<div style="background:#f0fdf4;border:1px solid #bbf7d0;border-radius:8px;padding:12px;margin:16px 0;text-align:center">
|
|
<p style="font-size:13px;color:#166534;margin:0">Confirmation Number</p>
|
|
<p style="font-size:20px;font-weight:700;color:#166534;margin:4px 0 0">{confirmation}</p>
|
|
</div>
|
|
<p>A confirmation PDF has been saved to your account. USAC will
|
|
calculate your quarterly USF contribution based on the revenue
|
|
data submitted.</p>
|
|
<p style="font-size:13px;color:#6b7280;margin-top:1rem">
|
|
Order: {order_number}<br>
|
|
Questions? Contact <a href="mailto:ops@performancewest.net">ops@performancewest.net</a>.
|
|
</p>
|
|
</div>
|
|
</div>""",
|
|
)
|
|
|
|
def _send_html_email(self, to: str, subject: str, html: str) -> None:
|
|
try:
|
|
import smtplib
|
|
from email.mime.multipart import MIMEMultipart
|
|
from email.mime.text import MIMEText
|
|
|
|
from scripts._email_plaintext import html_to_text
|
|
|
|
msg = MIMEMultipart("alternative")
|
|
msg["From"] = os.environ.get("SMTP_FROM", "Performance West <noreply@performancewest.net>")
|
|
msg["To"] = to
|
|
msg["Subject"] = subject
|
|
msg.attach(MIMEText(html_to_text(html), "plain"))
|
|
msg.attach(MIMEText(html, "html"))
|
|
|
|
with smtplib.SMTP(
|
|
os.environ.get("SMTP_HOST", "co.carrierone.com"),
|
|
int(os.environ.get("SMTP_PORT", "587")),
|
|
timeout=30,
|
|
) as s:
|
|
s.starttls()
|
|
s.login(os.environ.get("SMTP_USER", ""), os.environ.get("SMTP_PASS", ""))
|
|
s.send_message(msg)
|
|
except Exception as exc:
|
|
logger.warning("Email send failed: %s", exc)
|