Signal/python-backend/api/main.py

"""Signal API — FastAPI backend for CSV ingestion, scoring, and export."""

import csv
import hashlib
import io
import os
import sys
from datetime import date, date as date_type
from functools import lru_cache
from pathlib import Path
from typing import Optional

import jwt
from jwt import PyJWKClient, ExpiredSignatureError, InvalidTokenError
from fastapi import Depends, FastAPI, File, Header, HTTPException, UploadFile
from fastapi.middleware.cors import CORSMiddleware
from fastapi.responses import StreamingResponse
from pydantic import BaseModel

# Ensure python-backend root is on path (works both locally and in Docker)
_backend_root = Path(__file__).parent.parent
if str(_backend_root) not in sys.path:
    sys.path.insert(0, str(_backend_root))

from core.coverage_calculator import ShipmentRecord, calculate_batch
from core.audit_logger import AuditAction, log_event
from core.doc_state_machine import compute_doc_state, DocState
from core.persistence import (
    persist_export,
    persist_upload,
    upsert_confirmed_visit,
    load_confirmed_visits_for_org,
    get_or_create_org,
    upsert_doc_status,
    load_doc_statuses_for_org,
)
from api.normalizer import _normalize_plan_type, normalize_csv
from core.overrides import (
    Override,
    OverrideKey,
    overrides_from_saved,
)
from core.worklist_readiness import (
    build_readiness_index,
    merged_to_record,
    readiness_for_record,
    readiness_stats,
    resolve_group,
)

app = FastAPI(title="Signal API", version="1.0.0", docs_url="/docs")

# CORS — set ALLOWED_ORIGINS_REGEX for a pattern (recommended for Vercel previews),
# or ALLOWED_ORIGINS as a comma-separated list for explicit control.
_origins_regex = os.getenv("ALLOWED_ORIGINS_REGEX", "")
_origins_env = os.getenv("ALLOWED_ORIGINS", "")
_allowed_origins: list[str] = (
    [o.strip() for o in _origins_env.split(",") if o.strip()]
    if _origins_env
    else [
        "http://localhost:5173",
        "http://localhost:5174",
        "http://127.0.0.1:5173",
    ]
)

_cors_kwargs: dict = {
    "allow_credentials": True,
    "allow_methods": ["*"],
    "allow_headers": ["*"],
}
if _origins_regex:
    _cors_kwargs["allow_origin_regex"] = _origins_regex
else:
    _cors_kwargs["allow_origins"] = _allowed_origins

app.add_middleware(CORSMiddleware, **_cors_kwargs)

# API key auth — enforced when SIGNAL_API_KEY env var is set.
# In dev (no env var), all requests pass. In production, X-API-Key header is required.
_api_key = os.getenv("SIGNAL_API_KEY", "")
_clerk_jwks_url = os.getenv("CLERK_JWKS_URL", "")


@lru_cache(maxsize=1)
def _get_jwks_client() -> PyJWKClient:
    if not _clerk_jwks_url:
        raise RuntimeError("CLERK_JWKS_URL is not set")
    return PyJWKClient(
        _clerk_jwks_url, cache_keys=True, cache_jwk_set=True, lifespan=300
    )


def verify_clerk_token(authorization: str) -> dict:
    """Verify a Clerk Bearer JWT. Returns decoded claims or raises HTTP 401."""
    if not authorization.startswith("Bearer "):
        raise HTTPException(
            status_code=401, detail="Missing or malformed Authorization header"
        )
    token = authorization.removeprefix("Bearer ").strip()
    try:
        client = _get_jwks_client()
        signing_key = client.get_signing_key_from_jwt(token)
        return jwt.decode(
            token,
            signing_key.key,
            algorithms=["RS256"],
            options={"verify_exp": True, "verify_nbf": True},
        )
    except ExpiredSignatureError:
        raise HTTPException(status_code=401, detail="Token has expired")
    except InvalidTokenError as exc:
        raise HTTPException(status_code=401, detail=f"Invalid token: {exc}")
    except RuntimeError:
        raise HTTPException(status_code=503, detail="Auth service not configured")
    except Exception:
        raise HTTPException(status_code=401, detail="Token verification failed")


def require_auth(
    authorization: str = Header(default=""),
    x_api_key: str = Header(default=""),
) -> dict:
    """
    Accept either a Clerk JWT (Authorization: Bearer) or a direct API key (X-Api-Key).
    Returns a claims dict. API-key path returns a synthetic service-account dict.
    """
    # Fast path: API key (for direct access / testing)
    if _api_key and x_api_key == _api_key:
        return {"sub": "service-account", "via": "api_key"}
    # Clerk JWT path (frontend / production)
    if authorization.startswith("Bearer "):
        return verify_clerk_token(authorization)
    # Dev mode: no keys set — allow all
    if not _api_key and not _clerk_jwks_url:
        return {"sub": "dev", "via": "open"}
    raise HTTPException(status_code=401, detail="Authentication required")


DEVICE_DISPLAY = {
    "dexcom_g7": "Dexcom G7",
    "dexcom_g6": "Dexcom G6",
    "freestyle_libre_2": "FreeStyle Libre 2",
    "freestyle_libre_3": "FreeStyle Libre 3",
    "omnipod_5": "Omnipod 5",
}

FLAG_LABELS = {
    "SUPPLY_LAPSED": "Docs Required",
    "VISIT_REQUIRED": "Escalate",
    "TRANSFER_PENDING": "New Patient",
    "RENEWAL_CRITICAL": "Escalate",
    "RENEWAL_ELEVATED": "Confirm Appointment",
    "RENEWAL_SOON": "Begin Outreach",
    "RESUPPLY_READY": "Clear to Ship",
    "ACTIVE": "On Track",
    "NO_RECENT_SHIPMENT": "No Recent Shipment",
}

FLAG_ACTIONS = {
    "SUPPLY_LAPSED": "Pull patient file — verify documentation is current before initiating any new order",
    "VISIT_REQUIRED": "Contact prescriber's office and patient directly — ask: 'Was there a visit in the past 6 months to evaluate this patient's CGM dosing needs?' No shipment until visit is confirmed and documented",
    "TRANSFER_PENDING": "Obtain full documentation set (SWO, PECOS, PA, diagnosis) — Signal cannot assess status until records are on file",
    "RENEWAL_CRITICAL": "Contact prescriber's office and patient directly — ask: 'Was there a visit in the past 6 months to evaluate CGM dosing needs?' Appointment must be on calendar within 45 days",
    "RENEWAL_ELEVATED": "Follow up with prescriber's office — ask: 'Was there a visit in the past 6 months to evaluate CGM dosing needs?' Confirm appointment is scheduled",
    "RENEWAL_SOON": "Contact prescriber's office — ask: 'Was there a visit in the past 6 months to evaluate this patient's CGM dosing needs?' Schedule evaluation now if not yet completed",
    "RESUPPLY_READY": "Initiate resupply order — patient is within the 30-day resupply window",
    "ACTIVE": "No action needed — documentation current, visit more than 90 days out",
    "NO_RECENT_SHIPMENT": "Verify status with prescriber office — confirm patient is still active",
}


class DocStateOut(BaseModel):
    swo: str
    visit: str
    pecos: str
    pa: str
    diagnosis: str


class ReadinessItemOut(BaseModel):
    doc_type: str
    required_state: str  # REQUIRED | NOT_REQUIRED | NOT_EVALUATED
    input_state: str  # SUPPLIED | ABSENT
    quality: str  # GOOD | PENDING | BAD | NONE
    value: Optional[str] = None
    satisfied: bool
    # Citation: the payer_rules.json rule that decided required_state.
    # None when NOT_EVALUATED (no rule fired, nothing to cite).
    rule_id: Optional[str] = None


class LineOut(BaseModel):
    """One coverage line (patient x device x plan type) in the nested payload."""

    device_type: str
    device_display: str
    plan_type: Optional[str] = None  # None = plan type needed (never guessed)
    gradeable: bool
    line_status: Optional[str] = None  # LineStatus verbatim; None = ungraded
    timing_flag: Optional[str] = None  # CoverageFlag of the current shipment
    doc_items: Optional[list[ReadinessItemOut]] = None
    # Display fields per the locked dedup design (section 6)
    order_numbers: list[str] = []
    hcpcs_codes: list[str] = []
    total_quantity: int = 0
    shipment_date: Optional[str] = None  # current shipment DOS
    payer: Optional[str] = None
    prior_shipment_count: int = 0


class PatientOut(BaseModel):
    """One patient in the nested payload; tabs count these, not rows (P6)."""

    patient_id: str
    rollup_status: Optional[str] = None  # None = no graded gradeable lines
    plan_type_needed: bool = False
    cgm_line_count: int = 0
    total_line_count: int = 0
    lines: list[LineOut] = []


class RecordOut(BaseModel):
    patient_id: str
    device_type: str
    device_display: str
    payer: str
    component: str
    days_until_coverage_end: int
    days_until_visit_due: Optional[int] = None
    flag: str
    priority_score: int
    coverage_end_date: str
    next_visit_due_date: Optional[str] = None
    action: str
    status_label: str
    reason: str
    rule_version: str
    visit_date_confidence: str = "estimated"
    is_transfer: bool = False
    doc_state: Optional[DocStateOut] = None
    cascade: list[str] = []
    order_number: Optional[str] = None
    hcpcs: Optional[str] = None
    recommended_next_step_code: str = "NO_ACTION_NEEDED"
    # Real values the frontend must echo back on /api/confirm-visit so the
    # recompute uses actual data instead of fabricated defaults.
    last_shipment_date: Optional[str] = None
    quantity: int = 1
    csv_visit_date: Optional[str] = None
    csv_swo_status: Optional[str] = None
    csv_pecos_verified: Optional[str] = None
    csv_pa_status: Optional[str] = None
    csv_diagnosis_on_file: Optional[str] = None
    csv_transfer_from: Optional[str] = None
    # Readiness verdict (Phase 2a coexistence — additive, ignored by the
    # current frontend). readiness_status is a LineStatus value verbatim.
    plan_type: Optional[str] = None
    readiness_status: Optional[str] = None
    readiness_items: Optional[list[ReadinessItemOut]] = None


class UploadResponse(BaseModel):
    records: list[RecordOut]
    total: int
    skipped: int
    skipped_reasons: list[str]
    stats: dict
    mapping_summary: dict
    batch_id: str | None = None
    readiness_stats: dict | None = None
    # Nested patient payload (P3, additive). The flat `records` list stays
    # untouched until P6 switches the UI (locked migration strategy).
    patients: list[PatientOut] | None = None
    patient_stats: dict | None = None


def _build_reason(
    flag_val: str, days_until_end: int, days_until_visit: Optional[int]
) -> str:
    if flag_val == "SUPPLY_LAPSED":
        ago = abs(days_until_end)
        unit = "day" if ago == 1 else "days"
        return f"Supply lapsed {ago} {unit} ago. Prescriber contact required before next shipment."
    if flag_val == "VISIT_REQUIRED":
        if days_until_visit is not None and days_until_visit <= 0:
            overdue = abs(days_until_visit)
            unit = "day" if overdue == 1 else "days"
            return f"Qualifying visit overdue by {overdue} {unit}. Confirm documentation immediately."
        return (
            "Qualifying visit renewal required. Confirm documentation before resupply."
        )
    if flag_val == "RENEWAL_CRITICAL":
        if days_until_visit is not None:
            unit = "day" if days_until_visit == 1 else "days"
            return f"Qualifying visit due in {days_until_visit} {unit}. Contact prescriber immediately."
        return "Qualifying visit renewal critical. Confirm visit documentation before resupply."
    if flag_val == "RENEWAL_ELEVATED":
        if days_until_visit is not None:
            unit = "day" if days_until_visit == 1 else "days"
            return f"Qualifying visit due in {days_until_visit} {unit}. Schedule prescriber appointment soon."
        return "Qualifying visit renewal elevated. Schedule prescriber appointment."
    if flag_val == "RENEWAL_SOON":
        if days_until_visit is not None:
            unit = "day" if days_until_visit == 1 else "days"
            return f"Qualifying visit due in {days_until_visit} {unit}. Monitor and plan ahead."
        return "Qualifying visit renewal approaching. Monitor status."
    if flag_val == "RESUPPLY_READY":
        unit = "day" if days_until_end == 1 else "days"
        return f"Coverage ends in {days_until_end} {unit}. Patient is within resupply window — initiate shipment now."
    if flag_val == "TRANSFER_PENDING":
        return "Patient transferred from prior supplier. Verify all documentation before proceeding."
    if flag_val == "NO_RECENT_SHIPMENT":
        return "No shipment on file in 12+ months. Contact the prescriber office to confirm the patient is still active and has not transferred care."
    unit = "day" if days_until_end == 1 else "days"
    return f"Coverage on track. Resupply window opens in approximately {days_until_end} {unit}."


def _normalize_payer_type(payer: str) -> str:
    """Map raw payer string to doc_state_machine payer_type."""
    from core.coverage_calculator import _normalize_payer

    normalized = _normalize_payer(payer)
    # Ensure medicare_advantage is not collapsed to medicare
    if "medicare advantage" in payer.lower() or "medicare_advantage" in payer.lower():
        return "medicare_advantage"
    return normalized


def _recommended_next_step_code(
    flag_val: str, doc: Optional[DocStateOut], cascade: Optional[list] = None
) -> str:
    if doc and doc.pa == "Denied":
        return "PA_APPEAL"
    if doc and doc.pa == "Required — Not Started":
        return "INITIATE_PA"
    if doc and "Not Confirmed" in (doc.visit or ""):
        return "AWAITING_VISIT_CONFIRMATION"
    if doc and doc.swo in ("Pending", "Expired"):
        return "REQUEST_SWO_UPDATE"
    if doc and doc.diagnosis == "Missing":
        return "OBTAIN_DIAGNOSIS"
    if doc and doc.pecos == "Not Verified":
        return "VERIFY_PRESCRIBER_PECOS"
    if doc and doc.pa == "Pending":
        return "PA_FOLLOW_UP"
    if flag_val in ("RESUPPLY_READY", "ACTIVE") and not cascade:
        return "READY_TO_BILL"
    return "NO_ACTION_NEEDED"


VALID_DOC_STATUSES = {
    "swo": {"pending", "requested", "on_file"},
    "pa": {"not_required", "requested", "approved", "denied"},
}

# Single source of truth for stored-status labels lives with the override
# engine (the same labels feed grading and display).
from core.overrides import STORED_STATUS_LABELS as DOC_STATUS_LABELS  # noqa: E402


def _recompute_patient(
    patient_id: str,
    echo_lines: list["EchoLine"],
    saved_doc_statuses: dict,
    confirmed_visit: Optional[date] = None,
):
    """Stateless patient recompute (P4): rebuild the patient's lines from the
    echoed CSV-truth inputs, apply ALL persisted staff overrides (the write
    that triggered this call is already persisted) plus the confirmed visit,
    re-grade through the same engine, and roll up through the same truth
    table. Returns (lines_out, rollup_status)."""
    from core.rollup import rollup_patient
    from core.worklist_readiness import build_readiness_index as _bri

    records = []
    for el in echo_lines:
        try:
            ship = (
                date_type.fromisoformat(el.shipment_date)
                if el.shipment_date
                else date.today()
            )
        except ValueError:
            ship = date.today()
        csv_visit = None
        if el.csv_visit_date:
            try:
                csv_visit = date_type.fromisoformat(el.csv_visit_date)
            except ValueError:
                csv_visit = None
        records.append(
            ShipmentRecord(
                patient_id=patient_id,
                device_type=el.device_type,
                shipment_date=ship,
                quantity=1,
                payer="",
                csv_swo_status=el.csv_swo_status,
                csv_visit_date=csv_visit,
                csv_pecos_verified=el.csv_pecos_verified,
                csv_pa_status=el.csv_pa_status,
                csv_diagnosis_on_file=el.csv_diagnosis_on_file,
                plan_type=_normalize_plan_type(el.plan_type),
            )
        )

    patient_hash = hashlib.sha256(patient_id.encode()).hexdigest()
    confirmed = (
        {patient_hash: confirmed_visit} if confirmed_visit else None
    )
    overrides = overrides_from_saved(
        {patient_hash: saved_doc_statuses.get(patient_hash, {})}
    )
    index = _bri(records, confirmed_visits=confirmed, overrides=overrides)

    lines_out = []
    for line_key, line in index.lines.items():
        verdict = index.by_line.get(line_key)
        lines_out.append(
            LineRecomputeOut(
                device_type=line.device_type,
                plan_type=None if line.plan_type == "unknown" else line.plan_type,
                line_status=line.line_status,
                doc_items=_readiness_items_out(verdict),
            )
        )
    rollup = rollup_patient(list(index.lines.values()))
    return lines_out, (
        rollup.rollup_status.value if rollup.rollup_status else None
    )


class EchoLine(BaseModel):
    """One coverage line echoed back for a stateless patient recompute.
    Carries CSV-truth grading inputs only; the server applies all persisted
    staff overrides and confirmed visits on top, then re-grades."""

    device_type: str
    gradeable: bool = True
    plan_type: Optional[str] = None
    shipment_date: Optional[str] = None  # ISO; defaults to today (doc axis
    # grading does not depend on it)
    csv_swo_status: Optional[str] = None
    csv_visit_date: Optional[str] = None
    csv_pecos_verified: Optional[str] = None
    csv_pa_status: Optional[str] = None
    csv_diagnosis_on_file: Optional[str] = None


class LineRecomputeOut(BaseModel):
    device_type: str
    plan_type: Optional[str] = None
    line_status: Optional[str] = None
    doc_items: Optional[list["ReadinessItemOut"]] = None


class DocStatusRequest(BaseModel):
    patient_id: str
    doc_type: str
    status: str
    # Device scope (P4). '' or omitted writes a legacy patient-scoped row
    # that applies to all devices until a device-scoped row is saved.
    device_type: str = ""
    status_date: Optional[str] = None
    expiry_date: Optional[str] = None
    # Optional echo of the patient's lines for the {line, rollup} recompute.
    lines: Optional[list[EchoLine]] = None


def _saved_for_device(
    doc_statuses: dict, patient_hash: str, device_type: str
) -> dict:
    """Resolve the device-scoped saved statuses for one record: legacy ''
    (patient-scoped) rows apply to all devices; a device-scoped row wins."""
    by_pat = doc_statuses.get(patient_hash, {})
    merged = dict(by_pat.get("", {}))
    merged.update(by_pat.get(device_type, {}))
    return merged


def _readiness_items_out(readiness) -> Optional[list[ReadinessItemOut]]:
    """Serialize a ReadinessVerdict's doc items (shared by rows and lines)."""
    if not readiness:
        return None
    return [
        ReadinessItemOut(
            doc_type=i.doc_type,
            required_state=i.required_state.value,
            input_state=i.input_state.value,
            quality=i.quality.value,
            value=i.value,
            satisfied=i.satisfied,
            rule_id=i.rule_id,
        )
        for i in readiness.doc_items
    ]


def _to_record_out(
    r,
    record=None,
    confirmed_visit_date=None,
    saved_doc_statuses: dict | None = None,
    readiness=None,
) -> RecordOut:
    flag_val = r.flag.value if hasattr(r.flag, "value") else str(r.flag)

    # Compute doc state if we have the original record
    doc_state_out = None
    cascade = []
    if record is not None:
        normalized_payer = _normalize_payer_type(r.payer)
        doc = compute_doc_state(
            payer_type=normalized_payer,
            csv_swo_status=record.csv_swo_status,
            csv_visit_date=record.csv_visit_date,
            confirmed_visit_date=confirmed_visit_date,
            csv_pecos_verified=record.csv_pecos_verified,
            csv_pa_status=record.csv_pa_status,
            csv_diagnosis_on_file=record.csv_diagnosis_on_file,
            is_transfer=bool(record.csv_transfer_from),
        )
        _saved = saved_doc_statuses or {}
        swo_display = DOC_STATUS_LABELS.get(
            _saved.get("swo", {}).get("status", ""), doc.swo
        )
        pa_display = DOC_STATUS_LABELS.get(
            _saved.get("pa", {}).get("status", ""), doc.pa
        )
        doc_state_out = DocStateOut(
            swo=swo_display,
            visit=doc.visit,
            pecos=doc.pecos,
            pa=pa_display,
            diagnosis=doc.diagnosis,
        )
        cascade = doc.cascade

    return RecordOut(
        patient_id=r.patient_id,
        device_type=r.device_type,
        device_display=DEVICE_DISPLAY.get(r.device_type, r.device_type),
        payer=r.payer,
        component=r.component,
        days_until_coverage_end=r.days_until_coverage_end,
        days_until_visit_due=r.days_until_visit_due,
        flag=flag_val,
        priority_score=r.priority_score,
        coverage_end_date=r.coverage_end_date.isoformat(),
        next_visit_due_date=r.next_visit_due_date.isoformat()
        if r.next_visit_due_date
        else None,
        action=FLAG_ACTIONS.get(flag_val, "Review"),
        status_label=FLAG_LABELS.get(flag_val, flag_val),
        reason=_build_reason(
            flag_val, r.days_until_coverage_end, r.days_until_visit_due
        ),
        rule_version=r.rule_version,
        visit_date_confidence=getattr(r, "visit_date_confidence", "estimated"),
        is_transfer=getattr(r, "is_transfer", False),
        doc_state=doc_state_out,
        cascade=cascade,
        order_number=getattr(record, "order_number", None) if record else None,
        hcpcs=getattr(record, "hcpcs", None) if record else None,
        recommended_next_step_code=_recommended_next_step_code(
            flag_val, doc_state_out, cascade
        ),
        last_shipment_date=r.last_shipment_date.isoformat()
        if r.last_shipment_date
        else None,
        quantity=getattr(record, "quantity", 1) if record else 1,
        csv_visit_date=record.csv_visit_date.isoformat()
        if record and record.csv_visit_date
        else None,
        csv_swo_status=getattr(record, "csv_swo_status", None) if record else None,
        csv_pecos_verified=getattr(record, "csv_pecos_verified", None)
        if record
        else None,
        csv_pa_status=getattr(record, "csv_pa_status", None) if record else None,
        csv_diagnosis_on_file=getattr(record, "csv_diagnosis_on_file", None)
        if record
        else None,
        csv_transfer_from=getattr(record, "csv_transfer_from", None)
        if record
        else None,
        plan_type=getattr(record, "plan_type", None) if record else None,
        readiness_status=readiness.line_status.value if readiness else None,
        readiness_items=_readiness_items_out(readiness),
    )


def _compute_stats(records: list[RecordOut]) -> dict:
    flags = [r.flag for r in records]
    return {
        "total": len(records),
        "supply_lapsed": flags.count("SUPPLY_LAPSED"),
        "visit_required": flags.count("VISIT_REQUIRED"),
        "transfer_pending": flags.count("TRANSFER_PENDING"),
        "renewal_critical": flags.count("RENEWAL_CRITICAL"),
        "renewal_elevated": flags.count("RENEWAL_ELEVATED"),
        "renewal_soon": flags.count("RENEWAL_SOON"),
        "resupply_ready": flags.count("RESUPPLY_READY"),
        "active": flags.count("ACTIVE"),
        "no_recent_shipment": flags.count("NO_RECENT_SHIPMENT"),
        "prescriber_action": (
            flags.count("SUPPLY_LAPSED")
            + flags.count("VISIT_REQUIRED")
            + flags.count("RENEWAL_CRITICAL")
            + flags.count("RENEWAL_ELEVATED")
        ),
    }


def _build_patients_payload(index, reachable_line_keys, results):
    """Assemble the nested patients payload (P3) from the lines reachable in
    THIS response, so patients/lines always reconcile with the flat records.
    Rollup rules are locked in core/rollup.py (worst gradeable line; PTN caps
    at Action Needed; non-gradeable lines excluded from the light)."""
    from core.dedup import UNKNOWN_PLAN_TYPE
    from core.rollup import rollup_patients

    reachable = {
        k: index.lines[k] for k in reachable_line_keys if k in index.lines
    }
    if not reachable:
        return [], {"patients_total": 0}

    from core.dedup import _as_date

    # Line-level timing flag = the worst-urgency row of the line's CURRENT
    # shipment group (twin rows can differ; worst wins, fail-safe).
    flag_by_group: dict = {}
    for r in results:
        gk = (r.patient_id, r.device_type, _as_date(r.last_shipment_date))
        flag_val = r.flag.value if hasattr(r.flag, "value") else str(r.flag)
        prev = flag_by_group.get(gk)
        if prev is None or r.priority_score > prev[1]:
            flag_by_group[gk] = (flag_val, r.priority_score)

    # DISPLAY ordering, intentionally different from rollup.SEVERITY: the
    # worklist queues At Risk above Plan Type Needed (urgency), while the
    # rollup's capping rule treats PTN as the worst LINE state. Do not unify.
    severity = {
        "At Risk": 4,
        "Plan Type Needed": 3,
        "Action Needed": 2,
        "On Track": 1,
        "Clear to Ship": 0,
    }
    patients = []
    for pid, ru in rollup_patients(reachable).items():
        lines_out = []
        for line in sorted(
            ru.lines,
            key=lambda l: (
                -(severity.get(l.line_status, -1)),
                l.device_type,
            ),
        ):
            line_key = (line.patient_id, line.device_type, line.plan_type)
            verdict = index.by_line.get(line_key)
            cur = line.current_shipment
            timing = flag_by_group.get(
                (line.patient_id, line.device_type, _as_date(cur.shipment_date))
            )
            lines_out.append(
                LineOut(
                    device_type=line.device_type,
                    device_display=DEVICE_DISPLAY.get(
                        line.device_type, line.device_type
                    ),
                    plan_type=None
                    if line.plan_type == UNKNOWN_PLAN_TYPE
                    else line.plan_type,
                    gradeable=line.gradeable,
                    line_status=line.line_status,
                    timing_flag=timing[0] if timing else None,
                    doc_items=_readiness_items_out(verdict),
                    order_numbers=cur.order_numbers,
                    hcpcs_codes=cur.hcpcs_codes,
                    total_quantity=cur.total_quantity,
                    shipment_date=cur.shipment_date.isoformat(),
                    payer=cur.payer or None,
                    prior_shipment_count=len(line.prior_shipments),
                )
            )
        patients.append(
            PatientOut(
                patient_id=pid,
                rollup_status=ru.rollup_status.value
                if ru.rollup_status
                else None,
                plan_type_needed=ru.plan_type_needed,
                cgm_line_count=ru.cgm_line_count,
                total_line_count=ru.total_line_count,
                lines=lines_out,
            )
        )

    # Worklist order: worst patients first, stable by patient_id.
    patients.sort(
        key=lambda p: (-(severity.get(p.rollup_status, -1)), p.patient_id)
    )

    stats: dict = {"patients_total": len(patients)}
    for p in patients:
        key = p.rollup_status if p.rollup_status else "ungraded"
        stats[key] = stats.get(key, 0) + 1
    return patients, stats


@app.get("/health")
def health():
    return {"status": "ok", "service": "signal-api", "version": "1.0.0"}


@app.get("/health/db")
def health_db():
    from core.supabase_client import get_client
    import os

    client = get_client()
    if not client:
        return {
            "status": "unavailable",
            "supabase_url_set": bool(os.getenv("SUPABASE_URL")),
            "service_key_set": bool(os.getenv("SUPABASE_SERVICE_KEY")),
        }
    try:
        result = client.table("organizations").select("id").limit(1).execute()
        return {"status": "ok", "org_count": len(result.data)}
    except Exception as e:
        return {"status": "error", "detail": str(e)}


@app.post("/api/upload", response_model=UploadResponse)
async def upload_csv(
    file: UploadFile = File(...),
    claims: dict = Depends(require_auth),
):
    if not (file.filename or "").endswith(".csv"):
        raise HTTPException(status_code=400, detail="File must be a .csv")

    content = await file.read()
    try:
        text = content.decode("utf-8")
    except UnicodeDecodeError:
        text = content.decode("latin-1")

    records, skipped_reasons, mapping_summary = normalize_csv(text)

    if not records:
        from core.supabase_client import get_client

        db_conn = get_client()
        log_event(
            action=AuditAction.CSV_INGEST,
            resource_id=file.filename or "unknown",
            user_id=claims.get("sub", "demo_user"),
            outcome="failure",
            ip_address="0.0.0.0",
            detail="No processable rows",
            db_conn=db_conn,
        )
        raise HTTPException(
            status_code=422,
            detail={
                "message": "No processable rows found in the uploaded file.",
                "skipped": skipped_reasons[:10],
                "mapping_summary": mapping_summary,
            },
        )

    # Load confirmed visit dates from Supabase for this org
    clerk_org_id = (
        claims.get("o", {}).get("id") if isinstance(claims.get("o"), dict) else None
    )
    org_id = get_or_create_org(clerk_org_id=clerk_org_id)
    confirmed_visits = load_confirmed_visits_for_org(org_id) if org_id else {}
    doc_statuses = load_doc_statuses_for_org(org_id) if org_id else {}
    from core.supabase_client import get_client

    db_conn = get_client()

    # P5 deprecation fence: when any record lacks a client-mapped plan_type,
    # the TIMING rules fall back to the payer-name guess for those records
    # (readiness never does — it reads Plan Type Needed). Once per batch.
    _unmapped = sum(1 for rec in records if rec.plan_type is None)
    if _unmapped:
        import logging

        logging.getLogger(__name__).warning(
            "plan_type not provided for %d/%d records. Timing rules guess "
            "from the payer name (legacy path, to be removed); readiness "
            "grades these records Plan Type Needed. Supply plan_type in "
            "the CSV import.",
            _unmapped,
            len(records),
        )

    results = calculate_batch(
        records, as_of=date.today(), confirmed_visits=confirmed_visits
    )

    # Build a lookup from a tuple key to original record for doc state computation.
    record_lookup = {}
    for r in records:
        key = (r.patient_id, r.shipment_date, r.device_type, r.order_number)
        record_lookup[key] = r

    # Readiness index (Phase 2a: additive, augments the legacy scoring —
    # replaces nothing). Row-to-line membership comes from the dedup output
    # itself, never re-derived from a row's own plan_type. Staff overrides
    # (P4) feed the grading inputs, device-scoped.
    index = build_readiness_index(
        records, confirmed_visits, overrides=overrides_from_saved(doc_statuses)
    )

    out = []
    used_lines = {}
    reachable_line_keys = set()
    for r in results:
        verdict, line_key, merged = resolve_group(
            index, r.patient_id, r.device_type, r.last_shipment_date
        )
        if line_key is not None:
            reachable_line_keys.add(line_key)
        if verdict is not None and line_key is not None:
            used_lines[line_key] = verdict
        # Exact row match first (byte-identical legacy path). CoverageResult
        # carries no order_number, so the exact lookup misses for CSVs with an
        # order-number column; the dedup merge then supplies the record fields
        # (latest-non-null policy — the same values the verdict graded from).
        rec = record_lookup.get(
            (
                r.patient_id,
                r.last_shipment_date,
                r.device_type,
                getattr(r, "order_number", None),
            )
        )
        if rec is None and merged is not None:
            rec = merged_to_record(merged)
        out.append(
            _to_record_out(
                r,
                record=rec,
                confirmed_visit_date=confirmed_visits.get(
                    hashlib.sha256(r.patient_id.encode()).hexdigest()
                ),
                saved_doc_statuses=_saved_for_device(
                    doc_statuses,
                    hashlib.sha256(r.patient_id.encode()).hexdigest(),
                    r.device_type,
                ),
                readiness=verdict,
            )
        )

    # Failed-data honesty (Kisa 2026-07-07): a row the scoring engine drops
    # (e.g. a component with no coverage rule) must appear in the skipped
    # list, never vanish — and BEFORE persistence so the stored batch record
    # matches the count the supplier saw. Signal never acts on rows whose
    # required information was not provided; it also never hides them.
    from core.dedup import _as_date as _asd

    processed_groups = {
        (r.patient_id, r.device_type, _asd(r.last_shipment_date))
        for r in results
    }
    _reported_drops = set()
    for rec in records:
        g = (rec.patient_id, rec.device_type, _asd(rec.shipment_date))
        if g not in processed_groups and g not in _reported_drops:
            _reported_drops.add(g)
            skipped_reasons.append(
                f"{rec.patient_id}: {rec.device_type} ({rec.component}) has "
                "no coverage rule for this component — row not scored"
            )

    # Use real user ID and db_conn where possible if we implement it, hardcoding
    # IP as 0.0.0.0 for now unless req object available
    from core.supabase_client import get_client

    db_conn = get_client()
    log_event(
        action=AuditAction.CSV_INGEST,
        resource_id=file.filename or "unknown",
        user_id=claims.get("sub", "demo_user"),
        outcome="success",
        ip_address="0.0.0.0",
        detail=f"{len(out)} records scored",
        db_conn=db_conn,
    )

    batch_id = persist_upload(
        filename=file.filename or "unknown",
        content_bytes=content,
        shipment_records=records,
        coverage_results=results,
        skipped_count=len(skipped_reasons),
        mapping_summary=mapping_summary,
        clerk_org_id=clerk_org_id,
    )

    patients_out, patient_stats = _build_patients_payload(
        index, reachable_line_keys, results
    )

    return UploadResponse(
        records=out,
        total=len(out),
        skipped=len(skipped_reasons),
        skipped_reasons=skipped_reasons[:20],
        stats=_compute_stats(out),
        mapping_summary=mapping_summary,
        batch_id=batch_id,
        # Only lines reachable from rows in THIS response, so the stats always
        # reconcile with the records beside them (a record calculate_batch
        # skipped cannot leave a phantom line in the counts).
        readiness_stats=readiness_stats(used_lines),
        patients=patients_out,
        patient_stats=patient_stats,
    )


class LineExportRow(BaseModel):
    """One device coverage line for the per-line work-queue export
    (granularity locked by Kisa 2026-07-07)."""

    patient_id: str
    device_display: str = ""
    payer: str = ""
    plan_type: Optional[str] = None
    rollup_status: Optional[str] = None
    line_status: Optional[str] = None
    timing: str = ""
    order_numbers: str = ""
    hcpcs_codes: str = ""
    total_quantity: int = 0
    shipment_date: str = ""
    swo: str = ""
    visit: str = ""
    pecos: str = ""
    pa: str = ""
    diagnosis: str = ""


class ExportRequest(BaseModel):
    records: list[RecordOut] = []
    batch_id: str | None = None
    # Per-line export (P6): when present, the CSV is one row per device
    # coverage line instead of one row per legacy record.
    lines: Optional[list[LineExportRow]] = None


@app.post("/api/export")
async def export_work_queue(
    body: ExportRequest,
    claims: dict = Depends(require_auth),
):
    """Generate a downloadable work-queue CSV from a list of scored records,
    or one row per device coverage line when `lines` is provided (P6)."""
    records = body.records
    output = io.StringIO()
    writer = csv.writer(output)

    if body.lines is not None:
        writer.writerow(
            [
                "Patient ID",
                "Device",
                "Payer",
                "Plan Type",
                "Patient Status",
                "Line Status",
                "Timing",
                "Order Numbers",
                "HCPCS",
                "Quantity",
                "Last Shipment",
                "SWO",
                "Visit",
                "PECOS",
                "PA",
                "Diagnosis",
            ]
        )
        for l in body.lines:
            writer.writerow(
                [
                    l.patient_id,
                    l.device_display,
                    l.payer,
                    l.plan_type or "Plan type needed",
                    l.rollup_status or "",
                    l.line_status or "",
                    l.timing,
                    l.order_numbers,
                    l.hcpcs_codes,
                    l.total_quantity,
                    l.shipment_date,
                    l.swo,
                    l.visit,
                    l.pecos,
                    l.pa,
                    l.diagnosis,
                ]
            )
        output.seek(0)
        today = date.today().isoformat()
        export_filename = f"signal-work-queue-{today}.csv"
        from core.supabase_client import get_client

        log_event(
            action=AuditAction.WORKLIST_EXPORT,
            resource_id=export_filename,
            user_id=claims.get("sub", "demo_user"),
            outcome="success",
            ip_address="0.0.0.0",
            detail=f"{len(body.lines)} coverage lines exported",
            db_conn=get_client(),
        )
        persist_export(
            batch_id=body.batch_id,
            filename=export_filename,
            row_count=len(body.lines),
        )
        return StreamingResponse(
            io.BytesIO(output.getvalue().encode("utf-8")),
            media_type="text/csv",
            headers={
                "Content-Disposition": f"attachment; filename={export_filename}"
            },
        )

    writer.writerow(
        [
            "Order Number",
            "HCPCS",
            "Patient ID",
            "Device",
            "Payer",
            "Status",
            "Recommended Next Step",
            "Priority Score",
            "Days Until Resupply End",
            "Next Visit Due",
            "Recommended Action",
            "Resupply End Date",
            "Reason",
            "Visit Confidence",
            "SWO Status",
            "Visit Status",
            "PA Status",
            "Diagnosis Status",
            "PECOS Status",
        ]
    )
    for r in records:
        doc = r.doc_state
        writer.writerow(
            [
                r.order_number or "",
                r.hcpcs or "",
                r.patient_id,
                r.device_display,
                r.payer,
                r.status_label,
                r.recommended_next_step_code,
                r.priority_score,
                r.days_until_coverage_end,
                r.next_visit_due_date or "",
                r.action,
                r.coverage_end_date,
                r.reason,
                r.visit_date_confidence,
                doc.swo if doc else "",
                doc.visit if doc else "",
                doc.pa if doc else "",
                doc.diagnosis if doc else "",
                doc.pecos if doc else "",
            ]
        )

    output.seek(0)
    today = date.today().isoformat()
    export_filename = f"signal-work-queue-{today}.csv"
    from core.supabase_client import get_client

    db_conn = get_client()
    log_event(
        action=AuditAction.WORKLIST_EXPORT,
        resource_id=export_filename,
        user_id=claims.get("sub", "demo_user"),
        outcome="success",
        ip_address="0.0.0.0",
        detail=f"{len(records)} records exported",
        db_conn=db_conn,
    )
    persist_export(
        batch_id=body.batch_id, filename=export_filename, row_count=len(records)
    )
    return StreamingResponse(
        io.BytesIO(output.getvalue().encode("utf-8")),
        media_type="text/csv",
        headers={
            "Content-Disposition": f"attachment; filename=signal-work-queue-{today}.csv"
        },
    )


class ConfirmVisitRequest(BaseModel):
    patient_id: str
    confirmed_date: str  # ISO date string YYYY-MM-DD
    # Fields needed to recompute coverage after confirmation
    shipment_date: str
    payer: str
    device_type: str
    quantity: int = 1
    component: str = "sensor"
    # Doc status fields echoed from the original upload so the recompute
    # preserves checklist state instead of regressing it to defaults
    csv_visit_date: str | None = None
    csv_swo_status: str | None = None
    csv_pecos_verified: str | None = None
    csv_pa_status: str | None = None
    csv_diagnosis_on_file: str | None = None
    csv_transfer_from: str | None = None
    order_number: str | None = None
    hcpcs: str | None = None
    # Client-mapped plan type echoed from the upload — never guessed. None
    # keeps the recompute honest: the verdict reads Plan Type Needed.
    plan_type: str | None = None
    # Optional echo of ALL the patient's lines: the confirmed visit fans out
    # to every line (the visit is the person), so the P4 recompute re-grades
    # each and returns the re-rolled patient alongside the legacy record.
    lines: Optional[list[EchoLine]] = None


@app.post("/api/confirm-visit")
async def confirm_visit(
    body: ConfirmVisitRequest,
    claims: dict = Depends(require_auth),
):
    """
    Store a staff-confirmed qualifying visit date and return the updated coverage record.
    The confirmed date persists across all future CSV imports for this patient_id.
    """
    from core.coverage_calculator import calculate_coverage

    # Validate dates
    try:
        confirmed = date_type.fromisoformat(body.confirmed_date)
        shipment = date_type.fromisoformat(body.shipment_date)
    except ValueError as e:
        raise HTTPException(
            status_code=400, detail=f"Invalid date: {e}. Use YYYY-MM-DD."
        ) from e

    if confirmed > date_type.today():
        raise HTTPException(
            status_code=400, detail="Confirmed date cannot be in the future."
        )

    # Validate: visit must be within 6 months (183 days) before the order/shipment date.
    # 183 days = ~6 months, consistent with CMS CGM qualifying visit window.
    from datetime import timedelta as _td

    six_months_before = shipment - _td(days=183)
    if confirmed < six_months_before:
        raise HTTPException(
            status_code=400,
            detail=(
                f"Visit date {confirmed.isoformat()} is more than 6 months before "
                f"the order date {shipment.isoformat()}. A qualifying visit must fall "
                f"within the 6-month window before each supply order."
            ),
        )

    # Get org
    clerk_org_id = (
        claims.get("o", {}).get("id") if isinstance(claims.get("o"), dict) else None
    )
    user_id = claims.get("sub", "demo_user")
    org_id = get_or_create_org(clerk_org_id=clerk_org_id)
    if not org_id:
        raise HTTPException(status_code=503, detail="Organization not found.")

    # Hash and store
    patient_hash = hashlib.sha256(body.patient_id.encode()).hexdigest()
    success = upsert_confirmed_visit(
        org_id, patient_hash, confirmed, confirmed_by="staff"
    )
    if not success:
        raise HTTPException(status_code=503, detail="Failed to save confirmed visit.")

    # Recompute coverage with confirmed visit date, preserving the doc status
    # fields echoed from the original upload
    csv_visit = None
    if body.csv_visit_date:
        try:
            csv_visit = date_type.fromisoformat(body.csv_visit_date)
        except ValueError:
            csv_visit = None
    record = ShipmentRecord(
        patient_id=body.patient_id,
        device_type=body.device_type,
        shipment_date=shipment,
        quantity=body.quantity,
        payer=body.payer,
        component=body.component,
        csv_visit_date=csv_visit,
        csv_swo_status=body.csv_swo_status,
        csv_pecos_verified=body.csv_pecos_verified,
        csv_pa_status=body.csv_pa_status,
        csv_diagnosis_on_file=body.csv_diagnosis_on_file,
        csv_transfer_from=body.csv_transfer_from,
        order_number=body.order_number,
        hcpcs=body.hcpcs,
        # Same canonicalization the CSV path applies (value map, never
        # guessed) so RecordOut.plan_type is consistent across both paths.
        plan_type=_normalize_plan_type(body.plan_type),
    )
    result = calculate_coverage(record, confirmed_visit_date=confirmed)
    verdict = readiness_for_record(record, confirmed_visit_date=confirmed)

    from core.supabase_client import get_client

    db_conn = get_client()
    log_event(
        action=AuditAction.CSV_INGEST,
        resource_id=f"confirm-visit:{body.patient_id[:8]}",
        user_id=user_id,
        outcome="success",
        ip_address="0.0.0.0",  # IP address pass-through requires Request object, leaving 0.0.0.0 for now unless requested
        detail=f"Visit confirmed {confirmed.isoformat()}",
        db_conn=db_conn,
    )

    record_out = _to_record_out(
        result, record=record, confirmed_visit_date=confirmed, readiness=verdict
    )
    if not body.lines:
        return record_out

    # P4: re-grade the patient's echoed lines with the new confirmed date
    # (visit fans out to all lines) and return the re-rolled patient. All
    # legacy RecordOut fields stay present.
    saved = load_doc_statuses_for_org(org_id)
    lines_out, rollup_status = _recompute_patient(
        body.patient_id, body.lines, saved, confirmed_visit=confirmed
    )
    payload = record_out.model_dump()
    payload["lines"] = [l.model_dump() for l in lines_out]
    payload["rollup_status"] = rollup_status
    return payload


@app.put("/api/doc-status")
async def update_doc_status(
    body: DocStatusRequest,
    claims: dict = Depends(require_auth),
):
    """Store a staff-updated SWO or PA status. Persists across future CSV imports."""
    if body.doc_type not in VALID_DOC_STATUSES:
        raise HTTPException(
            status_code=400,
            detail=f"Invalid doc_type '{body.doc_type}'. Must be 'swo' or 'pa'.",
        )
    if body.status not in VALID_DOC_STATUSES[body.doc_type]:
        valid = ", ".join(sorted(VALID_DOC_STATUSES[body.doc_type]))
        raise HTTPException(
            status_code=400,
            detail=f"Invalid status '{body.status}' for {body.doc_type}. Valid: {valid}.",
        )

    from datetime import date as date_type2

    status_date = None
    expiry_date = None
    if body.status_date:
        try:
            status_date = date_type2.fromisoformat(body.status_date)
        except ValueError as err:
            raise HTTPException(
                status_code=400,
                detail=f"Invalid status_date: {body.status_date}. Use YYYY-MM-DD.",
            ) from err
    if body.expiry_date:
        try:
            expiry_date = date_type2.fromisoformat(body.expiry_date)
        except ValueError as err:
            raise HTTPException(
                status_code=400,
                detail=f"Invalid expiry_date: {body.expiry_date}. Use YYYY-MM-DD.",
            ) from err

    clerk_org_id = (
        claims.get("o", {}).get("id") if isinstance(claims.get("o"), dict) else None
    )
    user_id = claims.get("sub", "demo_user")
    org_id = get_or_create_org(clerk_org_id=clerk_org_id)
    if not org_id:
        raise HTTPException(status_code=503, detail="Organization not found.")

    patient_hash = hashlib.sha256(body.patient_id.encode()).hexdigest()
    success = upsert_doc_status(
        org_id,
        patient_hash,
        body.doc_type,
        body.status,
        status_date,
        expiry_date,
        device_type=body.device_type or "",
    )
    if not success:
        raise HTTPException(status_code=503, detail="Failed to save doc status.")

    from core.supabase_client import get_client

    db_conn = get_client()
    log_event(
        action=AuditAction.CSV_INGEST,
        resource_id=f"update-doc-status:{body.patient_id[:8]}",
        user_id=user_id,
        outcome="success",
        ip_address="0.0.0.0",
        detail=f"Updated doc status {body.doc_type} to {body.status}",
        db_conn=db_conn,
    )

    response = {
        "patient_id": body.patient_id,
        "doc_type": body.doc_type,
        "status": body.status,
        "device_type": body.device_type or "",
        "display": DOC_STATUS_LABELS.get(body.status, body.status),
    }

    # P4 recompute contract: when the client echoes the patient's lines,
    # re-grade them (the write above is already persisted, so a fresh load
    # picks it up) and return {lines, rollup} from the same engine.
    if body.lines:
        saved = load_doc_statuses_for_org(org_id)
        confirmed = load_confirmed_visits_for_org(org_id).get(patient_hash)
        lines_out, rollup_status = _recompute_patient(
            body.patient_id, body.lines, saved, confirmed_visit=confirmed
        )
        response["lines"] = [l.model_dump() for l in lines_out]
        response["rollup_status"] = rollup_status

    return response