f28b5bf39e
Full client-mapped plan_type pipeline: complete header alias set (bare 'plan'/'plan_name' stay payer aliases), value canonicalization map (Medicare Part B / FFS / MA / Part C / private / employer etc), unrecognized values grade Plan Type Needed with a hashed once-per-value warning. Once-per-batch deprecation fence when records lack plan_type (timing guesses from payer, legacy; readiness never does). CSVImport mapping review gains Plan Type. All 66 demo/drift fixture CSVs authored with plan_type columns (variant headers + variant values on the drift corpus); corpus sweep: 66/66 map, 100% typed. Echo paths canonicalize through the same value map (audit Low fixed). _normalize_payer and the default config entry remain untouched (Phase 2c, Kisa-gated). Independent audit: SHIP. 169 tests green. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
385 lines
14 KiB
Python
385 lines
14 KiB
Python
"""
|
||
CSV header normalization for Signal.
|
||
|
||
Maps messy supplier CSV exports to canonical ShipmentRecord fields.
|
||
Tolerates header drift, alternative column names, and common date formats.
|
||
"""
|
||
|
||
import csv
|
||
import hashlib
|
||
import io
|
||
import logging
|
||
import re
|
||
from datetime import date, datetime
|
||
from typing import Optional
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
import sys
|
||
from pathlib import Path
|
||
sys.path.insert(0, str(Path(__file__).parent.parent))
|
||
from core.coverage_calculator import ShipmentRecord
|
||
|
||
HEADER_MAP: dict[str, list[str]] = {
|
||
"patient_id": [
|
||
"patient_id", "patientid", "patient id", "pt_id", "pt id",
|
||
"mrn", "account_number", "account number", "account_no",
|
||
"patient_account", "acct_no", "acct no", "acct #", "acct#",
|
||
"id", "patient", "member_id", "member id",
|
||
"external patient ref", "external_patient_ref", "external ref",
|
||
],
|
||
"device_type": [
|
||
"device_type", "device type", "device", "devicetype",
|
||
"product_type", "product type", "product", "item",
|
||
"item_description", "item description", "hcpcs_description",
|
||
"hcpcs description", "description", "product_name",
|
||
"dme", "dme description", "dme_description", "dme desc",
|
||
"equipment", "equipment description",
|
||
],
|
||
"shipment_date": [
|
||
"shipment_date", "shipment date", "ship_date", "ship date",
|
||
"dispense_date", "dispense date", "dispensedate",
|
||
"service_date", "service date",
|
||
"order_date", "order date", "date_of_service", "dos",
|
||
"fill_date", "fill date", "last_ship_date", "last ship date",
|
||
],
|
||
"quantity": [
|
||
"quantity", "qty", "units", "count", "qty_dispensed",
|
||
"units_dispensed", "quantity_dispensed", "qty_shipped",
|
||
],
|
||
"payer": [
|
||
"payer", "insurance", "insurance_name", "insurance name",
|
||
"plan", "plan_name", "plan name", "payer_name", "payer name",
|
||
"primary_payer", "primary payer", "ins_name", "carrier",
|
||
],
|
||
# Client-mapped plan type — grading-critical, NEVER derived from the payer
|
||
# name. Bare "plan"/"plan_name" stay payer aliases (a "Plan" column in the
|
||
# wild is a payer name); stealing them would silently re-map existing CSVs.
|
||
"plan_type": [
|
||
"plan_type", "plan type", "plantype", "plan category", "plan_category",
|
||
"coverage type", "coverage_type", "insurance_type", "insurance type",
|
||
"payer_type", "payer type", "lob", "line_of_business",
|
||
"line of business", "benefit_type", "benefit type",
|
||
],
|
||
"component": [
|
||
"component", "item_type", "component_type", "type", "supply_type",
|
||
],
|
||
"csv_visit_date": [
|
||
"visit_date", "visit date", "qualifying_visit_date", "qualifying visit date",
|
||
"last_visit_date", "last visit date", "face_to_face_date", "f2f_date",
|
||
"encounter_date", "encounter date", "physician_visit_date",
|
||
],
|
||
"csv_swo_status": [
|
||
"swo_status", "swo status", "swo", "standing_written_order",
|
||
"standing written order", "order_status", "order status",
|
||
],
|
||
"csv_pecos_verified": [
|
||
"pecos_verified", "pecos verified", "pecos", "pecos_status",
|
||
"enrollment_verified", "enrollment verified",
|
||
],
|
||
"csv_pa_status": [
|
||
"pa_status", "pa status", "prior_auth_status", "prior auth status",
|
||
"prior_authorization_status", "auth_status", "pa", "authorization",
|
||
],
|
||
"csv_diagnosis_on_file": [
|
||
"diagnosis_on_file", "diagnosis on file", "diagnosis", "dx_on_file",
|
||
"dx on file", "icd_on_file", "icd on file",
|
||
],
|
||
"csv_transfer_from": [
|
||
"transfer_from", "transfer from", "previous_supplier", "previous supplier",
|
||
"prior_supplier", "prior supplier", "transfer_status", "transferred_from",
|
||
],
|
||
"order_number": [
|
||
"order_number", "order number", "order_no", "order no", "order#",
|
||
"claim_number", "claim number", "brightree_order", "order_id",
|
||
"rx_number", "rx number", "rx#", "dispense_number",
|
||
],
|
||
"hcpcs": [
|
||
"hcpcs", "hcpcs_code", "hcpcs code", "procedure_code", "procedure code",
|
||
"billing_code", "billing code", "item_code", "item code", "hcpc",
|
||
],
|
||
}
|
||
|
||
DEVICE_MAP: dict[str, str] = {
|
||
# Dexcom G7 variants
|
||
"dexcom g7": "dexcom_g7",
|
||
"dexcom_g7": "dexcom_g7",
|
||
"dexcomg7": "dexcom_g7",
|
||
"dexcom g-7": "dexcom_g7",
|
||
"g7": "dexcom_g7",
|
||
"g7 sensor": "dexcom_g7",
|
||
"dexcom g7 sensor": "dexcom_g7",
|
||
"dexcom g7 cgm": "dexcom_g7",
|
||
# Dexcom G6 variants
|
||
"dexcom g6": "dexcom_g6",
|
||
"dexcom_g6": "dexcom_g6",
|
||
"dexcomg6": "dexcom_g6",
|
||
"dexcom g-6": "dexcom_g6",
|
||
"g6": "dexcom_g6",
|
||
"g6 sensor": "dexcom_g6",
|
||
"dexcom g6 sensor": "dexcom_g6",
|
||
"dexcom g6 cgm": "dexcom_g6",
|
||
# FreeStyle Libre 2 variants
|
||
"freestyle libre 2": "freestyle_libre_2",
|
||
"freestyle_libre_2": "freestyle_libre_2",
|
||
"freestylelibre2": "freestyle_libre_2",
|
||
"libre 2": "freestyle_libre_2",
|
||
"libre2": "freestyle_libre_2",
|
||
"fsl2": "freestyle_libre_2",
|
||
"fs libre 2": "freestyle_libre_2",
|
||
"freestyle libre 2 sensor": "freestyle_libre_2",
|
||
# FreeStyle Libre 3 variants
|
||
"freestyle libre 3": "freestyle_libre_3",
|
||
"freestyle_libre_3": "freestyle_libre_3",
|
||
"freestylelibre3": "freestyle_libre_3",
|
||
"libre 3": "freestyle_libre_3",
|
||
"libre3": "freestyle_libre_3",
|
||
"fsl3": "freestyle_libre_3",
|
||
"fs libre 3": "freestyle_libre_3",
|
||
"freestyle libre 3 sensor": "freestyle_libre_3",
|
||
# FreeStyle Libre (no version — default to current)
|
||
"freestyle libre": "freestyle_libre_3",
|
||
"freestylelibre": "freestyle_libre_3",
|
||
"libre": "freestyle_libre_3",
|
||
"fsl": "freestyle_libre_3",
|
||
# Omnipod variants
|
||
"omnipod 5": "omnipod_5",
|
||
"omnipod_5": "omnipod_5",
|
||
"omnipod5": "omnipod_5",
|
||
"omnipod": "omnipod_5",
|
||
"op5": "omnipod_5",
|
||
# HCPCS codes used as device descriptions in some billing exports
|
||
"e2103": "dexcom_g7",
|
||
"a4239": "dexcom_g7",
|
||
"a4253": "freestyle_libre_3",
|
||
# Generic CGM descriptions
|
||
"cgm": "dexcom_g7",
|
||
"continuous glucose monitor": "dexcom_g7",
|
||
"continuous glucose monitoring": "dexcom_g7",
|
||
"glucose monitor": "dexcom_g7",
|
||
"glucose monitoring": "dexcom_g7",
|
||
"cgm sensor": "dexcom_g7",
|
||
"cgm supply": "dexcom_g7",
|
||
"cgm supplies": "dexcom_g7",
|
||
"dme cgm": "dexcom_g7",
|
||
}
|
||
|
||
DATE_FORMATS = [
|
||
"%Y-%m-%d",
|
||
"%m/%d/%Y",
|
||
"%m-%d-%Y",
|
||
"%d/%m/%Y",
|
||
"%m/%d/%y",
|
||
"%Y%m%d",
|
||
"%d-%b-%Y",
|
||
"%b %d, %Y",
|
||
"%B %d, %Y",
|
||
"%m/%d/%Y %H:%M:%S",
|
||
"%Y-%m-%dT%H:%M:%S",
|
||
]
|
||
|
||
|
||
# Canonicalization map for CLIENT-SUPPLIED plan-type VALUES (P5). The client
|
||
# asserted the plan type; Signal only canonicalizes spelling. Deriving plan
|
||
# type from the payer NAME remains forbidden (the never-guess lock).
|
||
PLAN_TYPE_VALUES = {
|
||
"medicare": "medicare", "medicare ffs": "medicare", "ffs": "medicare",
|
||
"medicare part b": "medicare", "part b": "medicare",
|
||
"traditional medicare": "medicare", "original medicare": "medicare",
|
||
"medicare advantage": "medicare_advantage", "ma": "medicare_advantage",
|
||
"medicare part c": "medicare_advantage", "part c": "medicare_advantage",
|
||
"medicare_advantage": "medicare_advantage",
|
||
"medicaid": "medicaid", "mcd": "medicaid", "state medicaid": "medicaid",
|
||
"commercial": "commercial", "private": "commercial",
|
||
"employer": "commercial", "group": "commercial",
|
||
}
|
||
|
||
|
||
def _normalize_plan_type(raw: "Optional[str]") -> Optional[str]:
|
||
"""Canonicalize a CLIENT-SUPPLIED plan type value. Never called on payer
|
||
names. Unrecognized (including blank) -> None -> Plan Type Needed.
|
||
Never guess."""
|
||
s = (raw or "").strip().lower()
|
||
return PLAN_TYPE_VALUES.get(s)
|
||
|
||
|
||
def _normalize_key(s: str) -> str:
|
||
return s.strip().lower().replace("-", " ").replace("_", " ")
|
||
|
||
|
||
def _map_header(raw: str) -> Optional[str]:
|
||
key = _normalize_key(raw)
|
||
for canonical, aliases in HEADER_MAP.items():
|
||
if key in [_normalize_key(a) for a in aliases]:
|
||
return canonical
|
||
return None
|
||
|
||
|
||
def _map_header_with_confidence(raw: str) -> tuple[Optional[str], str]:
|
||
"""Return (canonical_field, confidence) where confidence is 'high' or 'inferred'."""
|
||
key = _normalize_key(raw)
|
||
for canonical, aliases in HEADER_MAP.items():
|
||
if key == _normalize_key(canonical):
|
||
return canonical, "high"
|
||
if key in [_normalize_key(a) for a in aliases]:
|
||
return canonical, "inferred"
|
||
return None, "unmapped"
|
||
|
||
|
||
def _parse_date(value: str) -> Optional[date]:
|
||
value = value.strip()
|
||
for fmt in DATE_FORMATS:
|
||
try:
|
||
return datetime.strptime(value, fmt).date()
|
||
except ValueError:
|
||
continue
|
||
return None
|
||
|
||
|
||
def _normalize_device(value: str) -> Optional[str]:
|
||
if not value or not value.strip():
|
||
return None
|
||
key = _normalize_key(value)
|
||
key_compact = re.sub(r"\s+", "", key)
|
||
# Exact / alias match
|
||
for alias, canonical in DEVICE_MAP.items():
|
||
alias_compact = re.sub(r"\s+", "", alias)
|
||
if key == alias or key_compact == alias_compact:
|
||
return canonical
|
||
# Fuzzy fallback — detect CGM family by keyword so real-world naming
|
||
# variants from billing exports don't silently drop patients.
|
||
if "dexcom" in key:
|
||
return "dexcom_g7" if "g7" in key_compact else "dexcom_g6"
|
||
if "libre" in key or "freestyle" in key:
|
||
return "freestyle_libre_3" if "3" in key else "freestyle_libre_2"
|
||
if "omnipod" in key:
|
||
return "omnipod_5"
|
||
if any(k in key for k in ("cgm", "continuous glucose", "glucose monitor", "e2103", "a4239", "a4253")):
|
||
return "dexcom_g7"
|
||
return None
|
||
|
||
|
||
def normalize_csv(text: str) -> tuple[list[ShipmentRecord], list[str], dict]:
|
||
"""
|
||
Parse raw CSV text and return (records, skipped_reasons, mapping_summary).
|
||
Tolerates header drift and normalizes device/payer/date values.
|
||
|
||
mapping_summary format:
|
||
{
|
||
"mapped": {canonical_field: {"raw_header": str, "confidence": "high"|"inferred"}},
|
||
"unmapped_columns": [str],
|
||
"required_missing": [str],
|
||
}
|
||
"""
|
||
reader = csv.DictReader(io.StringIO(text.strip().lstrip("")))
|
||
if not reader.fieldnames:
|
||
return [], ["No headers found in file"], {}
|
||
|
||
column_map: dict[str, str] = {}
|
||
mapping_detail: dict[str, dict] = {}
|
||
unmapped_columns: list[str] = []
|
||
|
||
for raw_header in reader.fieldnames:
|
||
canonical, confidence = _map_header_with_confidence(raw_header)
|
||
if canonical:
|
||
column_map[raw_header] = canonical
|
||
mapping_detail[canonical] = {"raw_header": raw_header, "confidence": confidence}
|
||
else:
|
||
unmapped_columns.append(raw_header)
|
||
|
||
required_fields = {"patient_id", "device_type", "shipment_date"}
|
||
required_missing = [f for f in required_fields if f not in mapping_detail]
|
||
|
||
mapping_summary = {
|
||
"mapped": mapping_detail,
|
||
"unmapped_columns": unmapped_columns,
|
||
"required_missing": required_missing,
|
||
}
|
||
|
||
records: list[ShipmentRecord] = []
|
||
skipped: list[str] = []
|
||
_warned_plan_values: set = set() # one warning per distinct value per file
|
||
|
||
for i, row in enumerate(reader, start=2): # noqa: B007
|
||
mapped: dict[str, str] = {}
|
||
for raw_h, canonical in column_map.items():
|
||
mapped[canonical] = (row.get(raw_h) or "").strip()
|
||
|
||
patient_id = mapped.get("patient_id", "").strip()
|
||
if not patient_id:
|
||
skipped.append(f"Row {i}: missing patient_id")
|
||
continue
|
||
|
||
raw_device = mapped.get("device_type", "")
|
||
device_type = _normalize_device(raw_device)
|
||
if not device_type:
|
||
skipped.append(f"Row {i} ({patient_id}): unrecognized device '{raw_device}'")
|
||
continue
|
||
|
||
raw_date = mapped.get("shipment_date", "")
|
||
shipment_date = _parse_date(raw_date)
|
||
if not shipment_date:
|
||
skipped.append(f"Row {i} ({patient_id}): unparseable date '{raw_date}'")
|
||
continue
|
||
|
||
raw_qty = mapped.get("quantity", "1")
|
||
try:
|
||
quantity = max(1, int(float(raw_qty)))
|
||
except (ValueError, TypeError):
|
||
quantity = 1
|
||
|
||
# Raw payer string passes through untouched. coverage_calculator._normalize_payer
|
||
# is the single normalization point; the raw value survives for display.
|
||
payer = mapped.get("payer", "").strip()
|
||
component = (mapped.get("component", "sensor") or "sensor").lower().strip()
|
||
if component not in ("sensor", "transmitter", "pod"):
|
||
component = "sensor"
|
||
|
||
# Optional doc fields — only populated if the CSV contains these columns
|
||
csv_visit_date: Optional[date] = None
|
||
raw_visit = mapped.get("csv_visit_date", "")
|
||
if raw_visit:
|
||
csv_visit_date = _parse_date(raw_visit) # None if unparseable (non-error)
|
||
|
||
csv_swo_status = mapped.get("csv_swo_status") or None
|
||
csv_pecos_verified = mapped.get("csv_pecos_verified") or None
|
||
csv_pa_status = mapped.get("csv_pa_status") or None
|
||
csv_diagnosis_on_file = mapped.get("csv_diagnosis_on_file") or None
|
||
transfer_raw = mapped.get("csv_transfer_from", "").strip()
|
||
csv_transfer_from = transfer_raw if transfer_raw else None
|
||
order_number = mapped.get("order_number") or None
|
||
hcpcs = mapped.get("hcpcs") or None
|
||
|
||
# Client-supplied plan type, canonicalized through the value map
|
||
# (spelling only — the client asserted the plan type). Unrecognized
|
||
# values grade as "Plan Type Needed"; never guessed from payer name.
|
||
raw_plan = (mapped.get("plan_type") or "").strip()
|
||
plan_type = _normalize_plan_type(raw_plan)
|
||
if raw_plan and plan_type is None and raw_plan.lower() not in _warned_plan_values:
|
||
_warned_plan_values.add(raw_plan.lower())
|
||
logger.warning(
|
||
"Unrecognized plan_type value '%s' (first seen row %d, "
|
||
"patient hash %s) -> grades as Plan Type Needed",
|
||
raw_plan, i,
|
||
hashlib.sha256(patient_id.encode()).hexdigest()[:12],
|
||
)
|
||
|
||
records.append(ShipmentRecord(
|
||
patient_id=patient_id,
|
||
device_type=device_type,
|
||
shipment_date=shipment_date,
|
||
quantity=quantity,
|
||
payer=payer,
|
||
component=component,
|
||
csv_visit_date=csv_visit_date,
|
||
csv_swo_status=csv_swo_status,
|
||
csv_pecos_verified=csv_pecos_verified,
|
||
csv_pa_status=csv_pa_status,
|
||
csv_diagnosis_on_file=csv_diagnosis_on_file,
|
||
csv_transfer_from=csv_transfer_from,
|
||
order_number=order_number,
|
||
hcpcs=hcpcs,
|
||
plan_type=plan_type,
|
||
))
|
||
|
||
return records, skipped, mapping_summary
|