"""Parse and clean the raw PAX CSV. Verifies the claims in mission/data.md: 192,464 rows, 12 device IDs, 8 usable logical locations, three excluded test devices, and the density-01 merge. The raw file is never written to git; a cached parquet copy lives in data/cache/. """ from __future__ import annotations import json import logging from pathlib import Path import pandas as pd from . import config log = logging.getLogger(__name__) EXPECTED_CHECKS = { "raw_rows": 192_464, "density_rows": 96_232, "battery_rows": 96_232, "device_ids": 12, "sensor_names": 11, "usable_sensors": 8, "density_01_rows_multi_sensor": 12_670, } def _logical_sensor(name: str, device_id: str) -> str: return config.DEVICE_TO_SENSOR.get(device_id, name) def load_raw(path: Path | None = None) -> pd.DataFrame: path = path or config.raw_csv_path() if not path.exists(): raise FileNotFoundError( f"raw CSV not found at {path}. Set PULSECAST_RAW_CSV or place the " "challenge download at that path (never commit it)." ) df = pd.read_csv(path, parse_dates=["time"], dtype={"sensor_id": "string", "name": "string"}) df["time"] = pd.to_datetime(df["time"], utc=True) expected = {"time", "sensor_id", "name", "longitude", "latitude", "type", "value"} missing = expected - set(df.columns) if missing: raise ValueError(f"raw CSV missing columns: {sorted(missing)}") return df def clean(df: pd.DataFrame) -> tuple[pd.DataFrame, pd.DataFrame, dict]: """Return (density, battery, audit) with logical sensors and exclusions applied.""" density = df[df["type"] == "densityData"].copy() battery = df[df["type"] == "batteryData"].copy() for frame in (density, battery): frame["sensor"] = [ _logical_sensor(n, d) for n, d in zip(frame["name"], frame["sensor_id"], strict=True) ] excluded_rows = { name: int((density["sensor"] == name).sum()) for name in config.EXCLUDED_SENSORS } density = density[~density["sensor"].isin(config.EXCLUDED_SENSORS)].copy() battery = battery[~battery["sensor"].isin(config.EXCLUDED_SENSORS)].copy() before = len(density) density = density[ density["value"].between(config.VALUE_MIN, config.VALUE_MAX, inclusive="both") ].copy() dropped_out_of_range = before - len(density) usable = sorted(density["sensor"].unique().tolist()) if set(usable) != set(config.USABLE_SENSORS): raise ValueError( f"usable sensor set changed: {usable}; expected {sorted(config.USABLE_SENSORS)}" ) multi = density[density["time"] >= pd.Timestamp(config.MULTI_SENSOR_START)] # The device merge spans the full history: the old device is outside the # 8-sensor window, so counting only `multi` would hide it. density_01_devices = ( density[density["sensor"] == "density-01"] .groupby("sensor_id", observed=True) .agg(rows=("value", "size"), first=("time", "min"), last=("time", "max")) ) # One reading per (time, device, type); duplicates would double-weight a bin. dup_keys = int(density.duplicated(subset=["time", "sensor_id"]).sum()) if dup_keys: density = density.drop_duplicates(subset=["time", "sensor_id"], keep="last") cadence = ( density.sort_values(["sensor", "time"]) .groupby("sensor", observed=True)["time"] .diff() .dt.total_seconds() .div(60) .dropna() ) audit = { "raw_rows": len(df), "density_rows": len(df[df["type"] == "densityData"]), "battery_rows": len(df[df["type"] == "batteryData"]), "device_ids": int(df["sensor_id"].nunique()), "sensor_names": int(df["name"].nunique()), "usable_sensors": len(usable), "sensor_ids": usable, "excluded_sensors": { name: {"rows": excluded_rows[name], "reason": reason} for name, reason in config.EXCLUDED_SENSORS.items() }, "density_01_devices": { str(device): { "rows": int(row["rows"]), "first": _iso(row["first"]), "last": _iso(row["last"]), } for device, row in density_01_devices.iterrows() }, "density_01_rows_multi_sensor": int( (multi["sensor"] == "density-01").sum() ), "time_range": { "from": _iso(density["time"].min()), "to": _iso(density["time"].max()), }, "multi_sensor_from": _iso(multi["time"].min()), "cadence_median_min": round(float(cadence.median()), 2), "dropped_out_of_range": int(dropped_out_of_range), "duplicate_keys_dropped": dup_keys, "checks": [], "all_checks_ok": True, } for key, expected in EXPECTED_CHECKS.items(): actual = audit[key] ok = int(actual) == int(expected) audit["checks"].append({"name": key, "expected": expected, "actual": actual, "ok": ok}) if not ok: audit["all_checks_ok"] = False log.warning("data audit mismatch %s: expected %s, got %s", key, expected, actual) return density, battery, audit def write_audit(audit: dict, path: Path | None = None) -> Path: path = path or config.artifacts_dir() / "data_audit.json" path.parent.mkdir(parents=True, exist_ok=True) path.write_text(json.dumps(audit, indent=2, ensure_ascii=False) + "\n") return path def load_cached_density() -> pd.DataFrame: path = config.cache_dir() / "density_clean.parquet" if not path.exists(): raise FileNotFoundError(f"{path} missing; run `make data` first") return pd.read_parquet(path) def _iso(ts) -> str | None: if ts is None or pd.isna(ts): return None return pd.Timestamp(ts).tz_convert("UTC").strftime("%Y-%m-%dT%H:%M:%SZ")