Files
ML-py/src/pulsecast/parse.py
T
Noel Sabia 939a8db482 feat: complete ML pipeline, FastAPI service, small model, tests, image
- finish the global HistGB bundle, scenario forecasts and 28-day ablation
- export sensors/series/forecasts/metrics/context plus baked weather frames
- add FastAPI health/meta/predict/simulate and background /train jobs
- add the p75 busy classifier, retraining from CSV, and hot-swap persistence
- add pytest coverage, multi-stage non-root Dockerfile, README and model card

Holdout: weather 7d MAE 3.32 / WAPE 27.5% / R2 0.777 vs seasonal naive
MAE 4.23 / WAPE 35.0% / R2 0.656 (5,350 eval rows).
2026-11-14 05:01:16 +01:00

165 lines
5.8 KiB
Python

"""Parse and clean the raw PAX CSV.
Verifies the claims in mission/data.md: 192,464 rows, 12 device IDs, 8 usable
logical locations, three excluded test devices, and the density-01 merge.
The raw file is never written to git; a cached parquet copy lives in data/cache/.
"""
from __future__ import annotations
import json
import logging
from pathlib import Path
import pandas as pd
from . import config
log = logging.getLogger(__name__)
EXPECTED_CHECKS = {
"raw_rows": 192_464,
"density_rows": 96_232,
"battery_rows": 96_232,
"device_ids": 12,
"sensor_names": 11,
"usable_sensors": 8,
"density_01_rows_multi_sensor": 12_670,
}
def _logical_sensor(name: str, device_id: str) -> str:
return config.DEVICE_TO_SENSOR.get(device_id, name)
def load_raw(path: Path | None = None) -> pd.DataFrame:
path = path or config.raw_csv_path()
if not path.exists():
raise FileNotFoundError(
f"raw CSV not found at {path}. Set PULSECAST_RAW_CSV or place the "
"challenge download at that path (never commit it)."
)
df = pd.read_csv(path, parse_dates=["time"], dtype={"sensor_id": "string", "name": "string"})
df["time"] = pd.to_datetime(df["time"], utc=True)
expected = {"time", "sensor_id", "name", "longitude", "latitude", "type", "value"}
missing = expected - set(df.columns)
if missing:
raise ValueError(f"raw CSV missing columns: {sorted(missing)}")
return df
def clean(df: pd.DataFrame) -> tuple[pd.DataFrame, pd.DataFrame, dict]:
"""Return (density, battery, audit) with logical sensors and exclusions applied."""
density = df[df["type"] == "densityData"].copy()
battery = df[df["type"] == "batteryData"].copy()
for frame in (density, battery):
frame["sensor"] = [
_logical_sensor(n, d) for n, d in zip(frame["name"], frame["sensor_id"], strict=True)
]
excluded_rows = {
name: int((density["sensor"] == name).sum())
for name in config.EXCLUDED_SENSORS
}
density = density[~density["sensor"].isin(config.EXCLUDED_SENSORS)].copy()
battery = battery[~battery["sensor"].isin(config.EXCLUDED_SENSORS)].copy()
before = len(density)
density = density[
density["value"].between(config.VALUE_MIN, config.VALUE_MAX, inclusive="both")
].copy()
dropped_out_of_range = before - len(density)
usable = sorted(density["sensor"].unique().tolist())
if set(usable) != set(config.USABLE_SENSORS):
raise ValueError(
f"usable sensor set changed: {usable}; expected {sorted(config.USABLE_SENSORS)}"
)
multi = density[density["time"] >= pd.Timestamp(config.MULTI_SENSOR_START)]
# The device merge spans the full history: the old device is outside the
# 8-sensor window, so counting only `multi` would hide it.
density_01_devices = (
density[density["sensor"] == "density-01"]
.groupby("sensor_id", observed=True)
.agg(rows=("value", "size"), first=("time", "min"), last=("time", "max"))
)
# One reading per (time, device, type); duplicates would double-weight a bin.
dup_keys = int(density.duplicated(subset=["time", "sensor_id"]).sum())
if dup_keys:
density = density.drop_duplicates(subset=["time", "sensor_id"], keep="last")
cadence = (
density.sort_values(["sensor", "time"])
.groupby("sensor", observed=True)["time"]
.diff()
.dt.total_seconds()
.div(60)
.dropna()
)
audit = {
"raw_rows": len(df),
"density_rows": len(df[df["type"] == "densityData"]),
"battery_rows": len(df[df["type"] == "batteryData"]),
"device_ids": int(df["sensor_id"].nunique()),
"sensor_names": int(df["name"].nunique()),
"usable_sensors": len(usable),
"sensor_ids": usable,
"excluded_sensors": {
name: {"rows": excluded_rows[name], "reason": reason}
for name, reason in config.EXCLUDED_SENSORS.items()
},
"density_01_devices": {
str(device): {
"rows": int(row["rows"]),
"first": _iso(row["first"]),
"last": _iso(row["last"]),
}
for device, row in density_01_devices.iterrows()
},
"density_01_rows_multi_sensor": int(
(multi["sensor"] == "density-01").sum()
),
"time_range": {
"from": _iso(density["time"].min()),
"to": _iso(density["time"].max()),
},
"multi_sensor_from": _iso(multi["time"].min()),
"cadence_median_min": round(float(cadence.median()), 2),
"dropped_out_of_range": int(dropped_out_of_range),
"duplicate_keys_dropped": dup_keys,
"checks": [],
"all_checks_ok": True,
}
for key, expected in EXPECTED_CHECKS.items():
actual = audit[key]
ok = int(actual) == int(expected)
audit["checks"].append({"name": key, "expected": expected, "actual": actual, "ok": ok})
if not ok:
audit["all_checks_ok"] = False
log.warning("data audit mismatch %s: expected %s, got %s", key, expected, actual)
return density, battery, audit
def write_audit(audit: dict, path: Path | None = None) -> Path:
path = path or config.artifacts_dir() / "data_audit.json"
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(audit, indent=2, ensure_ascii=False) + "\n")
return path
def load_cached_density() -> pd.DataFrame:
path = config.cache_dir() / "density_clean.parquet"
if not path.exists():
raise FileNotFoundError(f"{path} missing; run `make data` first")
return pd.read_parquet(path)
def _iso(ts) -> str | None:
if ts is None or pd.isna(ts):
return None
return pd.Timestamp(ts).tz_convert("UTC").strftime("%Y-%m-%dT%H:%M:%SZ")