- finish the global HistGB bundle, scenario forecasts and 28-day ablation - export sensors/series/forecasts/metrics/context plus baked weather frames - add FastAPI health/meta/predict/simulate and background /train jobs - add the p75 busy classifier, retraining from CSV, and hot-swap persistence - add pytest coverage, multi-stage non-root Dockerfile, README and model card Holdout: weather 7d MAE 3.32 / WAPE 27.5% / R2 0.777 vs seasonal naive MAE 4.23 / WAPE 35.0% / R2 0.656 (5,350 eval rows).
165 lines
5.8 KiB
Python
165 lines
5.8 KiB
Python
"""Parse and clean the raw PAX CSV.
|
|
|
|
Verifies the claims in mission/data.md: 192,464 rows, 12 device IDs, 8 usable
|
|
logical locations, three excluded test devices, and the density-01 merge.
|
|
The raw file is never written to git; a cached parquet copy lives in data/cache/.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import logging
|
|
from pathlib import Path
|
|
|
|
import pandas as pd
|
|
|
|
from . import config
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
EXPECTED_CHECKS = {
|
|
"raw_rows": 192_464,
|
|
"density_rows": 96_232,
|
|
"battery_rows": 96_232,
|
|
"device_ids": 12,
|
|
"sensor_names": 11,
|
|
"usable_sensors": 8,
|
|
"density_01_rows_multi_sensor": 12_670,
|
|
}
|
|
|
|
|
|
def _logical_sensor(name: str, device_id: str) -> str:
|
|
return config.DEVICE_TO_SENSOR.get(device_id, name)
|
|
|
|
|
|
def load_raw(path: Path | None = None) -> pd.DataFrame:
|
|
path = path or config.raw_csv_path()
|
|
if not path.exists():
|
|
raise FileNotFoundError(
|
|
f"raw CSV not found at {path}. Set PULSECAST_RAW_CSV or place the "
|
|
"challenge download at that path (never commit it)."
|
|
)
|
|
df = pd.read_csv(path, parse_dates=["time"], dtype={"sensor_id": "string", "name": "string"})
|
|
df["time"] = pd.to_datetime(df["time"], utc=True)
|
|
expected = {"time", "sensor_id", "name", "longitude", "latitude", "type", "value"}
|
|
missing = expected - set(df.columns)
|
|
if missing:
|
|
raise ValueError(f"raw CSV missing columns: {sorted(missing)}")
|
|
return df
|
|
|
|
|
|
def clean(df: pd.DataFrame) -> tuple[pd.DataFrame, pd.DataFrame, dict]:
|
|
"""Return (density, battery, audit) with logical sensors and exclusions applied."""
|
|
density = df[df["type"] == "densityData"].copy()
|
|
battery = df[df["type"] == "batteryData"].copy()
|
|
|
|
for frame in (density, battery):
|
|
frame["sensor"] = [
|
|
_logical_sensor(n, d) for n, d in zip(frame["name"], frame["sensor_id"], strict=True)
|
|
]
|
|
|
|
excluded_rows = {
|
|
name: int((density["sensor"] == name).sum())
|
|
for name in config.EXCLUDED_SENSORS
|
|
}
|
|
density = density[~density["sensor"].isin(config.EXCLUDED_SENSORS)].copy()
|
|
battery = battery[~battery["sensor"].isin(config.EXCLUDED_SENSORS)].copy()
|
|
|
|
before = len(density)
|
|
density = density[
|
|
density["value"].between(config.VALUE_MIN, config.VALUE_MAX, inclusive="both")
|
|
].copy()
|
|
dropped_out_of_range = before - len(density)
|
|
|
|
usable = sorted(density["sensor"].unique().tolist())
|
|
if set(usable) != set(config.USABLE_SENSORS):
|
|
raise ValueError(
|
|
f"usable sensor set changed: {usable}; expected {sorted(config.USABLE_SENSORS)}"
|
|
)
|
|
|
|
multi = density[density["time"] >= pd.Timestamp(config.MULTI_SENSOR_START)]
|
|
# The device merge spans the full history: the old device is outside the
|
|
# 8-sensor window, so counting only `multi` would hide it.
|
|
density_01_devices = (
|
|
density[density["sensor"] == "density-01"]
|
|
.groupby("sensor_id", observed=True)
|
|
.agg(rows=("value", "size"), first=("time", "min"), last=("time", "max"))
|
|
)
|
|
|
|
# One reading per (time, device, type); duplicates would double-weight a bin.
|
|
dup_keys = int(density.duplicated(subset=["time", "sensor_id"]).sum())
|
|
if dup_keys:
|
|
density = density.drop_duplicates(subset=["time", "sensor_id"], keep="last")
|
|
|
|
cadence = (
|
|
density.sort_values(["sensor", "time"])
|
|
.groupby("sensor", observed=True)["time"]
|
|
.diff()
|
|
.dt.total_seconds()
|
|
.div(60)
|
|
.dropna()
|
|
)
|
|
|
|
audit = {
|
|
"raw_rows": len(df),
|
|
"density_rows": len(df[df["type"] == "densityData"]),
|
|
"battery_rows": len(df[df["type"] == "batteryData"]),
|
|
"device_ids": int(df["sensor_id"].nunique()),
|
|
"sensor_names": int(df["name"].nunique()),
|
|
"usable_sensors": len(usable),
|
|
"sensor_ids": usable,
|
|
"excluded_sensors": {
|
|
name: {"rows": excluded_rows[name], "reason": reason}
|
|
for name, reason in config.EXCLUDED_SENSORS.items()
|
|
},
|
|
"density_01_devices": {
|
|
str(device): {
|
|
"rows": int(row["rows"]),
|
|
"first": _iso(row["first"]),
|
|
"last": _iso(row["last"]),
|
|
}
|
|
for device, row in density_01_devices.iterrows()
|
|
},
|
|
"density_01_rows_multi_sensor": int(
|
|
(multi["sensor"] == "density-01").sum()
|
|
),
|
|
"time_range": {
|
|
"from": _iso(density["time"].min()),
|
|
"to": _iso(density["time"].max()),
|
|
},
|
|
"multi_sensor_from": _iso(multi["time"].min()),
|
|
"cadence_median_min": round(float(cadence.median()), 2),
|
|
"dropped_out_of_range": int(dropped_out_of_range),
|
|
"duplicate_keys_dropped": dup_keys,
|
|
"checks": [],
|
|
"all_checks_ok": True,
|
|
}
|
|
for key, expected in EXPECTED_CHECKS.items():
|
|
actual = audit[key]
|
|
ok = int(actual) == int(expected)
|
|
audit["checks"].append({"name": key, "expected": expected, "actual": actual, "ok": ok})
|
|
if not ok:
|
|
audit["all_checks_ok"] = False
|
|
log.warning("data audit mismatch %s: expected %s, got %s", key, expected, actual)
|
|
return density, battery, audit
|
|
|
|
|
|
def write_audit(audit: dict, path: Path | None = None) -> Path:
|
|
path = path or config.artifacts_dir() / "data_audit.json"
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_text(json.dumps(audit, indent=2, ensure_ascii=False) + "\n")
|
|
return path
|
|
|
|
|
|
def load_cached_density() -> pd.DataFrame:
|
|
path = config.cache_dir() / "density_clean.parquet"
|
|
if not path.exists():
|
|
raise FileNotFoundError(f"{path} missing; run `make data` first")
|
|
return pd.read_parquet(path)
|
|
|
|
|
|
def _iso(ts) -> str | None:
|
|
if ts is None or pd.isna(ts):
|
|
return None
|
|
return pd.Timestamp(ts).tz_convert("UTC").strftime("%Y-%m-%dT%H:%M:%SZ")
|