feat: complete ML pipeline, FastAPI service, small model, tests, image

- finish the global HistGB bundle, scenario forecasts and 28-day ablation
- export sensors/series/forecasts/metrics/context plus baked weather frames
- add FastAPI health/meta/predict/simulate and background /train jobs
- add the p75 busy classifier, retraining from CSV, and hot-swap persistence
- add pytest coverage, multi-stage non-root Dockerfile, README and model card

Holdout: weather 7d MAE 3.32 / WAPE 27.5% / R2 0.777 vs seasonal naive
MAE 4.23 / WAPE 35.0% / R2 0.656 (5,350 eval rows).
This commit is contained in:
Noel Sabia committed 2026-11-14 05:01:16 +01:00
1 parent 68402803b2
commit 939a8db482
43 files changed
+4090 -105

No files matched your search

+25 -10
View File
@@ -5,7 +5,6 @@ from __future__ import annotations
import argparse
import json
import logging
from pathlib import Path
import pandas as pd
@@ -52,7 +51,7 @@ def export_sensors(density: pd.DataFrame, battery: pd.DataFrame) -> list[dict]:
"mean": round(float(rows["value"].mean()), 1),
"p95": round(float(rows["value"].quantile(0.95)), 1),
"battery": (
int(round(float(battery_latest.loc[sensor, "value"])))
round(float(battery_latest.loc[sensor, "value"]))
if sensor in battery_latest.index
else None
),
@@ -94,9 +93,9 @@ def _weather_now(forecast: pd.DataFrame, meta: dict) -> dict:
"t": rfc3339(ts),
"temp_c": round(float(current.get("temp_c", row["temp_c"])), 1),
"precip_mm": round(float(current.get("precip_mm", row["precip_mm"])), 2),
"cloud_pct": int(round(float(current.get("cloud_pct", row["cloud_pct"])))),
"cloud_pct": round(float(current.get("cloud_pct", row["cloud_pct"]))),
"wind_kmh": round(float(current.get("wind_kmh", row["wind_kmh"])), 1),
"humidity_pct": int(round(float(current.get("humidity_pct", row["humidity_pct"])))),
"humidity_pct": round(float(current.get("humidity_pct", row["humidity_pct"]))),
}
@@ -109,9 +108,9 @@ def export_context(forecast: pd.DataFrame, meta: dict) -> dict:
"t": rfc3339(row.time),
"temp_c": round(float(row.temp_c), 1),
"precip_mm": round(float(row.precip_mm), 2),
"cloud_pct": int(round(float(row.cloud_pct))),
"cloud_pct": round(float(row.cloud_pct)),
"wind_kmh": round(float(row.wind_kmh), 1),
"humidity_pct": int(round(float(row.humidity_pct))),
"humidity_pct": round(float(row.humidity_pct)),
}
for row in window.itertuples(index=False)
]
@@ -140,6 +139,14 @@ def run_data(*, refresh_weather: bool = False) -> dict:
raw = load_raw()
density, battery, audit = clean(raw)
write_audit(audit)
# The product window is the 8-sensor period. density-01 has 13 months of
# history, but using it alone would add single-sensor annual seasonality that
# the other seven locations cannot support. Keep the full-file audit above.
window_start = pd.Timestamp(config.MULTI_SENSOR_START)
density = density[density["time"] >= window_start].copy()
battery = battery[battery["time"] >= window_start].copy()
density[["time", "sensor", "sensor_id", "value"]].to_parquet(
cache / "density_clean.parquet", index=False
)
@@ -147,7 +154,7 @@ def run_data(*, refresh_weather: bool = False) -> dict:
cache / "battery_clean.parquet", index=False
)
log.info(
"parsed %s rows; %s usable sensors; %s density rows",
"parsed %s rows; %s usable sensors; %s density rows in the 8-sensor window",
audit["raw_rows"],
audit["usable_sensors"],
len(density),
@@ -163,6 +170,14 @@ def run_data(*, refresh_weather: bool = False) -> dict:
features.to_parquet(cache / "hourly_features.parquet", index=False)
log.info("built %s hourly feature rows", len(features))
# Bake compact weather frames beside the JSON so the FastAPI image can
# retrain and serve /predict without the git-ignored data/cache directory.
archive.to_parquet(artifacts / "weather_archive.parquet", index=False)
forecast.to_parquet(artifacts / "weather_forecast.parquet", index=False)
(artifacts / "weather_meta.json").write_text(
json.dumps(meta, indent=2, ensure_ascii=False) + "\n"
)
data_version = minute_iso(audit["time_range"]["to"])
sensors = export_sensors(density, battery)
(artifacts / "sensors.json").write_text(
@@ -174,9 +189,9 @@ def run_data(*, refresh_weather: bool = False) -> dict:
summary = {
"sensors": len(sensors),
"series_sensors": len(series["series"]),
"hourly_rows": int(len(hourly)),
"feature_rows": int(len(features)),
"weather_rows": int(len(archive)),
"hourly_rows": len(hourly),
"feature_rows": len(features),
"weather_rows": len(archive),
"context_forecast_hours": len(context["weather"]["forecast"]),
"data_version": data_version,
"audit_ok": audit["all_checks_ok"],