feat: complete ML pipeline, FastAPI service, small model, tests, image
- finish the global HistGB bundle, scenario forecasts and 28-day ablation - export sensors/series/forecasts/metrics/context plus baked weather frames - add FastAPI health/meta/predict/simulate and background /train jobs - add the p75 busy classifier, retraining from CSV, and hot-swap persistence - add pytest coverage, multi-stage non-root Dockerfile, README and model card Holdout: weather 7d MAE 3.32 / WAPE 27.5% / R2 0.777 vs seasonal naive MAE 4.23 / WAPE 35.0% / R2 0.656 (5,350 eval rows).
This commit is contained in:
1 parent
68402803b2
commit
939a8db482
43 files changed
+4090
-105
No files matched your search
+25
-10
@@ -5,7 +5,6 @@ from __future__ import annotations
|
||||
import argparse
|
||||
import json
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
import pandas as pd
|
||||
|
||||
@@ -52,7 +51,7 @@ def export_sensors(density: pd.DataFrame, battery: pd.DataFrame) -> list[dict]:
|
||||
"mean": round(float(rows["value"].mean()), 1),
|
||||
"p95": round(float(rows["value"].quantile(0.95)), 1),
|
||||
"battery": (
|
||||
int(round(float(battery_latest.loc[sensor, "value"])))
|
||||
round(float(battery_latest.loc[sensor, "value"]))
|
||||
if sensor in battery_latest.index
|
||||
else None
|
||||
),
|
||||
@@ -94,9 +93,9 @@ def _weather_now(forecast: pd.DataFrame, meta: dict) -> dict:
|
||||
"t": rfc3339(ts),
|
||||
"temp_c": round(float(current.get("temp_c", row["temp_c"])), 1),
|
||||
"precip_mm": round(float(current.get("precip_mm", row["precip_mm"])), 2),
|
||||
"cloud_pct": int(round(float(current.get("cloud_pct", row["cloud_pct"])))),
|
||||
"cloud_pct": round(float(current.get("cloud_pct", row["cloud_pct"]))),
|
||||
"wind_kmh": round(float(current.get("wind_kmh", row["wind_kmh"])), 1),
|
||||
"humidity_pct": int(round(float(current.get("humidity_pct", row["humidity_pct"])))),
|
||||
"humidity_pct": round(float(current.get("humidity_pct", row["humidity_pct"]))),
|
||||
}
|
||||
|
||||
|
||||
@@ -109,9 +108,9 @@ def export_context(forecast: pd.DataFrame, meta: dict) -> dict:
|
||||
"t": rfc3339(row.time),
|
||||
"temp_c": round(float(row.temp_c), 1),
|
||||
"precip_mm": round(float(row.precip_mm), 2),
|
||||
"cloud_pct": int(round(float(row.cloud_pct))),
|
||||
"cloud_pct": round(float(row.cloud_pct)),
|
||||
"wind_kmh": round(float(row.wind_kmh), 1),
|
||||
"humidity_pct": int(round(float(row.humidity_pct))),
|
||||
"humidity_pct": round(float(row.humidity_pct)),
|
||||
}
|
||||
for row in window.itertuples(index=False)
|
||||
]
|
||||
@@ -140,6 +139,14 @@ def run_data(*, refresh_weather: bool = False) -> dict:
|
||||
raw = load_raw()
|
||||
density, battery, audit = clean(raw)
|
||||
write_audit(audit)
|
||||
|
||||
# The product window is the 8-sensor period. density-01 has 13 months of
|
||||
# history, but using it alone would add single-sensor annual seasonality that
|
||||
# the other seven locations cannot support. Keep the full-file audit above.
|
||||
window_start = pd.Timestamp(config.MULTI_SENSOR_START)
|
||||
density = density[density["time"] >= window_start].copy()
|
||||
battery = battery[battery["time"] >= window_start].copy()
|
||||
|
||||
density[["time", "sensor", "sensor_id", "value"]].to_parquet(
|
||||
cache / "density_clean.parquet", index=False
|
||||
)
|
||||
@@ -147,7 +154,7 @@ def run_data(*, refresh_weather: bool = False) -> dict:
|
||||
cache / "battery_clean.parquet", index=False
|
||||
)
|
||||
log.info(
|
||||
"parsed %s rows; %s usable sensors; %s density rows",
|
||||
"parsed %s rows; %s usable sensors; %s density rows in the 8-sensor window",
|
||||
audit["raw_rows"],
|
||||
audit["usable_sensors"],
|
||||
len(density),
|
||||
@@ -163,6 +170,14 @@ def run_data(*, refresh_weather: bool = False) -> dict:
|
||||
features.to_parquet(cache / "hourly_features.parquet", index=False)
|
||||
log.info("built %s hourly feature rows", len(features))
|
||||
|
||||
# Bake compact weather frames beside the JSON so the FastAPI image can
|
||||
# retrain and serve /predict without the git-ignored data/cache directory.
|
||||
archive.to_parquet(artifacts / "weather_archive.parquet", index=False)
|
||||
forecast.to_parquet(artifacts / "weather_forecast.parquet", index=False)
|
||||
(artifacts / "weather_meta.json").write_text(
|
||||
json.dumps(meta, indent=2, ensure_ascii=False) + "\n"
|
||||
)
|
||||
|
||||
data_version = minute_iso(audit["time_range"]["to"])
|
||||
sensors = export_sensors(density, battery)
|
||||
(artifacts / "sensors.json").write_text(
|
||||
@@ -174,9 +189,9 @@ def run_data(*, refresh_weather: bool = False) -> dict:
|
||||
summary = {
|
||||
"sensors": len(sensors),
|
||||
"series_sensors": len(series["series"]),
|
||||
"hourly_rows": int(len(hourly)),
|
||||
"feature_rows": int(len(features)),
|
||||
"weather_rows": int(len(archive)),
|
||||
"hourly_rows": len(hourly),
|
||||
"feature_rows": len(features),
|
||||
"weather_rows": len(archive),
|
||||
"context_forecast_hours": len(context["weather"]["forecast"]),
|
||||
"data_version": data_version,
|
||||
"audit_ok": audit["all_checks_ok"],
|
||||
|
||||
Reference in new issue
Block a user