Files
ML-py/src/pulsecast/pipeline.py
T
Noel Sabia 939a8db482 feat: complete ML pipeline, FastAPI service, small model, tests, image
- finish the global HistGB bundle, scenario forecasts and 28-day ablation
- export sensors/series/forecasts/metrics/context plus baked weather frames
- add FastAPI health/meta/predict/simulate and background /train jobs
- add the p75 busy classifier, retraining from CSV, and hot-swap persistence
- add pytest coverage, multi-stage non-root Dockerfile, README and model card

Holdout: weather 7d MAE 3.32 / WAPE 27.5% / R2 0.777 vs seasonal naive
MAE 4.23 / WAPE 35.0% / R2 0.656 (5,350 eval rows).
2026-11-14 05:01:16 +01:00

212 lines
7.7 KiB
Python

"""`make data`: parse CSV, aggregate hourly, join weather/holidays, export JSON."""
from __future__ import annotations
import argparse
import json
import logging
import pandas as pd
from . import config
from .features import build_features, to_hourly
from .parse import clean, load_raw, write_audit
from .weather import ensure_weather
log = logging.getLogger(__name__)
def rfc3339(ts) -> str:
return pd.Timestamp(ts).tz_convert("UTC").strftime("%Y-%m-%dT%H:%M:%SZ")
def minute_iso(value: str | None) -> str | None:
if not value:
return None
return pd.Timestamp(value).tz_convert("UTC").floor("min").strftime("%Y-%m-%dT%H:%M:%SZ")
def _series_points(frame: pd.DataFrame) -> list[dict]:
return [
{"t": rfc3339(t), "v": round(float(v), 2)}
for t, v in zip(frame["t"], frame["v"], strict=True)
]
def export_sensors(density: pd.DataFrame, battery: pd.DataFrame) -> list[dict]:
multi = density[density["time"] >= pd.Timestamp(config.MULTI_SENSOR_START)]
battery_latest = (
battery.sort_values("time").groupby("sensor", observed=True).tail(1).set_index("sensor")
)
out = []
for sensor in sorted(config.USABLE_SENSORS):
rows = multi[multi["sensor"] == sensor]
last = density[density["sensor"] == sensor]["time"].max()
out.append(
{
"id": sensor,
"name": config.DISPLAY_NAMES[sensor],
"lat": round(float(rows["latitude"].median()), 6),
"lon": round(float(rows["longitude"].median()), 6),
"mean": round(float(rows["value"].mean()), 1),
"p95": round(float(rows["value"].quantile(0.95)), 1),
"battery": (
round(float(battery_latest.loc[sensor, "value"]))
if sensor in battery_latest.index
else None
),
"last_seen": rfc3339(last),
}
)
return out
def export_series(hourly: pd.DataFrame, data_version: str) -> dict:
artifacts = config.artifacts_dir()
series_dir = artifacts / "series"
series_dir.mkdir(parents=True, exist_ok=True)
per_sensor: dict[str, list[dict]] = {}
for sensor in sorted(config.USABLE_SENSORS):
frame = hourly[hourly["sensor"] == sensor]
pts = _series_points(frame)
per_sensor[sensor] = pts
(series_dir / f"{sensor}.json").write_text(
json.dumps(
{"sensor": sensor, "step": "1h", "points": pts}, ensure_ascii=False
)
+ "\n"
)
payload = {"step": "1h", "data_version": data_version, "series": per_sensor}
(artifacts / "series.json").write_text(json.dumps(payload, ensure_ascii=False) + "\n")
return payload
def _weather_now(forecast: pd.DataFrame, meta: dict) -> dict:
current = meta.get("current") or {}
now_t = current.get("t")
if now_t:
ts = pd.Timestamp(now_t, tz="UTC") if pd.Timestamp(now_t).tz is None else pd.Timestamp(now_t).tz_convert("UTC")
else:
ts = pd.Timestamp(meta["fetched_at"]).tz_convert("UTC")
row = forecast.iloc[(forecast["time"] - ts).abs().argmin()]
return {
"t": rfc3339(ts),
"temp_c": round(float(current.get("temp_c", row["temp_c"])), 1),
"precip_mm": round(float(current.get("precip_mm", row["precip_mm"])), 2),
"cloud_pct": round(float(current.get("cloud_pct", row["cloud_pct"]))),
"wind_kmh": round(float(current.get("wind_kmh", row["wind_kmh"])), 1),
"humidity_pct": round(float(current.get("humidity_pct", row["humidity_pct"]))),
}
def export_context(forecast: pd.DataFrame, meta: dict) -> dict:
now = _weather_now(forecast, meta)
start = pd.Timestamp(now["t"]).floor("h")
window = forecast[(forecast["time"] >= start) & (forecast["time"] < start + pd.Timedelta(days=7))]
points = [
{
"t": rfc3339(row.time),
"temp_c": round(float(row.temp_c), 1),
"precip_mm": round(float(row.precip_mm), 2),
"cloud_pct": round(float(row.cloud_pct)),
"wind_kmh": round(float(row.wind_kmh), 1),
"humidity_pct": round(float(row.humidity_pct)),
}
for row in window.itertuples(index=False)
]
holidays = [
{"date": date, "name": name} for date, name in sorted(config.HOLIDAYS.items())
]
payload = {
"weather": {"now": now, "forecast": points},
"holidays": holidays,
"updated_at": meta["fetched_at"],
"cached": False,
}
(config.artifacts_dir() / "context.json").write_text(
json.dumps(payload, indent=2, ensure_ascii=False) + "\n"
)
return payload
def run_data(*, refresh_weather: bool = False) -> dict:
logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s")
artifacts = config.artifacts_dir()
cache = config.cache_dir()
artifacts.mkdir(parents=True, exist_ok=True)
cache.mkdir(parents=True, exist_ok=True)
raw = load_raw()
density, battery, audit = clean(raw)
write_audit(audit)
# The product window is the 8-sensor period. density-01 has 13 months of
# history, but using it alone would add single-sensor annual seasonality that
# the other seven locations cannot support. Keep the full-file audit above.
window_start = pd.Timestamp(config.MULTI_SENSOR_START)
density = density[density["time"] >= window_start].copy()
battery = battery[battery["time"] >= window_start].copy()
density[["time", "sensor", "sensor_id", "value"]].to_parquet(
cache / "density_clean.parquet", index=False
)
battery[["time", "sensor", "sensor_id", "value"]].to_parquet(
cache / "battery_clean.parquet", index=False
)
log.info(
"parsed %s rows; %s usable sensors; %s density rows in the 8-sensor window",
audit["raw_rows"],
audit["usable_sensors"],
len(density),
)
hourly = to_hourly(density)
hourly.to_parquet(cache / "hourly.parquet", index=False)
archive, forecast, meta = ensure_weather(
density["time"].min(), density["time"].max(), refresh=refresh_weather
)
features = build_features(hourly, archive)
features.to_parquet(cache / "hourly_features.parquet", index=False)
log.info("built %s hourly feature rows", len(features))
# Bake compact weather frames beside the JSON so the FastAPI image can
# retrain and serve /predict without the git-ignored data/cache directory.
archive.to_parquet(artifacts / "weather_archive.parquet", index=False)
forecast.to_parquet(artifacts / "weather_forecast.parquet", index=False)
(artifacts / "weather_meta.json").write_text(
json.dumps(meta, indent=2, ensure_ascii=False) + "\n"
)
data_version = minute_iso(audit["time_range"]["to"])
sensors = export_sensors(density, battery)
(artifacts / "sensors.json").write_text(
json.dumps(sensors, indent=2, ensure_ascii=False) + "\n"
)
series = export_series(hourly, data_version)
context = export_context(forecast, meta)
summary = {
"sensors": len(sensors),
"series_sensors": len(series["series"]),
"hourly_rows": len(hourly),
"feature_rows": len(features),
"weather_rows": len(archive),
"context_forecast_hours": len(context["weather"]["forecast"]),
"data_version": data_version,
"audit_ok": audit["all_checks_ok"],
}
log.info("data artifacts written to %s: %s", artifacts, json.dumps(summary))
return summary
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--refresh-weather", action="store_true")
args = parser.parse_args()
run_data(refresh_weather=args.refresh_weather)
if __name__ == "__main__":
main()