feat: ml bootstrap #1

Merged
Noel merged 4 commits from feat/ml-bootstrap into main 2026-11-14 04:17:59 +00:00
2 changed files with 465 additions and 0 deletions
Showing only changes of commit 68402803b2 - Show all commits

No files matched your search

+196
View File
@@ -0,0 +1,196 @@
"""`make data`: parse CSV, aggregate hourly, join weather/holidays, export JSON."""
from __future__ import annotations
import argparse
import json
import logging
from pathlib import Path
import pandas as pd
from . import config
from .features import build_features, to_hourly
from .parse import clean, load_raw, write_audit
from .weather import ensure_weather
log = logging.getLogger(__name__)
def rfc3339(ts) -> str:
return pd.Timestamp(ts).tz_convert("UTC").strftime("%Y-%m-%dT%H:%M:%SZ")
def minute_iso(value: str | None) -> str | None:
if not value:
return None
return pd.Timestamp(value).tz_convert("UTC").floor("min").strftime("%Y-%m-%dT%H:%M:%SZ")
def _series_points(frame: pd.DataFrame) -> list[dict]:
return [
{"t": rfc3339(t), "v": round(float(v), 2)}
for t, v in zip(frame["t"], frame["v"], strict=True)
]
def export_sensors(density: pd.DataFrame, battery: pd.DataFrame) -> list[dict]:
multi = density[density["time"] >= pd.Timestamp(config.MULTI_SENSOR_START)]
battery_latest = (
battery.sort_values("time").groupby("sensor", observed=True).tail(1).set_index("sensor")
)
out = []
for sensor in sorted(config.USABLE_SENSORS):
rows = multi[multi["sensor"] == sensor]
last = density[density["sensor"] == sensor]["time"].max()
out.append(
{
"id": sensor,
"name": config.DISPLAY_NAMES[sensor],
"lat": round(float(rows["latitude"].median()), 6),
"lon": round(float(rows["longitude"].median()), 6),
"mean": round(float(rows["value"].mean()), 1),
"p95": round(float(rows["value"].quantile(0.95)), 1),
"battery": (
int(round(float(battery_latest.loc[sensor, "value"])))
if sensor in battery_latest.index
else None
),
"last_seen": rfc3339(last),
}
)
return out
def export_series(hourly: pd.DataFrame, data_version: str) -> dict:
artifacts = config.artifacts_dir()
series_dir = artifacts / "series"
series_dir.mkdir(parents=True, exist_ok=True)
per_sensor: dict[str, list[dict]] = {}
for sensor in sorted(config.USABLE_SENSORS):
frame = hourly[hourly["sensor"] == sensor]
pts = _series_points(frame)
per_sensor[sensor] = pts
(series_dir / f"{sensor}.json").write_text(
json.dumps(
{"sensor": sensor, "step": "1h", "points": pts}, ensure_ascii=False
)
+ "\n"
)
payload = {"step": "1h", "data_version": data_version, "series": per_sensor}
(artifacts / "series.json").write_text(json.dumps(payload, ensure_ascii=False) + "\n")
return payload
def _weather_now(forecast: pd.DataFrame, meta: dict) -> dict:
current = meta.get("current") or {}
now_t = current.get("t")
if now_t:
ts = pd.Timestamp(now_t, tz="UTC") if pd.Timestamp(now_t).tz is None else pd.Timestamp(now_t).tz_convert("UTC")
else:
ts = pd.Timestamp(meta["fetched_at"]).tz_convert("UTC")
row = forecast.iloc[(forecast["time"] - ts).abs().argmin()]
return {
"t": rfc3339(ts),
"temp_c": round(float(current.get("temp_c", row["temp_c"])), 1),
"precip_mm": round(float(current.get("precip_mm", row["precip_mm"])), 2),
"cloud_pct": int(round(float(current.get("cloud_pct", row["cloud_pct"])))),
"wind_kmh": round(float(current.get("wind_kmh", row["wind_kmh"])), 1),
"humidity_pct": int(round(float(current.get("humidity_pct", row["humidity_pct"])))),
}
def export_context(forecast: pd.DataFrame, meta: dict) -> dict:
now = _weather_now(forecast, meta)
start = pd.Timestamp(now["t"]).floor("h")
window = forecast[(forecast["time"] >= start) & (forecast["time"] < start + pd.Timedelta(days=7))]
points = [
{
"t": rfc3339(row.time),
"temp_c": round(float(row.temp_c), 1),
"precip_mm": round(float(row.precip_mm), 2),
"cloud_pct": int(round(float(row.cloud_pct))),
"wind_kmh": round(float(row.wind_kmh), 1),
"humidity_pct": int(round(float(row.humidity_pct))),
}
for row in window.itertuples(index=False)
]
holidays = [
{"date": date, "name": name} for date, name in sorted(config.HOLIDAYS.items())
]
payload = {
"weather": {"now": now, "forecast": points},
"holidays": holidays,
"updated_at": meta["fetched_at"],
"cached": False,
}
(config.artifacts_dir() / "context.json").write_text(
json.dumps(payload, indent=2, ensure_ascii=False) + "\n"
)
return payload
def run_data(*, refresh_weather: bool = False) -> dict:
logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s")
artifacts = config.artifacts_dir()
cache = config.cache_dir()
artifacts.mkdir(parents=True, exist_ok=True)
cache.mkdir(parents=True, exist_ok=True)
raw = load_raw()
density, battery, audit = clean(raw)
write_audit(audit)
density[["time", "sensor", "sensor_id", "value"]].to_parquet(
cache / "density_clean.parquet", index=False
)
battery[["time", "sensor", "sensor_id", "value"]].to_parquet(
cache / "battery_clean.parquet", index=False
)
log.info(
"parsed %s rows; %s usable sensors; %s density rows",
audit["raw_rows"],
audit["usable_sensors"],
len(density),
)
hourly = to_hourly(density)
hourly.to_parquet(cache / "hourly.parquet", index=False)
archive, forecast, meta = ensure_weather(
density["time"].min(), density["time"].max(), refresh=refresh_weather
)
features = build_features(hourly, archive)
features.to_parquet(cache / "hourly_features.parquet", index=False)
log.info("built %s hourly feature rows", len(features))
data_version = minute_iso(audit["time_range"]["to"])
sensors = export_sensors(density, battery)
(artifacts / "sensors.json").write_text(
json.dumps(sensors, indent=2, ensure_ascii=False) + "\n"
)
series = export_series(hourly, data_version)
context = export_context(forecast, meta)
summary = {
"sensors": len(sensors),
"series_sensors": len(series["series"]),
"hourly_rows": int(len(hourly)),
"feature_rows": int(len(features)),
"weather_rows": int(len(archive)),
"context_forecast_hours": len(context["weather"]["forecast"]),
"data_version": data_version,
"audit_ok": audit["all_checks_ok"],
}
log.info("data artifacts written to %s: %s", artifacts, json.dumps(summary))
return summary
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--refresh-weather", action="store_true")
args = parser.parse_args()
run_data(refresh_weather=args.refresh_weather)
if __name__ == "__main__":
main()
+269
View File
@@ -0,0 +1,269 @@
"""`make train`: fit the global model, evaluate, export metrics/forecasts/model."""
from __future__ import annotations
import json
import logging
from datetime import datetime, timezone
import joblib
import numpy as np
import pandas as pd
from . import config
from .features import seasonal_lookup
from .model import (
event_multiplier,
fit,
future_frame,
metrics,
points,
predict,
residual_quantiles,
weather_climatology,
)
from .pipeline import run_data
from .weather import load_cached_weather
log = logging.getLogger(__name__)
MODEL_SPECS = [
("seasonal_naive", None, "same local hour last week"),
(
"day_no_weather",
config.FEATURE_DAY_NO_HOLIDAY,
"calendar + lags 24/48/168 + 24h/168h rolling means",
),
(
"day_holiday",
config.FEATURE_DAY,
"day-ahead + Baden-Württemberg holiday flag",
),
(
"day_weather",
config.FEATURE_DAY_WEATHER,
"day-ahead + temperature + rain (observed weather)",
),
(
"week_no_weather",
config.FEATURE_WEEK_NO_WEATHER,
"7-day-safe calendar + previous same weekday/hour values",
),
(
"week_weather",
config.FEATURE_WEEK_WEATHER,
"7-day-safe + temperature + rain (production model)",
),
]
def _load_inputs() -> tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame, pd.DataFrame, dict, dict]:
cache = config.cache_dir()
if not (cache / "hourly_features.parquet").exists():
log.info("feature cache missing; running the data pipeline")
run_data()
hourly = pd.read_parquet(cache / "hourly.parquet")
features = pd.read_parquet(cache / "hourly_features.parquet")
archive, forecast, meta = load_cached_weather()
audit = json.loads((config.artifacts_dir() / "data_audit.json").read_text())
return hourly, features, archive, forecast, meta, audit
def _round_row(row: dict) -> dict:
return {
"name": row["name"],
"description": row["description"],
"mae": round(row["mae"], 2),
"wape": round(row["wape"], 1),
"r2": round(row["r2"], 3),
"n": row["n"],
}
def run_train() -> dict:
logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s")
artifacts = config.artifacts_dir()
artifacts.mkdir(parents=True, exist_ok=True)
hourly, features, archive, forecast, meta, audit = _load_inputs()
data_end = hourly["t"].max()
test_start = data_end.floor("h") - pd.Timedelta(days=config.TEST_DAYS)
train = features[features["t"] < test_start].copy()
test = features[features["t"] >= test_start].copy()
eval_frame = test[test["lag_week"].notna() & test["y"].notna()].copy()
if eval_frame.empty:
raise RuntimeError("holdout has no rows with a seasonal-naive baseline")
log.info(
"split: train=%s rows (%s..%s), test=%s rows, eval=%s rows (%s..%s)",
len(train),
train["t"].min(),
train["t"].max(),
len(test),
len(eval_frame),
eval_frame["t"].min(),
eval_frame["t"].max(),
)
results = []
preds: dict[str, np.ndarray] = {}
for name, cols, description in MODEL_SPECS:
if name == "seasonal_naive":
pred = eval_frame["lag_week"].to_numpy(dtype=float)
else:
assert cols is not None
model = fit(train, cols)
pred = np.clip(model.predict(eval_frame[cols]), 0.0, None)
preds[name] = pred
m = metrics(eval_frame["y"], pred)
results.append({"name": name, "description": description, **m})
log.info("%-16s MAE %.2f WAPE %.1f%% R2 %.3f", name, m["mae"], m["wape"], m["r2"])
by_name = {r["name"]: r for r in results}
baseline = by_name["seasonal_naive"]
headline = by_name["week_weather"]
# Production model: refit on all observed rows after the honest holdout.
production = fit(features, config.FEATURE_WEEK_WEATHER)
residual = residual_quantiles(eval_frame, preds["week_weather"])
sensor_mean = (
hourly.groupby("sensor", observed=True)["v"].mean().round(4).to_dict()
)
trained_at = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
model_version = f"histgb-{trained_at[:10]}"
data_version = pd.Timestamp(audit["time_range"]["to"]).floor("min").strftime(
"%Y-%m-%dT%H:%M:%SZ"
)
context_version = meta["fetched_at"]
bundle = {
"model": production,
"features": list(config.FEATURE_WEEK_WEATHER),
"seasonal_lookup": seasonal_lookup(hourly),
"sensor_mean": sensor_mean,
"residual_quantiles": residual,
"weather_climatology": weather_climatology(archive),
"metadata": {
"model_version": model_version,
"data_version": data_version,
"context_version": context_version,
"trained_at": trained_at,
"train_from": train["t"].min().strftime("%Y-%m-%dT%H:%M:%SZ"),
"train_to": features["t"].max().strftime("%Y-%m-%dT%H:%M:%SZ"),
},
}
joblib.dump(bundle, artifacts / "model.joblib")
holiday_effect = {
"mae_delta": round(
by_name["day_holiday"]["mae"] - by_name["day_no_weather"]["mae"], 2
),
"note": (
"Holiday flag alone changes MAE by this amount on the holdout; "
"weather is the main driver, holidays are context."
),
}
metrics_payload = {
"data_version": data_version,
"model_version": model_version,
"context_version": context_version,
"coverage": {
"sensors": int(audit["usable_sensors"]),
"from": config.MULTI_SENSOR_START,
"to": data_version,
"cadence_min": int(round(audit["cadence_median_min"])),
},
"metrics": {
"model": "weather-7d",
"mae": round(headline["mae"], 2),
"wape": round(headline["wape"], 1),
"r2": round(headline["r2"], 3),
"baseline": "seasonal_naive",
"baseline_mae": round(baseline["mae"], 2),
"baseline_wape": round(baseline["wape"], 1),
"baseline_r2": round(baseline["r2"], 3),
"n_eval": int(headline["n"]),
},
"trained_at": trained_at,
"train_from": bundle["metadata"]["train_from"],
"train_to": bundle["metadata"]["train_to"],
"test_from": test_start.strftime("%Y-%m-%dT%H:%M:%SZ"),
"test_to": data_end.strftime("%Y-%m-%dT%H:%M:%SZ"),
"features": list(config.FEATURE_WEEK_WEATHER),
"holiday_effect": holiday_effect,
"ablation": [_round_row(r) for r in results],
"limits": list(config.LIMITS),
}
(artifacts / "metrics.json").write_text(
json.dumps(metrics_payload, indent=2, ensure_ascii=False) + "\n"
)
start = pd.Timestamp.now(tz="UTC").ceil("h")
forecasts = _export_forecasts(bundle, forecast, start)
(artifacts / "forecasts.json").write_text(
json.dumps(forecasts, ensure_ascii=False) + "\n"
)
log.info(
"wrote metrics.json, forecasts.json (%s..%s) and model.joblib",
start,
start + pd.Timedelta(hours=167),
)
return {
"headline": metrics_payload["metrics"],
"ablation": metrics_payload["ablation"],
"model_version": model_version,
"forecast_start": start.strftime("%Y-%m-%dT%H:%M:%SZ"),
}
def _export_forecasts(bundle: dict, forecast: pd.DataFrame, start: pd.Timestamp) -> dict:
periods = config.HORIZON_STEPS["7d"]
sensors = sorted(config.USABLE_SENSORS)
out: dict[str, dict[str, dict[str, list[dict]]]] = {
sensor: {"48h": {}, "7d": {}} for sensor in sensors
}
base_calendar = None
for name, cfg in config.SCENARIOS.items():
event = cfg.get("event")
for sensor in sensors:
from .features import future_calendar
cal = future_calendar(start, periods)
if base_calendar is None:
base_calendar = cal
mask = (
cal["weekday"].isin(event["weekdays"]).to_numpy()
if event
else None
)
frame = future_frame(
bundle=bundle,
sensor=sensor,
start=start,
periods=periods,
weather_forecast=forecast,
temp_delta=cfg["temp_delta_c"],
precip_mm=cfg["precip_mm"],
weather_mask=mask,
)
pred = predict(bundle, frame) * event_multiplier(frame, cfg)
pts = points(frame, pred, bundle, sensor)
out[sensor]["7d"][name] = pts
out[sensor]["48h"][name] = pts[: config.HORIZON_STEPS["48h"]]
return {
"data_version": bundle["metadata"]["data_version"],
"model_version": bundle["metadata"]["model_version"],
"context_version": bundle["metadata"]["context_version"],
"generated_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
"forecast_start": start.strftime("%Y-%m-%dT%H:%M:%SZ"),
"forecasts": out,
}
def main() -> None:
run_train()
if __name__ == "__main__":
main()