feat: ml bootstrap #1
No files matched your search
@@ -0,0 +1,196 @@
|
||||
"""`make data`: parse CSV, aggregate hourly, join weather/holidays, export JSON."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
import pandas as pd
|
||||
|
||||
from . import config
|
||||
from .features import build_features, to_hourly
|
||||
from .parse import clean, load_raw, write_audit
|
||||
from .weather import ensure_weather
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def rfc3339(ts) -> str:
|
||||
return pd.Timestamp(ts).tz_convert("UTC").strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
|
||||
|
||||
def minute_iso(value: str | None) -> str | None:
|
||||
if not value:
|
||||
return None
|
||||
return pd.Timestamp(value).tz_convert("UTC").floor("min").strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
|
||||
|
||||
def _series_points(frame: pd.DataFrame) -> list[dict]:
|
||||
return [
|
||||
{"t": rfc3339(t), "v": round(float(v), 2)}
|
||||
for t, v in zip(frame["t"], frame["v"], strict=True)
|
||||
]
|
||||
|
||||
|
||||
def export_sensors(density: pd.DataFrame, battery: pd.DataFrame) -> list[dict]:
|
||||
multi = density[density["time"] >= pd.Timestamp(config.MULTI_SENSOR_START)]
|
||||
battery_latest = (
|
||||
battery.sort_values("time").groupby("sensor", observed=True).tail(1).set_index("sensor")
|
||||
)
|
||||
out = []
|
||||
for sensor in sorted(config.USABLE_SENSORS):
|
||||
rows = multi[multi["sensor"] == sensor]
|
||||
last = density[density["sensor"] == sensor]["time"].max()
|
||||
out.append(
|
||||
{
|
||||
"id": sensor,
|
||||
"name": config.DISPLAY_NAMES[sensor],
|
||||
"lat": round(float(rows["latitude"].median()), 6),
|
||||
"lon": round(float(rows["longitude"].median()), 6),
|
||||
"mean": round(float(rows["value"].mean()), 1),
|
||||
"p95": round(float(rows["value"].quantile(0.95)), 1),
|
||||
"battery": (
|
||||
int(round(float(battery_latest.loc[sensor, "value"])))
|
||||
if sensor in battery_latest.index
|
||||
else None
|
||||
),
|
||||
"last_seen": rfc3339(last),
|
||||
}
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def export_series(hourly: pd.DataFrame, data_version: str) -> dict:
|
||||
artifacts = config.artifacts_dir()
|
||||
series_dir = artifacts / "series"
|
||||
series_dir.mkdir(parents=True, exist_ok=True)
|
||||
per_sensor: dict[str, list[dict]] = {}
|
||||
for sensor in sorted(config.USABLE_SENSORS):
|
||||
frame = hourly[hourly["sensor"] == sensor]
|
||||
pts = _series_points(frame)
|
||||
per_sensor[sensor] = pts
|
||||
(series_dir / f"{sensor}.json").write_text(
|
||||
json.dumps(
|
||||
{"sensor": sensor, "step": "1h", "points": pts}, ensure_ascii=False
|
||||
)
|
||||
+ "\n"
|
||||
)
|
||||
payload = {"step": "1h", "data_version": data_version, "series": per_sensor}
|
||||
(artifacts / "series.json").write_text(json.dumps(payload, ensure_ascii=False) + "\n")
|
||||
return payload
|
||||
|
||||
|
||||
def _weather_now(forecast: pd.DataFrame, meta: dict) -> dict:
|
||||
current = meta.get("current") or {}
|
||||
now_t = current.get("t")
|
||||
if now_t:
|
||||
ts = pd.Timestamp(now_t, tz="UTC") if pd.Timestamp(now_t).tz is None else pd.Timestamp(now_t).tz_convert("UTC")
|
||||
else:
|
||||
ts = pd.Timestamp(meta["fetched_at"]).tz_convert("UTC")
|
||||
row = forecast.iloc[(forecast["time"] - ts).abs().argmin()]
|
||||
return {
|
||||
"t": rfc3339(ts),
|
||||
"temp_c": round(float(current.get("temp_c", row["temp_c"])), 1),
|
||||
"precip_mm": round(float(current.get("precip_mm", row["precip_mm"])), 2),
|
||||
"cloud_pct": int(round(float(current.get("cloud_pct", row["cloud_pct"])))),
|
||||
"wind_kmh": round(float(current.get("wind_kmh", row["wind_kmh"])), 1),
|
||||
"humidity_pct": int(round(float(current.get("humidity_pct", row["humidity_pct"])))),
|
||||
}
|
||||
|
||||
|
||||
def export_context(forecast: pd.DataFrame, meta: dict) -> dict:
|
||||
now = _weather_now(forecast, meta)
|
||||
start = pd.Timestamp(now["t"]).floor("h")
|
||||
window = forecast[(forecast["time"] >= start) & (forecast["time"] < start + pd.Timedelta(days=7))]
|
||||
points = [
|
||||
{
|
||||
"t": rfc3339(row.time),
|
||||
"temp_c": round(float(row.temp_c), 1),
|
||||
"precip_mm": round(float(row.precip_mm), 2),
|
||||
"cloud_pct": int(round(float(row.cloud_pct))),
|
||||
"wind_kmh": round(float(row.wind_kmh), 1),
|
||||
"humidity_pct": int(round(float(row.humidity_pct))),
|
||||
}
|
||||
for row in window.itertuples(index=False)
|
||||
]
|
||||
holidays = [
|
||||
{"date": date, "name": name} for date, name in sorted(config.HOLIDAYS.items())
|
||||
]
|
||||
payload = {
|
||||
"weather": {"now": now, "forecast": points},
|
||||
"holidays": holidays,
|
||||
"updated_at": meta["fetched_at"],
|
||||
"cached": False,
|
||||
}
|
||||
(config.artifacts_dir() / "context.json").write_text(
|
||||
json.dumps(payload, indent=2, ensure_ascii=False) + "\n"
|
||||
)
|
||||
return payload
|
||||
|
||||
|
||||
def run_data(*, refresh_weather: bool = False) -> dict:
|
||||
logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s")
|
||||
artifacts = config.artifacts_dir()
|
||||
cache = config.cache_dir()
|
||||
artifacts.mkdir(parents=True, exist_ok=True)
|
||||
cache.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
raw = load_raw()
|
||||
density, battery, audit = clean(raw)
|
||||
write_audit(audit)
|
||||
density[["time", "sensor", "sensor_id", "value"]].to_parquet(
|
||||
cache / "density_clean.parquet", index=False
|
||||
)
|
||||
battery[["time", "sensor", "sensor_id", "value"]].to_parquet(
|
||||
cache / "battery_clean.parquet", index=False
|
||||
)
|
||||
log.info(
|
||||
"parsed %s rows; %s usable sensors; %s density rows",
|
||||
audit["raw_rows"],
|
||||
audit["usable_sensors"],
|
||||
len(density),
|
||||
)
|
||||
|
||||
hourly = to_hourly(density)
|
||||
hourly.to_parquet(cache / "hourly.parquet", index=False)
|
||||
|
||||
archive, forecast, meta = ensure_weather(
|
||||
density["time"].min(), density["time"].max(), refresh=refresh_weather
|
||||
)
|
||||
features = build_features(hourly, archive)
|
||||
features.to_parquet(cache / "hourly_features.parquet", index=False)
|
||||
log.info("built %s hourly feature rows", len(features))
|
||||
|
||||
data_version = minute_iso(audit["time_range"]["to"])
|
||||
sensors = export_sensors(density, battery)
|
||||
(artifacts / "sensors.json").write_text(
|
||||
json.dumps(sensors, indent=2, ensure_ascii=False) + "\n"
|
||||
)
|
||||
series = export_series(hourly, data_version)
|
||||
context = export_context(forecast, meta)
|
||||
|
||||
summary = {
|
||||
"sensors": len(sensors),
|
||||
"series_sensors": len(series["series"]),
|
||||
"hourly_rows": int(len(hourly)),
|
||||
"feature_rows": int(len(features)),
|
||||
"weather_rows": int(len(archive)),
|
||||
"context_forecast_hours": len(context["weather"]["forecast"]),
|
||||
"data_version": data_version,
|
||||
"audit_ok": audit["all_checks_ok"],
|
||||
}
|
||||
log.info("data artifacts written to %s: %s", artifacts, json.dumps(summary))
|
||||
return summary
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--refresh-weather", action="store_true")
|
||||
args = parser.parse_args()
|
||||
run_data(refresh_weather=args.refresh_weather)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,269 @@
|
||||
"""`make train`: fit the global model, evaluate, export metrics/forecasts/model."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
|
||||
import joblib
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
from . import config
|
||||
from .features import seasonal_lookup
|
||||
from .model import (
|
||||
event_multiplier,
|
||||
fit,
|
||||
future_frame,
|
||||
metrics,
|
||||
points,
|
||||
predict,
|
||||
residual_quantiles,
|
||||
weather_climatology,
|
||||
)
|
||||
from .pipeline import run_data
|
||||
from .weather import load_cached_weather
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
MODEL_SPECS = [
|
||||
("seasonal_naive", None, "same local hour last week"),
|
||||
(
|
||||
"day_no_weather",
|
||||
config.FEATURE_DAY_NO_HOLIDAY,
|
||||
"calendar + lags 24/48/168 + 24h/168h rolling means",
|
||||
),
|
||||
(
|
||||
"day_holiday",
|
||||
config.FEATURE_DAY,
|
||||
"day-ahead + Baden-Württemberg holiday flag",
|
||||
),
|
||||
(
|
||||
"day_weather",
|
||||
config.FEATURE_DAY_WEATHER,
|
||||
"day-ahead + temperature + rain (observed weather)",
|
||||
),
|
||||
(
|
||||
"week_no_weather",
|
||||
config.FEATURE_WEEK_NO_WEATHER,
|
||||
"7-day-safe calendar + previous same weekday/hour values",
|
||||
),
|
||||
(
|
||||
"week_weather",
|
||||
config.FEATURE_WEEK_WEATHER,
|
||||
"7-day-safe + temperature + rain (production model)",
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def _load_inputs() -> tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame, pd.DataFrame, dict, dict]:
|
||||
cache = config.cache_dir()
|
||||
if not (cache / "hourly_features.parquet").exists():
|
||||
log.info("feature cache missing; running the data pipeline")
|
||||
run_data()
|
||||
hourly = pd.read_parquet(cache / "hourly.parquet")
|
||||
features = pd.read_parquet(cache / "hourly_features.parquet")
|
||||
archive, forecast, meta = load_cached_weather()
|
||||
audit = json.loads((config.artifacts_dir() / "data_audit.json").read_text())
|
||||
return hourly, features, archive, forecast, meta, audit
|
||||
|
||||
|
||||
def _round_row(row: dict) -> dict:
|
||||
return {
|
||||
"name": row["name"],
|
||||
"description": row["description"],
|
||||
"mae": round(row["mae"], 2),
|
||||
"wape": round(row["wape"], 1),
|
||||
"r2": round(row["r2"], 3),
|
||||
"n": row["n"],
|
||||
}
|
||||
|
||||
|
||||
def run_train() -> dict:
|
||||
logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s")
|
||||
artifacts = config.artifacts_dir()
|
||||
artifacts.mkdir(parents=True, exist_ok=True)
|
||||
hourly, features, archive, forecast, meta, audit = _load_inputs()
|
||||
|
||||
data_end = hourly["t"].max()
|
||||
test_start = data_end.floor("h") - pd.Timedelta(days=config.TEST_DAYS)
|
||||
train = features[features["t"] < test_start].copy()
|
||||
test = features[features["t"] >= test_start].copy()
|
||||
eval_frame = test[test["lag_week"].notna() & test["y"].notna()].copy()
|
||||
if eval_frame.empty:
|
||||
raise RuntimeError("holdout has no rows with a seasonal-naive baseline")
|
||||
log.info(
|
||||
"split: train=%s rows (%s..%s), test=%s rows, eval=%s rows (%s..%s)",
|
||||
len(train),
|
||||
train["t"].min(),
|
||||
train["t"].max(),
|
||||
len(test),
|
||||
len(eval_frame),
|
||||
eval_frame["t"].min(),
|
||||
eval_frame["t"].max(),
|
||||
)
|
||||
|
||||
results = []
|
||||
preds: dict[str, np.ndarray] = {}
|
||||
for name, cols, description in MODEL_SPECS:
|
||||
if name == "seasonal_naive":
|
||||
pred = eval_frame["lag_week"].to_numpy(dtype=float)
|
||||
else:
|
||||
assert cols is not None
|
||||
model = fit(train, cols)
|
||||
pred = np.clip(model.predict(eval_frame[cols]), 0.0, None)
|
||||
preds[name] = pred
|
||||
m = metrics(eval_frame["y"], pred)
|
||||
results.append({"name": name, "description": description, **m})
|
||||
log.info("%-16s MAE %.2f WAPE %.1f%% R2 %.3f", name, m["mae"], m["wape"], m["r2"])
|
||||
|
||||
by_name = {r["name"]: r for r in results}
|
||||
baseline = by_name["seasonal_naive"]
|
||||
headline = by_name["week_weather"]
|
||||
|
||||
# Production model: refit on all observed rows after the honest holdout.
|
||||
production = fit(features, config.FEATURE_WEEK_WEATHER)
|
||||
residual = residual_quantiles(eval_frame, preds["week_weather"])
|
||||
sensor_mean = (
|
||||
hourly.groupby("sensor", observed=True)["v"].mean().round(4).to_dict()
|
||||
)
|
||||
trained_at = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
model_version = f"histgb-{trained_at[:10]}"
|
||||
data_version = pd.Timestamp(audit["time_range"]["to"]).floor("min").strftime(
|
||||
"%Y-%m-%dT%H:%M:%SZ"
|
||||
)
|
||||
context_version = meta["fetched_at"]
|
||||
|
||||
bundle = {
|
||||
"model": production,
|
||||
"features": list(config.FEATURE_WEEK_WEATHER),
|
||||
"seasonal_lookup": seasonal_lookup(hourly),
|
||||
"sensor_mean": sensor_mean,
|
||||
"residual_quantiles": residual,
|
||||
"weather_climatology": weather_climatology(archive),
|
||||
"metadata": {
|
||||
"model_version": model_version,
|
||||
"data_version": data_version,
|
||||
"context_version": context_version,
|
||||
"trained_at": trained_at,
|
||||
"train_from": train["t"].min().strftime("%Y-%m-%dT%H:%M:%SZ"),
|
||||
"train_to": features["t"].max().strftime("%Y-%m-%dT%H:%M:%SZ"),
|
||||
},
|
||||
}
|
||||
joblib.dump(bundle, artifacts / "model.joblib")
|
||||
|
||||
holiday_effect = {
|
||||
"mae_delta": round(
|
||||
by_name["day_holiday"]["mae"] - by_name["day_no_weather"]["mae"], 2
|
||||
),
|
||||
"note": (
|
||||
"Holiday flag alone changes MAE by this amount on the holdout; "
|
||||
"weather is the main driver, holidays are context."
|
||||
),
|
||||
}
|
||||
|
||||
metrics_payload = {
|
||||
"data_version": data_version,
|
||||
"model_version": model_version,
|
||||
"context_version": context_version,
|
||||
"coverage": {
|
||||
"sensors": int(audit["usable_sensors"]),
|
||||
"from": config.MULTI_SENSOR_START,
|
||||
"to": data_version,
|
||||
"cadence_min": int(round(audit["cadence_median_min"])),
|
||||
},
|
||||
"metrics": {
|
||||
"model": "weather-7d",
|
||||
"mae": round(headline["mae"], 2),
|
||||
"wape": round(headline["wape"], 1),
|
||||
"r2": round(headline["r2"], 3),
|
||||
"baseline": "seasonal_naive",
|
||||
"baseline_mae": round(baseline["mae"], 2),
|
||||
"baseline_wape": round(baseline["wape"], 1),
|
||||
"baseline_r2": round(baseline["r2"], 3),
|
||||
"n_eval": int(headline["n"]),
|
||||
},
|
||||
"trained_at": trained_at,
|
||||
"train_from": bundle["metadata"]["train_from"],
|
||||
"train_to": bundle["metadata"]["train_to"],
|
||||
"test_from": test_start.strftime("%Y-%m-%dT%H:%M:%SZ"),
|
||||
"test_to": data_end.strftime("%Y-%m-%dT%H:%M:%SZ"),
|
||||
"features": list(config.FEATURE_WEEK_WEATHER),
|
||||
"holiday_effect": holiday_effect,
|
||||
"ablation": [_round_row(r) for r in results],
|
||||
"limits": list(config.LIMITS),
|
||||
}
|
||||
(artifacts / "metrics.json").write_text(
|
||||
json.dumps(metrics_payload, indent=2, ensure_ascii=False) + "\n"
|
||||
)
|
||||
|
||||
start = pd.Timestamp.now(tz="UTC").ceil("h")
|
||||
forecasts = _export_forecasts(bundle, forecast, start)
|
||||
(artifacts / "forecasts.json").write_text(
|
||||
json.dumps(forecasts, ensure_ascii=False) + "\n"
|
||||
)
|
||||
log.info(
|
||||
"wrote metrics.json, forecasts.json (%s..%s) and model.joblib",
|
||||
start,
|
||||
start + pd.Timedelta(hours=167),
|
||||
)
|
||||
return {
|
||||
"headline": metrics_payload["metrics"],
|
||||
"ablation": metrics_payload["ablation"],
|
||||
"model_version": model_version,
|
||||
"forecast_start": start.strftime("%Y-%m-%dT%H:%M:%SZ"),
|
||||
}
|
||||
|
||||
|
||||
def _export_forecasts(bundle: dict, forecast: pd.DataFrame, start: pd.Timestamp) -> dict:
|
||||
periods = config.HORIZON_STEPS["7d"]
|
||||
sensors = sorted(config.USABLE_SENSORS)
|
||||
out: dict[str, dict[str, dict[str, list[dict]]]] = {
|
||||
sensor: {"48h": {}, "7d": {}} for sensor in sensors
|
||||
}
|
||||
base_calendar = None
|
||||
for name, cfg in config.SCENARIOS.items():
|
||||
event = cfg.get("event")
|
||||
for sensor in sensors:
|
||||
from .features import future_calendar
|
||||
|
||||
cal = future_calendar(start, periods)
|
||||
if base_calendar is None:
|
||||
base_calendar = cal
|
||||
mask = (
|
||||
cal["weekday"].isin(event["weekdays"]).to_numpy()
|
||||
if event
|
||||
else None
|
||||
)
|
||||
frame = future_frame(
|
||||
bundle=bundle,
|
||||
sensor=sensor,
|
||||
start=start,
|
||||
periods=periods,
|
||||
weather_forecast=forecast,
|
||||
temp_delta=cfg["temp_delta_c"],
|
||||
precip_mm=cfg["precip_mm"],
|
||||
weather_mask=mask,
|
||||
)
|
||||
pred = predict(bundle, frame) * event_multiplier(frame, cfg)
|
||||
pts = points(frame, pred, bundle, sensor)
|
||||
out[sensor]["7d"][name] = pts
|
||||
out[sensor]["48h"][name] = pts[: config.HORIZON_STEPS["48h"]]
|
||||
|
||||
return {
|
||||
"data_version": bundle["metadata"]["data_version"],
|
||||
"model_version": bundle["metadata"]["model_version"],
|
||||
"context_version": bundle["metadata"]["context_version"],
|
||||
"generated_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
|
||||
"forecast_start": start.strftime("%Y-%m-%dT%H:%M:%SZ"),
|
||||
"forecasts": out,
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
run_train()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in new issue
Block a user