feat: add pipeline and training entrypoints
This commit is contained in:
1 parent
8a81af771e
commit
68402803b2
2 files changed
+465
No files matched your search
@@ -0,0 +1,196 @@
|
|||||||
|
"""`make data`: parse CSV, aggregate hourly, join weather/holidays, export JSON."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pandas as pd
|
||||||
|
|
||||||
|
from . import config
|
||||||
|
from .features import build_features, to_hourly
|
||||||
|
from .parse import clean, load_raw, write_audit
|
||||||
|
from .weather import ensure_weather
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
def rfc3339(ts) -> str:
|
||||||
|
return pd.Timestamp(ts).tz_convert("UTC").strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||||
|
|
||||||
|
|
||||||
|
def minute_iso(value: str | None) -> str | None:
|
||||||
|
if not value:
|
||||||
|
return None
|
||||||
|
return pd.Timestamp(value).tz_convert("UTC").floor("min").strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||||
|
|
||||||
|
|
||||||
|
def _series_points(frame: pd.DataFrame) -> list[dict]:
|
||||||
|
return [
|
||||||
|
{"t": rfc3339(t), "v": round(float(v), 2)}
|
||||||
|
for t, v in zip(frame["t"], frame["v"], strict=True)
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def export_sensors(density: pd.DataFrame, battery: pd.DataFrame) -> list[dict]:
|
||||||
|
multi = density[density["time"] >= pd.Timestamp(config.MULTI_SENSOR_START)]
|
||||||
|
battery_latest = (
|
||||||
|
battery.sort_values("time").groupby("sensor", observed=True).tail(1).set_index("sensor")
|
||||||
|
)
|
||||||
|
out = []
|
||||||
|
for sensor in sorted(config.USABLE_SENSORS):
|
||||||
|
rows = multi[multi["sensor"] == sensor]
|
||||||
|
last = density[density["sensor"] == sensor]["time"].max()
|
||||||
|
out.append(
|
||||||
|
{
|
||||||
|
"id": sensor,
|
||||||
|
"name": config.DISPLAY_NAMES[sensor],
|
||||||
|
"lat": round(float(rows["latitude"].median()), 6),
|
||||||
|
"lon": round(float(rows["longitude"].median()), 6),
|
||||||
|
"mean": round(float(rows["value"].mean()), 1),
|
||||||
|
"p95": round(float(rows["value"].quantile(0.95)), 1),
|
||||||
|
"battery": (
|
||||||
|
int(round(float(battery_latest.loc[sensor, "value"])))
|
||||||
|
if sensor in battery_latest.index
|
||||||
|
else None
|
||||||
|
),
|
||||||
|
"last_seen": rfc3339(last),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def export_series(hourly: pd.DataFrame, data_version: str) -> dict:
|
||||||
|
artifacts = config.artifacts_dir()
|
||||||
|
series_dir = artifacts / "series"
|
||||||
|
series_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
per_sensor: dict[str, list[dict]] = {}
|
||||||
|
for sensor in sorted(config.USABLE_SENSORS):
|
||||||
|
frame = hourly[hourly["sensor"] == sensor]
|
||||||
|
pts = _series_points(frame)
|
||||||
|
per_sensor[sensor] = pts
|
||||||
|
(series_dir / f"{sensor}.json").write_text(
|
||||||
|
json.dumps(
|
||||||
|
{"sensor": sensor, "step": "1h", "points": pts}, ensure_ascii=False
|
||||||
|
)
|
||||||
|
+ "\n"
|
||||||
|
)
|
||||||
|
payload = {"step": "1h", "data_version": data_version, "series": per_sensor}
|
||||||
|
(artifacts / "series.json").write_text(json.dumps(payload, ensure_ascii=False) + "\n")
|
||||||
|
return payload
|
||||||
|
|
||||||
|
|
||||||
|
def _weather_now(forecast: pd.DataFrame, meta: dict) -> dict:
|
||||||
|
current = meta.get("current") or {}
|
||||||
|
now_t = current.get("t")
|
||||||
|
if now_t:
|
||||||
|
ts = pd.Timestamp(now_t, tz="UTC") if pd.Timestamp(now_t).tz is None else pd.Timestamp(now_t).tz_convert("UTC")
|
||||||
|
else:
|
||||||
|
ts = pd.Timestamp(meta["fetched_at"]).tz_convert("UTC")
|
||||||
|
row = forecast.iloc[(forecast["time"] - ts).abs().argmin()]
|
||||||
|
return {
|
||||||
|
"t": rfc3339(ts),
|
||||||
|
"temp_c": round(float(current.get("temp_c", row["temp_c"])), 1),
|
||||||
|
"precip_mm": round(float(current.get("precip_mm", row["precip_mm"])), 2),
|
||||||
|
"cloud_pct": int(round(float(current.get("cloud_pct", row["cloud_pct"])))),
|
||||||
|
"wind_kmh": round(float(current.get("wind_kmh", row["wind_kmh"])), 1),
|
||||||
|
"humidity_pct": int(round(float(current.get("humidity_pct", row["humidity_pct"])))),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def export_context(forecast: pd.DataFrame, meta: dict) -> dict:
|
||||||
|
now = _weather_now(forecast, meta)
|
||||||
|
start = pd.Timestamp(now["t"]).floor("h")
|
||||||
|
window = forecast[(forecast["time"] >= start) & (forecast["time"] < start + pd.Timedelta(days=7))]
|
||||||
|
points = [
|
||||||
|
{
|
||||||
|
"t": rfc3339(row.time),
|
||||||
|
"temp_c": round(float(row.temp_c), 1),
|
||||||
|
"precip_mm": round(float(row.precip_mm), 2),
|
||||||
|
"cloud_pct": int(round(float(row.cloud_pct))),
|
||||||
|
"wind_kmh": round(float(row.wind_kmh), 1),
|
||||||
|
"humidity_pct": int(round(float(row.humidity_pct))),
|
||||||
|
}
|
||||||
|
for row in window.itertuples(index=False)
|
||||||
|
]
|
||||||
|
holidays = [
|
||||||
|
{"date": date, "name": name} for date, name in sorted(config.HOLIDAYS.items())
|
||||||
|
]
|
||||||
|
payload = {
|
||||||
|
"weather": {"now": now, "forecast": points},
|
||||||
|
"holidays": holidays,
|
||||||
|
"updated_at": meta["fetched_at"],
|
||||||
|
"cached": False,
|
||||||
|
}
|
||||||
|
(config.artifacts_dir() / "context.json").write_text(
|
||||||
|
json.dumps(payload, indent=2, ensure_ascii=False) + "\n"
|
||||||
|
)
|
||||||
|
return payload
|
||||||
|
|
||||||
|
|
||||||
|
def run_data(*, refresh_weather: bool = False) -> dict:
|
||||||
|
logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s")
|
||||||
|
artifacts = config.artifacts_dir()
|
||||||
|
cache = config.cache_dir()
|
||||||
|
artifacts.mkdir(parents=True, exist_ok=True)
|
||||||
|
cache.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
|
raw = load_raw()
|
||||||
|
density, battery, audit = clean(raw)
|
||||||
|
write_audit(audit)
|
||||||
|
density[["time", "sensor", "sensor_id", "value"]].to_parquet(
|
||||||
|
cache / "density_clean.parquet", index=False
|
||||||
|
)
|
||||||
|
battery[["time", "sensor", "sensor_id", "value"]].to_parquet(
|
||||||
|
cache / "battery_clean.parquet", index=False
|
||||||
|
)
|
||||||
|
log.info(
|
||||||
|
"parsed %s rows; %s usable sensors; %s density rows",
|
||||||
|
audit["raw_rows"],
|
||||||
|
audit["usable_sensors"],
|
||||||
|
len(density),
|
||||||
|
)
|
||||||
|
|
||||||
|
hourly = to_hourly(density)
|
||||||
|
hourly.to_parquet(cache / "hourly.parquet", index=False)
|
||||||
|
|
||||||
|
archive, forecast, meta = ensure_weather(
|
||||||
|
density["time"].min(), density["time"].max(), refresh=refresh_weather
|
||||||
|
)
|
||||||
|
features = build_features(hourly, archive)
|
||||||
|
features.to_parquet(cache / "hourly_features.parquet", index=False)
|
||||||
|
log.info("built %s hourly feature rows", len(features))
|
||||||
|
|
||||||
|
data_version = minute_iso(audit["time_range"]["to"])
|
||||||
|
sensors = export_sensors(density, battery)
|
||||||
|
(artifacts / "sensors.json").write_text(
|
||||||
|
json.dumps(sensors, indent=2, ensure_ascii=False) + "\n"
|
||||||
|
)
|
||||||
|
series = export_series(hourly, data_version)
|
||||||
|
context = export_context(forecast, meta)
|
||||||
|
|
||||||
|
summary = {
|
||||||
|
"sensors": len(sensors),
|
||||||
|
"series_sensors": len(series["series"]),
|
||||||
|
"hourly_rows": int(len(hourly)),
|
||||||
|
"feature_rows": int(len(features)),
|
||||||
|
"weather_rows": int(len(archive)),
|
||||||
|
"context_forecast_hours": len(context["weather"]["forecast"]),
|
||||||
|
"data_version": data_version,
|
||||||
|
"audit_ok": audit["all_checks_ok"],
|
||||||
|
}
|
||||||
|
log.info("data artifacts written to %s: %s", artifacts, json.dumps(summary))
|
||||||
|
return summary
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> None:
|
||||||
|
parser = argparse.ArgumentParser(description=__doc__)
|
||||||
|
parser.add_argument("--refresh-weather", action="store_true")
|
||||||
|
args = parser.parse_args()
|
||||||
|
run_data(refresh_weather=args.refresh_weather)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,269 @@
|
|||||||
|
"""`make train`: fit the global model, evaluate, export metrics/forecasts/model."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
|
||||||
|
import joblib
|
||||||
|
import numpy as np
|
||||||
|
import pandas as pd
|
||||||
|
|
||||||
|
from . import config
|
||||||
|
from .features import seasonal_lookup
|
||||||
|
from .model import (
|
||||||
|
event_multiplier,
|
||||||
|
fit,
|
||||||
|
future_frame,
|
||||||
|
metrics,
|
||||||
|
points,
|
||||||
|
predict,
|
||||||
|
residual_quantiles,
|
||||||
|
weather_climatology,
|
||||||
|
)
|
||||||
|
from .pipeline import run_data
|
||||||
|
from .weather import load_cached_weather
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
MODEL_SPECS = [
|
||||||
|
("seasonal_naive", None, "same local hour last week"),
|
||||||
|
(
|
||||||
|
"day_no_weather",
|
||||||
|
config.FEATURE_DAY_NO_HOLIDAY,
|
||||||
|
"calendar + lags 24/48/168 + 24h/168h rolling means",
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"day_holiday",
|
||||||
|
config.FEATURE_DAY,
|
||||||
|
"day-ahead + Baden-Württemberg holiday flag",
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"day_weather",
|
||||||
|
config.FEATURE_DAY_WEATHER,
|
||||||
|
"day-ahead + temperature + rain (observed weather)",
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"week_no_weather",
|
||||||
|
config.FEATURE_WEEK_NO_WEATHER,
|
||||||
|
"7-day-safe calendar + previous same weekday/hour values",
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"week_weather",
|
||||||
|
config.FEATURE_WEEK_WEATHER,
|
||||||
|
"7-day-safe + temperature + rain (production model)",
|
||||||
|
),
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def _load_inputs() -> tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame, pd.DataFrame, dict, dict]:
|
||||||
|
cache = config.cache_dir()
|
||||||
|
if not (cache / "hourly_features.parquet").exists():
|
||||||
|
log.info("feature cache missing; running the data pipeline")
|
||||||
|
run_data()
|
||||||
|
hourly = pd.read_parquet(cache / "hourly.parquet")
|
||||||
|
features = pd.read_parquet(cache / "hourly_features.parquet")
|
||||||
|
archive, forecast, meta = load_cached_weather()
|
||||||
|
audit = json.loads((config.artifacts_dir() / "data_audit.json").read_text())
|
||||||
|
return hourly, features, archive, forecast, meta, audit
|
||||||
|
|
||||||
|
|
||||||
|
def _round_row(row: dict) -> dict:
|
||||||
|
return {
|
||||||
|
"name": row["name"],
|
||||||
|
"description": row["description"],
|
||||||
|
"mae": round(row["mae"], 2),
|
||||||
|
"wape": round(row["wape"], 1),
|
||||||
|
"r2": round(row["r2"], 3),
|
||||||
|
"n": row["n"],
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def run_train() -> dict:
|
||||||
|
logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s")
|
||||||
|
artifacts = config.artifacts_dir()
|
||||||
|
artifacts.mkdir(parents=True, exist_ok=True)
|
||||||
|
hourly, features, archive, forecast, meta, audit = _load_inputs()
|
||||||
|
|
||||||
|
data_end = hourly["t"].max()
|
||||||
|
test_start = data_end.floor("h") - pd.Timedelta(days=config.TEST_DAYS)
|
||||||
|
train = features[features["t"] < test_start].copy()
|
||||||
|
test = features[features["t"] >= test_start].copy()
|
||||||
|
eval_frame = test[test["lag_week"].notna() & test["y"].notna()].copy()
|
||||||
|
if eval_frame.empty:
|
||||||
|
raise RuntimeError("holdout has no rows with a seasonal-naive baseline")
|
||||||
|
log.info(
|
||||||
|
"split: train=%s rows (%s..%s), test=%s rows, eval=%s rows (%s..%s)",
|
||||||
|
len(train),
|
||||||
|
train["t"].min(),
|
||||||
|
train["t"].max(),
|
||||||
|
len(test),
|
||||||
|
len(eval_frame),
|
||||||
|
eval_frame["t"].min(),
|
||||||
|
eval_frame["t"].max(),
|
||||||
|
)
|
||||||
|
|
||||||
|
results = []
|
||||||
|
preds: dict[str, np.ndarray] = {}
|
||||||
|
for name, cols, description in MODEL_SPECS:
|
||||||
|
if name == "seasonal_naive":
|
||||||
|
pred = eval_frame["lag_week"].to_numpy(dtype=float)
|
||||||
|
else:
|
||||||
|
assert cols is not None
|
||||||
|
model = fit(train, cols)
|
||||||
|
pred = np.clip(model.predict(eval_frame[cols]), 0.0, None)
|
||||||
|
preds[name] = pred
|
||||||
|
m = metrics(eval_frame["y"], pred)
|
||||||
|
results.append({"name": name, "description": description, **m})
|
||||||
|
log.info("%-16s MAE %.2f WAPE %.1f%% R2 %.3f", name, m["mae"], m["wape"], m["r2"])
|
||||||
|
|
||||||
|
by_name = {r["name"]: r for r in results}
|
||||||
|
baseline = by_name["seasonal_naive"]
|
||||||
|
headline = by_name["week_weather"]
|
||||||
|
|
||||||
|
# Production model: refit on all observed rows after the honest holdout.
|
||||||
|
production = fit(features, config.FEATURE_WEEK_WEATHER)
|
||||||
|
residual = residual_quantiles(eval_frame, preds["week_weather"])
|
||||||
|
sensor_mean = (
|
||||||
|
hourly.groupby("sensor", observed=True)["v"].mean().round(4).to_dict()
|
||||||
|
)
|
||||||
|
trained_at = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||||
|
model_version = f"histgb-{trained_at[:10]}"
|
||||||
|
data_version = pd.Timestamp(audit["time_range"]["to"]).floor("min").strftime(
|
||||||
|
"%Y-%m-%dT%H:%M:%SZ"
|
||||||
|
)
|
||||||
|
context_version = meta["fetched_at"]
|
||||||
|
|
||||||
|
bundle = {
|
||||||
|
"model": production,
|
||||||
|
"features": list(config.FEATURE_WEEK_WEATHER),
|
||||||
|
"seasonal_lookup": seasonal_lookup(hourly),
|
||||||
|
"sensor_mean": sensor_mean,
|
||||||
|
"residual_quantiles": residual,
|
||||||
|
"weather_climatology": weather_climatology(archive),
|
||||||
|
"metadata": {
|
||||||
|
"model_version": model_version,
|
||||||
|
"data_version": data_version,
|
||||||
|
"context_version": context_version,
|
||||||
|
"trained_at": trained_at,
|
||||||
|
"train_from": train["t"].min().strftime("%Y-%m-%dT%H:%M:%SZ"),
|
||||||
|
"train_to": features["t"].max().strftime("%Y-%m-%dT%H:%M:%SZ"),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
joblib.dump(bundle, artifacts / "model.joblib")
|
||||||
|
|
||||||
|
holiday_effect = {
|
||||||
|
"mae_delta": round(
|
||||||
|
by_name["day_holiday"]["mae"] - by_name["day_no_weather"]["mae"], 2
|
||||||
|
),
|
||||||
|
"note": (
|
||||||
|
"Holiday flag alone changes MAE by this amount on the holdout; "
|
||||||
|
"weather is the main driver, holidays are context."
|
||||||
|
),
|
||||||
|
}
|
||||||
|
|
||||||
|
metrics_payload = {
|
||||||
|
"data_version": data_version,
|
||||||
|
"model_version": model_version,
|
||||||
|
"context_version": context_version,
|
||||||
|
"coverage": {
|
||||||
|
"sensors": int(audit["usable_sensors"]),
|
||||||
|
"from": config.MULTI_SENSOR_START,
|
||||||
|
"to": data_version,
|
||||||
|
"cadence_min": int(round(audit["cadence_median_min"])),
|
||||||
|
},
|
||||||
|
"metrics": {
|
||||||
|
"model": "weather-7d",
|
||||||
|
"mae": round(headline["mae"], 2),
|
||||||
|
"wape": round(headline["wape"], 1),
|
||||||
|
"r2": round(headline["r2"], 3),
|
||||||
|
"baseline": "seasonal_naive",
|
||||||
|
"baseline_mae": round(baseline["mae"], 2),
|
||||||
|
"baseline_wape": round(baseline["wape"], 1),
|
||||||
|
"baseline_r2": round(baseline["r2"], 3),
|
||||||
|
"n_eval": int(headline["n"]),
|
||||||
|
},
|
||||||
|
"trained_at": trained_at,
|
||||||
|
"train_from": bundle["metadata"]["train_from"],
|
||||||
|
"train_to": bundle["metadata"]["train_to"],
|
||||||
|
"test_from": test_start.strftime("%Y-%m-%dT%H:%M:%SZ"),
|
||||||
|
"test_to": data_end.strftime("%Y-%m-%dT%H:%M:%SZ"),
|
||||||
|
"features": list(config.FEATURE_WEEK_WEATHER),
|
||||||
|
"holiday_effect": holiday_effect,
|
||||||
|
"ablation": [_round_row(r) for r in results],
|
||||||
|
"limits": list(config.LIMITS),
|
||||||
|
}
|
||||||
|
(artifacts / "metrics.json").write_text(
|
||||||
|
json.dumps(metrics_payload, indent=2, ensure_ascii=False) + "\n"
|
||||||
|
)
|
||||||
|
|
||||||
|
start = pd.Timestamp.now(tz="UTC").ceil("h")
|
||||||
|
forecasts = _export_forecasts(bundle, forecast, start)
|
||||||
|
(artifacts / "forecasts.json").write_text(
|
||||||
|
json.dumps(forecasts, ensure_ascii=False) + "\n"
|
||||||
|
)
|
||||||
|
log.info(
|
||||||
|
"wrote metrics.json, forecasts.json (%s..%s) and model.joblib",
|
||||||
|
start,
|
||||||
|
start + pd.Timedelta(hours=167),
|
||||||
|
)
|
||||||
|
return {
|
||||||
|
"headline": metrics_payload["metrics"],
|
||||||
|
"ablation": metrics_payload["ablation"],
|
||||||
|
"model_version": model_version,
|
||||||
|
"forecast_start": start.strftime("%Y-%m-%dT%H:%M:%SZ"),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _export_forecasts(bundle: dict, forecast: pd.DataFrame, start: pd.Timestamp) -> dict:
|
||||||
|
periods = config.HORIZON_STEPS["7d"]
|
||||||
|
sensors = sorted(config.USABLE_SENSORS)
|
||||||
|
out: dict[str, dict[str, dict[str, list[dict]]]] = {
|
||||||
|
sensor: {"48h": {}, "7d": {}} for sensor in sensors
|
||||||
|
}
|
||||||
|
base_calendar = None
|
||||||
|
for name, cfg in config.SCENARIOS.items():
|
||||||
|
event = cfg.get("event")
|
||||||
|
for sensor in sensors:
|
||||||
|
from .features import future_calendar
|
||||||
|
|
||||||
|
cal = future_calendar(start, periods)
|
||||||
|
if base_calendar is None:
|
||||||
|
base_calendar = cal
|
||||||
|
mask = (
|
||||||
|
cal["weekday"].isin(event["weekdays"]).to_numpy()
|
||||||
|
if event
|
||||||
|
else None
|
||||||
|
)
|
||||||
|
frame = future_frame(
|
||||||
|
bundle=bundle,
|
||||||
|
sensor=sensor,
|
||||||
|
start=start,
|
||||||
|
periods=periods,
|
||||||
|
weather_forecast=forecast,
|
||||||
|
temp_delta=cfg["temp_delta_c"],
|
||||||
|
precip_mm=cfg["precip_mm"],
|
||||||
|
weather_mask=mask,
|
||||||
|
)
|
||||||
|
pred = predict(bundle, frame) * event_multiplier(frame, cfg)
|
||||||
|
pts = points(frame, pred, bundle, sensor)
|
||||||
|
out[sensor]["7d"][name] = pts
|
||||||
|
out[sensor]["48h"][name] = pts[: config.HORIZON_STEPS["48h"]]
|
||||||
|
|
||||||
|
return {
|
||||||
|
"data_version": bundle["metadata"]["data_version"],
|
||||||
|
"model_version": bundle["metadata"]["model_version"],
|
||||||
|
"context_version": bundle["metadata"]["context_version"],
|
||||||
|
"generated_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
|
||||||
|
"forecast_start": start.strftime("%Y-%m-%dT%H:%M:%SZ"),
|
||||||
|
"forecasts": out,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> None:
|
||||||
|
run_train()
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
Reference in new issue
Block a user