diff --git a/src/pulsecast/pipeline.py b/src/pulsecast/pipeline.py new file mode 100644 index 0000000..14092b1 --- /dev/null +++ b/src/pulsecast/pipeline.py @@ -0,0 +1,196 @@ +"""`make data`: parse CSV, aggregate hourly, join weather/holidays, export JSON.""" + +from __future__ import annotations + +import argparse +import json +import logging +from pathlib import Path + +import pandas as pd + +from . import config +from .features import build_features, to_hourly +from .parse import clean, load_raw, write_audit +from .weather import ensure_weather + +log = logging.getLogger(__name__) + + +def rfc3339(ts) -> str: + return pd.Timestamp(ts).tz_convert("UTC").strftime("%Y-%m-%dT%H:%M:%SZ") + + +def minute_iso(value: str | None) -> str | None: + if not value: + return None + return pd.Timestamp(value).tz_convert("UTC").floor("min").strftime("%Y-%m-%dT%H:%M:%SZ") + + +def _series_points(frame: pd.DataFrame) -> list[dict]: + return [ + {"t": rfc3339(t), "v": round(float(v), 2)} + for t, v in zip(frame["t"], frame["v"], strict=True) + ] + + +def export_sensors(density: pd.DataFrame, battery: pd.DataFrame) -> list[dict]: + multi = density[density["time"] >= pd.Timestamp(config.MULTI_SENSOR_START)] + battery_latest = ( + battery.sort_values("time").groupby("sensor", observed=True).tail(1).set_index("sensor") + ) + out = [] + for sensor in sorted(config.USABLE_SENSORS): + rows = multi[multi["sensor"] == sensor] + last = density[density["sensor"] == sensor]["time"].max() + out.append( + { + "id": sensor, + "name": config.DISPLAY_NAMES[sensor], + "lat": round(float(rows["latitude"].median()), 6), + "lon": round(float(rows["longitude"].median()), 6), + "mean": round(float(rows["value"].mean()), 1), + "p95": round(float(rows["value"].quantile(0.95)), 1), + "battery": ( + int(round(float(battery_latest.loc[sensor, "value"]))) + if sensor in battery_latest.index + else None + ), + "last_seen": rfc3339(last), + } + ) + return out + + +def export_series(hourly: pd.DataFrame, data_version: str) -> dict: + artifacts = config.artifacts_dir() + series_dir = artifacts / "series" + series_dir.mkdir(parents=True, exist_ok=True) + per_sensor: dict[str, list[dict]] = {} + for sensor in sorted(config.USABLE_SENSORS): + frame = hourly[hourly["sensor"] == sensor] + pts = _series_points(frame) + per_sensor[sensor] = pts + (series_dir / f"{sensor}.json").write_text( + json.dumps( + {"sensor": sensor, "step": "1h", "points": pts}, ensure_ascii=False + ) + + "\n" + ) + payload = {"step": "1h", "data_version": data_version, "series": per_sensor} + (artifacts / "series.json").write_text(json.dumps(payload, ensure_ascii=False) + "\n") + return payload + + +def _weather_now(forecast: pd.DataFrame, meta: dict) -> dict: + current = meta.get("current") or {} + now_t = current.get("t") + if now_t: + ts = pd.Timestamp(now_t, tz="UTC") if pd.Timestamp(now_t).tz is None else pd.Timestamp(now_t).tz_convert("UTC") + else: + ts = pd.Timestamp(meta["fetched_at"]).tz_convert("UTC") + row = forecast.iloc[(forecast["time"] - ts).abs().argmin()] + return { + "t": rfc3339(ts), + "temp_c": round(float(current.get("temp_c", row["temp_c"])), 1), + "precip_mm": round(float(current.get("precip_mm", row["precip_mm"])), 2), + "cloud_pct": int(round(float(current.get("cloud_pct", row["cloud_pct"])))), + "wind_kmh": round(float(current.get("wind_kmh", row["wind_kmh"])), 1), + "humidity_pct": int(round(float(current.get("humidity_pct", row["humidity_pct"])))), + } + + +def export_context(forecast: pd.DataFrame, meta: dict) -> dict: + now = _weather_now(forecast, meta) + start = pd.Timestamp(now["t"]).floor("h") + window = forecast[(forecast["time"] >= start) & (forecast["time"] < start + pd.Timedelta(days=7))] + points = [ + { + "t": rfc3339(row.time), + "temp_c": round(float(row.temp_c), 1), + "precip_mm": round(float(row.precip_mm), 2), + "cloud_pct": int(round(float(row.cloud_pct))), + "wind_kmh": round(float(row.wind_kmh), 1), + "humidity_pct": int(round(float(row.humidity_pct))), + } + for row in window.itertuples(index=False) + ] + holidays = [ + {"date": date, "name": name} for date, name in sorted(config.HOLIDAYS.items()) + ] + payload = { + "weather": {"now": now, "forecast": points}, + "holidays": holidays, + "updated_at": meta["fetched_at"], + "cached": False, + } + (config.artifacts_dir() / "context.json").write_text( + json.dumps(payload, indent=2, ensure_ascii=False) + "\n" + ) + return payload + + +def run_data(*, refresh_weather: bool = False) -> dict: + logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s") + artifacts = config.artifacts_dir() + cache = config.cache_dir() + artifacts.mkdir(parents=True, exist_ok=True) + cache.mkdir(parents=True, exist_ok=True) + + raw = load_raw() + density, battery, audit = clean(raw) + write_audit(audit) + density[["time", "sensor", "sensor_id", "value"]].to_parquet( + cache / "density_clean.parquet", index=False + ) + battery[["time", "sensor", "sensor_id", "value"]].to_parquet( + cache / "battery_clean.parquet", index=False + ) + log.info( + "parsed %s rows; %s usable sensors; %s density rows", + audit["raw_rows"], + audit["usable_sensors"], + len(density), + ) + + hourly = to_hourly(density) + hourly.to_parquet(cache / "hourly.parquet", index=False) + + archive, forecast, meta = ensure_weather( + density["time"].min(), density["time"].max(), refresh=refresh_weather + ) + features = build_features(hourly, archive) + features.to_parquet(cache / "hourly_features.parquet", index=False) + log.info("built %s hourly feature rows", len(features)) + + data_version = minute_iso(audit["time_range"]["to"]) + sensors = export_sensors(density, battery) + (artifacts / "sensors.json").write_text( + json.dumps(sensors, indent=2, ensure_ascii=False) + "\n" + ) + series = export_series(hourly, data_version) + context = export_context(forecast, meta) + + summary = { + "sensors": len(sensors), + "series_sensors": len(series["series"]), + "hourly_rows": int(len(hourly)), + "feature_rows": int(len(features)), + "weather_rows": int(len(archive)), + "context_forecast_hours": len(context["weather"]["forecast"]), + "data_version": data_version, + "audit_ok": audit["all_checks_ok"], + } + log.info("data artifacts written to %s: %s", artifacts, json.dumps(summary)) + return summary + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--refresh-weather", action="store_true") + args = parser.parse_args() + run_data(refresh_weather=args.refresh_weather) + + +if __name__ == "__main__": + main() diff --git a/src/pulsecast/train.py b/src/pulsecast/train.py new file mode 100644 index 0000000..36013ca --- /dev/null +++ b/src/pulsecast/train.py @@ -0,0 +1,269 @@ +"""`make train`: fit the global model, evaluate, export metrics/forecasts/model.""" + +from __future__ import annotations + +import json +import logging +from datetime import datetime, timezone + +import joblib +import numpy as np +import pandas as pd + +from . import config +from .features import seasonal_lookup +from .model import ( + event_multiplier, + fit, + future_frame, + metrics, + points, + predict, + residual_quantiles, + weather_climatology, +) +from .pipeline import run_data +from .weather import load_cached_weather + +log = logging.getLogger(__name__) + +MODEL_SPECS = [ + ("seasonal_naive", None, "same local hour last week"), + ( + "day_no_weather", + config.FEATURE_DAY_NO_HOLIDAY, + "calendar + lags 24/48/168 + 24h/168h rolling means", + ), + ( + "day_holiday", + config.FEATURE_DAY, + "day-ahead + Baden-Württemberg holiday flag", + ), + ( + "day_weather", + config.FEATURE_DAY_WEATHER, + "day-ahead + temperature + rain (observed weather)", + ), + ( + "week_no_weather", + config.FEATURE_WEEK_NO_WEATHER, + "7-day-safe calendar + previous same weekday/hour values", + ), + ( + "week_weather", + config.FEATURE_WEEK_WEATHER, + "7-day-safe + temperature + rain (production model)", + ), +] + + +def _load_inputs() -> tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame, pd.DataFrame, dict, dict]: + cache = config.cache_dir() + if not (cache / "hourly_features.parquet").exists(): + log.info("feature cache missing; running the data pipeline") + run_data() + hourly = pd.read_parquet(cache / "hourly.parquet") + features = pd.read_parquet(cache / "hourly_features.parquet") + archive, forecast, meta = load_cached_weather() + audit = json.loads((config.artifacts_dir() / "data_audit.json").read_text()) + return hourly, features, archive, forecast, meta, audit + + +def _round_row(row: dict) -> dict: + return { + "name": row["name"], + "description": row["description"], + "mae": round(row["mae"], 2), + "wape": round(row["wape"], 1), + "r2": round(row["r2"], 3), + "n": row["n"], + } + + +def run_train() -> dict: + logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s") + artifacts = config.artifacts_dir() + artifacts.mkdir(parents=True, exist_ok=True) + hourly, features, archive, forecast, meta, audit = _load_inputs() + + data_end = hourly["t"].max() + test_start = data_end.floor("h") - pd.Timedelta(days=config.TEST_DAYS) + train = features[features["t"] < test_start].copy() + test = features[features["t"] >= test_start].copy() + eval_frame = test[test["lag_week"].notna() & test["y"].notna()].copy() + if eval_frame.empty: + raise RuntimeError("holdout has no rows with a seasonal-naive baseline") + log.info( + "split: train=%s rows (%s..%s), test=%s rows, eval=%s rows (%s..%s)", + len(train), + train["t"].min(), + train["t"].max(), + len(test), + len(eval_frame), + eval_frame["t"].min(), + eval_frame["t"].max(), + ) + + results = [] + preds: dict[str, np.ndarray] = {} + for name, cols, description in MODEL_SPECS: + if name == "seasonal_naive": + pred = eval_frame["lag_week"].to_numpy(dtype=float) + else: + assert cols is not None + model = fit(train, cols) + pred = np.clip(model.predict(eval_frame[cols]), 0.0, None) + preds[name] = pred + m = metrics(eval_frame["y"], pred) + results.append({"name": name, "description": description, **m}) + log.info("%-16s MAE %.2f WAPE %.1f%% R2 %.3f", name, m["mae"], m["wape"], m["r2"]) + + by_name = {r["name"]: r for r in results} + baseline = by_name["seasonal_naive"] + headline = by_name["week_weather"] + + # Production model: refit on all observed rows after the honest holdout. + production = fit(features, config.FEATURE_WEEK_WEATHER) + residual = residual_quantiles(eval_frame, preds["week_weather"]) + sensor_mean = ( + hourly.groupby("sensor", observed=True)["v"].mean().round(4).to_dict() + ) + trained_at = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + model_version = f"histgb-{trained_at[:10]}" + data_version = pd.Timestamp(audit["time_range"]["to"]).floor("min").strftime( + "%Y-%m-%dT%H:%M:%SZ" + ) + context_version = meta["fetched_at"] + + bundle = { + "model": production, + "features": list(config.FEATURE_WEEK_WEATHER), + "seasonal_lookup": seasonal_lookup(hourly), + "sensor_mean": sensor_mean, + "residual_quantiles": residual, + "weather_climatology": weather_climatology(archive), + "metadata": { + "model_version": model_version, + "data_version": data_version, + "context_version": context_version, + "trained_at": trained_at, + "train_from": train["t"].min().strftime("%Y-%m-%dT%H:%M:%SZ"), + "train_to": features["t"].max().strftime("%Y-%m-%dT%H:%M:%SZ"), + }, + } + joblib.dump(bundle, artifacts / "model.joblib") + + holiday_effect = { + "mae_delta": round( + by_name["day_holiday"]["mae"] - by_name["day_no_weather"]["mae"], 2 + ), + "note": ( + "Holiday flag alone changes MAE by this amount on the holdout; " + "weather is the main driver, holidays are context." + ), + } + + metrics_payload = { + "data_version": data_version, + "model_version": model_version, + "context_version": context_version, + "coverage": { + "sensors": int(audit["usable_sensors"]), + "from": config.MULTI_SENSOR_START, + "to": data_version, + "cadence_min": int(round(audit["cadence_median_min"])), + }, + "metrics": { + "model": "weather-7d", + "mae": round(headline["mae"], 2), + "wape": round(headline["wape"], 1), + "r2": round(headline["r2"], 3), + "baseline": "seasonal_naive", + "baseline_mae": round(baseline["mae"], 2), + "baseline_wape": round(baseline["wape"], 1), + "baseline_r2": round(baseline["r2"], 3), + "n_eval": int(headline["n"]), + }, + "trained_at": trained_at, + "train_from": bundle["metadata"]["train_from"], + "train_to": bundle["metadata"]["train_to"], + "test_from": test_start.strftime("%Y-%m-%dT%H:%M:%SZ"), + "test_to": data_end.strftime("%Y-%m-%dT%H:%M:%SZ"), + "features": list(config.FEATURE_WEEK_WEATHER), + "holiday_effect": holiday_effect, + "ablation": [_round_row(r) for r in results], + "limits": list(config.LIMITS), + } + (artifacts / "metrics.json").write_text( + json.dumps(metrics_payload, indent=2, ensure_ascii=False) + "\n" + ) + + start = pd.Timestamp.now(tz="UTC").ceil("h") + forecasts = _export_forecasts(bundle, forecast, start) + (artifacts / "forecasts.json").write_text( + json.dumps(forecasts, ensure_ascii=False) + "\n" + ) + log.info( + "wrote metrics.json, forecasts.json (%s..%s) and model.joblib", + start, + start + pd.Timedelta(hours=167), + ) + return { + "headline": metrics_payload["metrics"], + "ablation": metrics_payload["ablation"], + "model_version": model_version, + "forecast_start": start.strftime("%Y-%m-%dT%H:%M:%SZ"), + } + + +def _export_forecasts(bundle: dict, forecast: pd.DataFrame, start: pd.Timestamp) -> dict: + periods = config.HORIZON_STEPS["7d"] + sensors = sorted(config.USABLE_SENSORS) + out: dict[str, dict[str, dict[str, list[dict]]]] = { + sensor: {"48h": {}, "7d": {}} for sensor in sensors + } + base_calendar = None + for name, cfg in config.SCENARIOS.items(): + event = cfg.get("event") + for sensor in sensors: + from .features import future_calendar + + cal = future_calendar(start, periods) + if base_calendar is None: + base_calendar = cal + mask = ( + cal["weekday"].isin(event["weekdays"]).to_numpy() + if event + else None + ) + frame = future_frame( + bundle=bundle, + sensor=sensor, + start=start, + periods=periods, + weather_forecast=forecast, + temp_delta=cfg["temp_delta_c"], + precip_mm=cfg["precip_mm"], + weather_mask=mask, + ) + pred = predict(bundle, frame) * event_multiplier(frame, cfg) + pts = points(frame, pred, bundle, sensor) + out[sensor]["7d"][name] = pts + out[sensor]["48h"][name] = pts[: config.HORIZON_STEPS["48h"]] + + return { + "data_version": bundle["metadata"]["data_version"], + "model_version": bundle["metadata"]["model_version"], + "context_version": bundle["metadata"]["context_version"], + "generated_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), + "forecast_start": start.strftime("%Y-%m-%dT%H:%M:%SZ"), + "forecasts": out, + } + + +def main() -> None: + run_train() + + +if __name__ == "__main__": + main()