feat: ml bootstrap #1

Merged
Noel merged 4 commits from feat/ml-bootstrap into main 2026-11-14 04:17:59 +00:00
9 changed files with 41 additions and 26 deletions
Showing only changes of commit dc83825de0 - Show all commits

No files matched your search

+7 -4
View File
@@ -14,16 +14,19 @@ honest holdout metrics → JSON artifacts → decision-ready API.
| Model | MAE | WAPE | R² |
|---|---:|---:|---:|
| Seasonal naive (same local hour last week) | 4.23 | 35.0% | 0.656 |
| Seasonal naive (exact 168-row lag) | 5.40 | 44.7% | 0.462 |
| Day-ahead, no weather | 3.96 | 32.8% | 0.706 |
| Day-ahead, + holiday flag | 3.84 | 31.8% | 0.732 |
| Day-ahead, + temperature/rain | 3.36 | 27.8% | 0.763 |
| 7-day-safe, no weather | 4.08 | 33.8% | 0.697 |
| **7-day-safe, + temperature/rain (production)** | **3.32** | **27.5%** | **0.777** |
Weather cuts error by ~22% versus the no-weather 7-day model and ~38% versus
seasonal naive. The mission reference numbers (5.40 / 44.7% / 0.462) come from a
different quick-baseline window; these numbers are reproducible from this repo.
The seasonal-naive row reproduces the mission reference exactly
(5.40 / 44.7% / 0.462), and the day-ahead weather row matches 3.36 / 27.8%.
Weather cuts error by ~19% versus the no-weather 7-day model and ~38% versus
seasonal naive. The production model is slightly better than the reference
because its seasonal feature uses the previous *observed* same weekday/hour
across sensor gaps; the baseline uses the exact 168-row lag.
Small busy-hour classifier (baked for the "train new data" demo):
+1 -1
View File
@@ -1 +1 @@
{"data_version": "2026-11-07T23:51:00Z", "model_version": "histgb-2026-11-14", "context_version": "2026-11-14T03:47:09Z", "generated_at": "2026-11-14T04:00:09Z", "forecast_start": "2026-11-14T05:00:00Z", "forecasts": {"density-01": {"48h": {"base": [{"t": "2026-11-14T05:00:00Z", "v": 1.17, "lo": 0.0, "hi": 6.04}, {"t": "2026-11-14T06:00:00Z", "v": 1.95, "lo": 0.0, "hi": 6.83}, {"t": "2026-11-14T07:00:00Z", "v": 2.94, "lo": 0.0, "hi": 7.81}, {"t": "2026-11-14T08:00:00Z", "v": 4.08, "lo": 0.0, "hi": 8.96}, {"t": "2026-11-14T09:00:00Z", "v": 5.68, "lo": 0.85, "hi": 10.55}, {"t": "2026-11-14T10:00:00Z", "v": 8.5, "lo": 3.67, "hi": 13.38}, {"t": "2026-11-14T11:00:00Z", "v": 9.66, "lo": 4.84, "hi": 14.54}, {"t": "2026-11-14T12:00:00Z", "v": 11.39, "lo": 6.56, "hi": 16.26}, {"t": "2026-11-14T13:00:00Z", "v": 11.32, "lo": 6.5, "hi": 16.2}, {"t": "2026-11-14T14:00:00Z", "v": 12.63, "lo": 7.81, "hi": 17.51}, {"t": "2026-11-14T15:00:00Z", "v": 14.78, "lo": 9.96, "hi": 19.66}, {"t": "2026-11-14T16:00:00Z", "v": 14.21, "lo": 9.39, "hi": 19.09}, {"t": "2026-11-14T17:00:00Z", "v": 14.52, "lo": 9.7, "hi": 19.4}, {"t": "2026-11-14T18:00:00Z", "v": 15.67, "lo": 10.84, "hi": 20.55}, {"t": "2026-11-14T19:00:00Z", "v": 14.37, "lo": 9.55, "hi": 19.25}, {"t": "2026-11-14T20:00:00Z", "v": 12.64, "lo": 7.81, "hi": 17.51}, {"t": "2026-11-14T21:00:00Z", "v": 14.52, "lo": 9.7, "hi": 19.4}, {"t": "2026-11-14T22:00:00Z", "v": 12.33, "lo": 7.5, "hi": 17.21}, {"t": "2026-11-14T23:00:00Z", "v": 9.87, "lo": 5.04, "hi": 14.74}, {"t": "2026-11-15T00:00:00Z", "v": 3.94, "lo": 0.0, "hi": 8.82}, {"t": "2026-11-15T01:00:00Z", "v": 1.34, "lo": 0.0, "hi": 6.22}, {"t": "2026-11-15T02:00:00Z", "v": 1.13, "lo": 0.0, "hi": 6.01}, {"t": "2026-11-15T03:00:00Z", "v": 0.99, "lo": 0.0, "hi": 5.86}, {"t": "2026-11-15T04:00:00Z", "v": 0.99, "lo": 0.0, "hi": 5.86}, {"t": "2026-11-15T05:00:00Z", "v": 1.01, "lo": 0.0, "hi": 5.88}, {"t": "2026-11-15T06:00:00Z", "v": 1.04, "lo": 0.0, "hi": 5.92}, {"t": "2026-11-15T07:00:00Z", "v": 1.55, "lo": 0.0, "hi": 6.43}, {"t": "2026-11-15T08:00:00Z", "v": 1.65, "lo": 0.0, "hi": 6.53}, {"t": "2026-11-15T09:00:00Z", "v": 5.61, "lo": 0.78, "hi": 10.48}, {"t": "2026-11-15T10:00:00Z", "v": 6.5, "lo": 1.68, "hi": 11.38}, {"t": "2026-11-15T11:00:00Z", "v": 7.51, "lo": 2.69, "hi": 12.39}, {"t": "2026-11-15T12:00:00Z", "v": 9.58, "lo": 4.76, "hi": 14.46}, {"t": "2026-11-15T13:00:00Z", "v": 11.7, "lo": 6.88, "hi": 16.58}, {"t": "2026-11-15T14:00:00Z", "v": 18.36, "lo": 13.54, "hi": 23.24}, {"t": "2026-11-15T15:00:00Z", "v": 17.33, "lo": 12.5, "hi": 22.21}, {"t": "2026-11-15T16:00:00Z", "v": 16.04, "lo": 11.21, "hi": 20.91}, {"t": "2026-11-15T17:00:00Z", "v": 11.34, "lo": 6.51, "hi": 16.21}, {"t": "2026-11-15T18:00:00Z", "v": 10.87, "lo": 6.05, "hi": 15.75}, {"t": "2026-11-15T19:00:00Z", "v": 8.45, "lo": 3.63, "hi": 13.33}, {"t": "2026-11-15T20:00:00Z", "v": 5.82, "lo": 0.99, "hi": 10.7}, {"t": "2026-11-15T21:00:00Z", "v": 3.71, "lo": 0.0, "hi": 8.59}, {"t": "2026-11-15T22:00:00Z", "v": 2.11, "lo": 0.0, "hi": 6.99}, {"t": "2026-11-15T23:00:00Z", "v": 1.47, "lo": 0.0, "hi": 6.34}, {"t": "2026-11-16T00:00:00Z", "v": 0.91, "lo": 0.0, "hi": 5.79}, {"t": "2026-11-16T01:00:00Z", "v": 0.88, "lo": 0.0, "hi": 5.76}, {"t": "2026-11-16T02:00:00Z", "v": 1.15, "lo": 0.0, "hi": 6.03}, {"t": "2026-11-16T03:00:00Z", "v": 1.15, "lo": 0.0, "hi": 6.03}, {"t": "2026-11-16T04:00:00Z", "v": 1.74, "lo": 0.0, "hi": 6.62}], "sunny-saturday-market": [{"t": "2026-11-14T05:00:00Z", "v": 1.17, "lo": 0.0, "hi": 6.04}, {"t": "2026-11-14T06:00:00Z", "v": 1.95, "lo": 0.0, "hi": 6.83}, {"t": "2026-11-14T07:00:00Z", "v": 2.94, "lo": 0.0, "hi": 7.81}, {"t": "2026-11-14T08:00:00Z", "v": 4.08, "lo": 0.0, "hi": 8.96}, {"t": "2026-11-14T09:00:00Z", "v": 7.63, "lo": 2.81, "hi": 12.51}, {"t": "2026-11-14T10:00:00Z", "v": 10.14, "lo": 5.32, "hi": 15.02}, {"t": "2026-11-14T11:00:00Z", "v": 11.16, "lo": 6.34, "hi": 16.04}, {"t": "2026-11-14T12:00:00Z", "v": 12.88, "lo": 8.05, "hi": 17.76}, {"t": "2026-11-14T13:00:00Z", "v": 12.02, "lo": 7.2, "hi": 16.9}, {"t": "2026-11-14T14:00:00Z", "v": 17.24, "lo": 12.42, "hi": 22.12}, {"t": "2026-11-14T15:00:00Z", "v": 15.8, "lo": 10.97, "hi": 20.67}, {"t": "2026-11-14T16:00:00Z", "v": 15.97, "lo": 11.15, "hi": 20.85}, {"t": "2026-11-14T17:00:00Z", "v": 15.77, "lo": 10.94, "hi": 20.65}, {"t": "2026-11-14T18:00:00Z", "v": 16.1, "lo": 11.27, "hi": 20.97}, {"t": "2026-11-14T19:00:00Z", "v": 17.49, "lo": 12.67, "hi": 22.37}, {"t": "2026-11-14T20:00:00Z", "v": 12.64, "lo": 7.81, "hi": 17.51}, {"t": "2026-11-14T21:00:00Z", "v": 14.52, "lo": 9.7, "hi": 19.4}, {"t": "2026-11-14T22:00:00Z", "v": 12.33, "lo": 7.5, "hi": 17.21}, {"t": "2026-11-14T23:00:00Z", "v": 9.87, "lo": 5.04, "hi": 14.74}, {"t": "2026-11-15T00:00:00Z", "v": 3.94, "lo": 0.0, "hi": 8.82}, {"t": "2026-11-15T01:00:00Z", "v": 1.34, "lo": 0.0, "hi": 6.22}, {"t": "2026-11-15T02:00:00Z", "v": 1.13, "lo": 0.0, "hi": 6.01}, {"t": "2026-11-15T03:00:00Z", "v": 0.99, "lo": 0.0, "hi": 5.86}, {"Line truncated
{"data_version": "2026-11-07T23:51:00Z", "model_version": "histgb-2026-11-14", "context_version": "2026-11-14T03:47:09Z", "generated_at": "2026-11-14T04:03:38Z", "forecast_start": "2026-11-14T05:00:00Z", "forecasts": {"density-01": {"48h": {"base": [{"t": "2026-11-14T05:00:00Z", "v": 1.17, "lo": 0.0, "hi": 6.04}, {"t": "2026-11-14T06:00:00Z", "v": 1.95, "lo": 0.0, "hi": 6.83}, {"t": "2026-11-14T07:00:00Z", "v": 2.94, "lo": 0.0, "hi": 7.81}, {"t": "2026-11-14T08:00:00Z", "v": 4.08, "lo": 0.0, "hi": 8.96}, {"t": "2026-11-14T09:00:00Z", "v": 5.68, "lo": 0.85, "hi": 10.55}, {"t": "2026-11-14T10:00:00Z", "v": 8.5, "lo": 3.67, "hi": 13.38}, {"t": "2026-11-14T11:00:00Z", "v": 9.66, "lo": 4.84, "hi": 14.54}, {"t": "2026-11-14T12:00:00Z", "v": 11.39, "lo": 6.56, "hi": 16.26}, {"t": "2026-11-14T13:00:00Z", "v": 11.32, "lo": 6.5, "hi": 16.2}, {"t": "2026-11-14T14:00:00Z", "v": 12.63, "lo": 7.81, "hi": 17.51}, {"t": "2026-11-14T15:00:00Z", "v": 14.78, "lo": 9.96, "hi": 19.66}, {"t": "2026-11-14T16:00:00Z", "v": 14.21, "lo": 9.39, "hi": 19.09}, {"t": "2026-11-14T17:00:00Z", "v": 14.52, "lo": 9.7, "hi": 19.4}, {"t": "2026-11-14T18:00:00Z", "v": 15.67, "lo": 10.84, "hi": 20.55}, {"t": "2026-11-14T19:00:00Z", "v": 14.37, "lo": 9.55, "hi": 19.25}, {"t": "2026-11-14T20:00:00Z", "v": 12.64, "lo": 7.81, "hi": 17.51}, {"t": "2026-11-14T21:00:00Z", "v": 14.52, "lo": 9.7, "hi": 19.4}, {"t": "2026-11-14T22:00:00Z", "v": 12.33, "lo": 7.5, "hi": 17.21}, {"t": "2026-11-14T23:00:00Z", "v": 9.87, "lo": 5.04, "hi": 14.74}, {"t": "2026-11-15T00:00:00Z", "v": 3.94, "lo": 0.0, "hi": 8.82}, {"t": "2026-11-15T01:00:00Z", "v": 1.34, "lo": 0.0, "hi": 6.22}, {"t": "2026-11-15T02:00:00Z", "v": 1.13, "lo": 0.0, "hi": 6.01}, {"t": "2026-11-15T03:00:00Z", "v": 0.99, "lo": 0.0, "hi": 5.86}, {"t": "2026-11-15T04:00:00Z", "v": 0.99, "lo": 0.0, "hi": 5.86}, {"t": "2026-11-15T05:00:00Z", "v": 1.01, "lo": 0.0, "hi": 5.88}, {"t": "2026-11-15T06:00:00Z", "v": 1.04, "lo": 0.0, "hi": 5.92}, {"t": "2026-11-15T07:00:00Z", "v": 1.55, "lo": 0.0, "hi": 6.43}, {"t": "2026-11-15T08:00:00Z", "v": 1.65, "lo": 0.0, "hi": 6.53}, {"t": "2026-11-15T09:00:00Z", "v": 5.61, "lo": 0.78, "hi": 10.48}, {"t": "2026-11-15T10:00:00Z", "v": 6.5, "lo": 1.68, "hi": 11.38}, {"t": "2026-11-15T11:00:00Z", "v": 7.51, "lo": 2.69, "hi": 12.39}, {"t": "2026-11-15T12:00:00Z", "v": 9.58, "lo": 4.76, "hi": 14.46}, {"t": "2026-11-15T13:00:00Z", "v": 11.7, "lo": 6.88, "hi": 16.58}, {"t": "2026-11-15T14:00:00Z", "v": 18.36, "lo": 13.54, "hi": 23.24}, {"t": "2026-11-15T15:00:00Z", "v": 17.33, "lo": 12.5, "hi": 22.21}, {"t": "2026-11-15T16:00:00Z", "v": 16.04, "lo": 11.21, "hi": 20.91}, {"t": "2026-11-15T17:00:00Z", "v": 11.34, "lo": 6.51, "hi": 16.21}, {"t": "2026-11-15T18:00:00Z", "v": 10.87, "lo": 6.05, "hi": 15.75}, {"t": "2026-11-15T19:00:00Z", "v": 8.45, "lo": 3.63, "hi": 13.33}, {"t": "2026-11-15T20:00:00Z", "v": 5.82, "lo": 0.99, "hi": 10.7}, {"t": "2026-11-15T21:00:00Z", "v": 3.71, "lo": 0.0, "hi": 8.59}, {"t": "2026-11-15T22:00:00Z", "v": 2.11, "lo": 0.0, "hi": 6.99}, {"t": "2026-11-15T23:00:00Z", "v": 1.47, "lo": 0.0, "hi": 6.34}, {"t": "2026-11-16T00:00:00Z", "v": 0.91, "lo": 0.0, "hi": 5.79}, {"t": "2026-11-16T01:00:00Z", "v": 0.88, "lo": 0.0, "hi": 5.76}, {"t": "2026-11-16T02:00:00Z", "v": 1.15, "lo": 0.0, "hi": 6.03}, {"t": "2026-11-16T03:00:00Z", "v": 1.15, "lo": 0.0, "hi": 6.03}, {"t": "2026-11-16T04:00:00Z", "v": 1.74, "lo": 0.0, "hi": 6.62}], "sunny-saturday-market": [{"t": "2026-11-14T05:00:00Z", "v": 1.17, "lo": 0.0, "hi": 6.04}, {"t": "2026-11-14T06:00:00Z", "v": 1.95, "lo": 0.0, "hi": 6.83}, {"t": "2026-11-14T07:00:00Z", "v": 2.94, "lo": 0.0, "hi": 7.81}, {"t": "2026-11-14T08:00:00Z", "v": 4.08, "lo": 0.0, "hi": 8.96}, {"t": "2026-11-14T09:00:00Z", "v": 7.63, "lo": 2.81, "hi": 12.51}, {"t": "2026-11-14T10:00:00Z", "v": 10.14, "lo": 5.32, "hi": 15.02}, {"t": "2026-11-14T11:00:00Z", "v": 11.16, "lo": 6.34, "hi": 16.04}, {"t": "2026-11-14T12:00:00Z", "v": 12.88, "lo": 8.05, "hi": 17.76}, {"t": "2026-11-14T13:00:00Z", "v": 12.02, "lo": 7.2, "hi": 16.9}, {"t": "2026-11-14T14:00:00Z", "v": 17.24, "lo": 12.42, "hi": 22.12}, {"t": "2026-11-14T15:00:00Z", "v": 15.8, "lo": 10.97, "hi": 20.67}, {"t": "2026-11-14T16:00:00Z", "v": 15.97, "lo": 11.15, "hi": 20.85}, {"t": "2026-11-14T17:00:00Z", "v": 15.77, "lo": 10.94, "hi": 20.65}, {"t": "2026-11-14T18:00:00Z", "v": 16.1, "lo": 11.27, "hi": 20.97}, {"t": "2026-11-14T19:00:00Z", "v": 17.49, "lo": 12.67, "hi": 22.37}, {"t": "2026-11-14T20:00:00Z", "v": 12.64, "lo": 7.81, "hi": 17.51}, {"t": "2026-11-14T21:00:00Z", "v": 14.52, "lo": 9.7, "hi": 19.4}, {"t": "2026-11-14T22:00:00Z", "v": 12.33, "lo": 7.5, "hi": 17.21}, {"t": "2026-11-14T23:00:00Z", "v": 9.87, "lo": 5.04, "hi": 14.74}, {"t": "2026-11-15T00:00:00Z", "v": 3.94, "lo": 0.0, "hi": 8.82}, {"t": "2026-11-15T01:00:00Z", "v": 1.34, "lo": 0.0, "hi": 6.22}, {"t": "2026-11-15T02:00:00Z", "v": 1.13, "lo": 0.0, "hi": 6.01}, {"t": "2026-11-15T03:00:00Z", "v": 0.99, "lo": 0.0, "hi": 5.86}, {"Line truncated
+8 -8
View File
@@ -14,12 +14,12 @@
"wape": 27.5,
"r2": 0.777,
"baseline": "seasonal_naive",
"baseline_mae": 4.23,
"baseline_wape": 35.0,
"baseline_r2": 0.656,
"baseline_mae": 5.4,
"baseline_wape": 44.7,
"baseline_r2": 0.462,
"n_eval": 5350
},
"trained_at": "2026-11-14T04:00:08Z",
"trained_at": "2026-11-14T04:03:37Z",
"train_from": "2026-06-12T00:00:00Z",
"train_to": "2026-11-07T23:00:00Z",
"test_from": "2026-10-10T23:00:00Z",
@@ -43,10 +43,10 @@
"ablation": [
{
"name": "seasonal_naive",
"description": "same local hour last week",
"mae": 4.23,
"wape": 35.0,
"r2": 0.656,
"description": "exact 168-row lag (same observed hour last week)",
"mae": 5.4,
"wape": 44.7,
"r2": 0.462,
"n": 5350
},
{
Binary file not shown.
+2 -2
View File
@@ -1,6 +1,6 @@
{
"model_version": "logit-20261114-040010",
"trained_at": "2026-11-14T04:00:10Z",
"model_version": "logit-20261114-040339",
"trained_at": "2026-11-14T04:03:39Z",
"features": [
"hour",
"weekday",
Binary file not shown.
+10 -6
View File
@@ -50,21 +50,24 @@ Time-based 28-day holdout, no shuffling. Seasonal naive is the baseline. WAPE is
| Model | MAE | WAPE | R² |
|---|---:|---:|---:|
| Seasonal naive (same local hour last week) | 4.23 | 35.0% | 0.656 |
| Seasonal naive (exact 168-row lag) | 5.40 | 44.7% | 0.462 |
| Day-ahead, no weather | 3.96 | 32.8% | 0.706 |
| Day-ahead, + holiday flag | 3.84 | 31.8% | 0.732 |
| Day-ahead, + temperature/rain | 3.36 | 27.8% | 0.763 |
| 7-day-safe, no weather | 4.08 | 33.8% | 0.697 |
| **7-day-safe, + temperature/rain (production)** | **3.32** | **27.5%** | **0.777** |
Weather is the useful signal: adding temperature and rain cuts MAE by ~22%
Weather is the useful signal: adding temperature and rain cuts MAE by ~19%
versus the 7-day no-weather model and ~38% versus seasonal naive. The holiday
flag alone changes day-ahead MAE by −0.12 in this run; holidays are context, not
a headline accuracy claim.
The mission's quick-baseline numbers (seasonal naive 5.40 / 44.7% / 0.462) came
from a different window and feature setup. The direction and the weather gain
reproduce; the exact decimals are not copied.
The seasonal-naive row reproduces the mission reference exactly
(5.40 / 44.7% / 0.462), and the day-ahead weather row matches 3.36 / 27.8%. The
production model is slightly better than the reference because `lag_week` uses
the previous *observed* same weekday/hour across sensor gaps, while the baseline
uses the exact 168-row lag. R² differs slightly from the reference 0.779
(0.777 here) because of that feature choice.
## Uncertainty band
@@ -87,7 +90,8 @@ guarantee.
## No-leakage rules
- All splits are time-based; no random shuffling.
- Seasonal lags use the previous *observed* same weekday/hour; future lags are
- The seasonal-naive baseline uses an exact 168-row lag. The model's
`lag_week` uses the previous *observed* same weekday/hour; future lags are
never filled from future targets.
- Rolling means are shifted at least 24 hours back.
- The production 7-day model uses only ≥168-hour-old information, so a 168-hour
+10 -2
View File
@@ -70,8 +70,16 @@ def _add_rollups(grid: pd.DataFrame) -> pd.DataFrame:
def _add_seasonal_lags(df: pd.DataFrame) -> pd.DataFrame:
"""Previous observed same weekday/hour values, strictly in the past."""
out = df.sort_values(["sensor", "weekday", "hour", "t"]).copy()
"""Previous observed same weekday/hour values, strictly in the past.
``lag_168`` is the exact 168-row (one observed week) lag used as the
seasonal-naive baseline. The model's ``lag_week`` deliberately uses the
previous *observed* same weekday/hour instead, so sensor gaps do not make
the feature unusable.
"""
chrono = df.sort_values(["sensor", "t"]).copy()
chrono["lag_168"] = chrono.groupby("sensor", observed=True)["y"].shift(168)
out = chrono.sort_values(["sensor", "weekday", "hour", "t"]).copy()
grouped = out.groupby(["sensor", "weekday", "hour"], observed=True)["y"]
out["lag_week"] = grouped.shift(1)
prior = pd.concat(
+3 -3
View File
@@ -29,7 +29,7 @@ from .weather import load_cached_weather
log = logging.getLogger(__name__)
MODEL_SPECS = [
("seasonal_naive", None, "same local hour last week"),
("seasonal_naive", None, "exact 168-row lag (same observed hour last week)"),
(
"day_no_weather",
config.FEATURE_DAY_NO_HOLIDAY,
@@ -107,7 +107,7 @@ def train_bundle(
test_start = data_end.floor("h") - pd.Timedelta(days=config.TEST_DAYS)
train = features[features["t"] < test_start].copy()
test = features[features["t"] >= test_start].copy()
eval_frame = test[test["lag_week"].notna() & test["y"].notna()].copy()
eval_frame = test[test["lag_168"].notna() & test["y"].notna()].copy()
if eval_frame.empty:
raise RuntimeError("holdout has no rows with a seasonal-naive baseline")
if len(train) < 100:
@@ -128,7 +128,7 @@ def train_bundle(
preds: dict[str, np.ndarray] = {}
for name, cols, description in MODEL_SPECS:
if name == "seasonal_naive":
pred = eval_frame["lag_week"].to_numpy(dtype=float)
pred = eval_frame["lag_168"].to_numpy(dtype=float)
else:
assert cols is not None
model = fit(train, cols)