feat: per-sensor holdout metrics and small-model confusion matrix
- train.py emits per-sensor MAE/WAPE/R2 vs the seasonal-naive baseline - small.py emits the full confusion matrix and per-sensor classifier metrics - metrics.json now carries the small-model summary for paper support - regenerate baked artifacts; tests assert the tables are internally consistent
This commit is contained in:
1 parent
4ea7212b70
commit
000aaed7a5
8 files changed
+386
-10
No files matched your search
@@ -1 +1 @@
|
|||||||
{"data_version": "2026-11-07T23:51:00Z", "model_version": "histgb-2026-11-14", "context_version": "2026-11-14T03:47:09Z", "generated_at": "2026-11-14T04:03:38Z", "forecast_start": "2026-11-14T05:00:00Z", "forecasts": {"density-01": {"48h": {"base": [{"t": "2026-11-14T05:00:00Z", "v": 1.17, "lo": 0.0, "hi": 6.04}, {"t": "2026-11-14T06:00:00Z", "v": 1.95, "lo": 0.0, "hi": 6.83}, {"t": "2026-11-14T07:00:00Z", "v": 2.94, "lo": 0.0, "hi": 7.81}, {"t": "2026-11-14T08:00:00Z", "v": 4.08, "lo": 0.0, "hi": 8.96}, {"t": "2026-11-14T09:00:00Z", "v": 5.68, "lo": 0.85, "hi": 10.55}, {"t": "2026-11-14T10:00:00Z", "v": 8.5, "lo": 3.67, "hi": 13.38}, {"t": "2026-11-14T11:00:00Z", "v": 9.66, "lo": 4.84, "hi": 14.54}, {"t": "2026-11-14T12:00:00Z", "v": 11.39, "lo": 6.56, "hi": 16.26}, {"t": "2026-11-14T13:00:00Z", "v": 11.32, "lo": 6.5, "hi": 16.2}, {"t": "2026-11-14T14:00:00Z", "v": 12.63, "lo": 7.81, "hi": 17.51}, {"t": "2026-11-14T15:00:00Z", "v": 14.78, "lo": 9.96, "hi": 19.66}, {"t": "2026-11-14T16:00:00Z", "v": 14.21, "lo": 9.39, "hi": 19.09}, {"t": "2026-11-14T17:00:00Z", "v": 14.52, "lo": 9.7, "hi": 19.4}, {"t": "2026-11-14T18:00:00Z", "v": 15.67, "lo": 10.84, "hi": 20.55}, {"t": "2026-11-14T19:00:00Z", "v": 14.37, "lo": 9.55, "hi": 19.25}, {"t": "2026-11-14T20:00:00Z", "v": 12.64, "lo": 7.81, "hi": 17.51}, {"t": "2026-11-14T21:00:00Z", "v": 14.52, "lo": 9.7, "hi": 19.4}, {"t": "2026-11-14T22:00:00Z", "v": 12.33, "lo": 7.5, "hi": 17.21}, {"t": "2026-11-14T23:00:00Z", "v": 9.87, "lo": 5.04, "hi": 14.74}, {"t": "2026-11-15T00:00:00Z", "v": 3.94, "lo": 0.0, "hi": 8.82}, {"t": "2026-11-15T01:00:00Z", "v": 1.34, "lo": 0.0, "hi": 6.22}, {"t": "2026-11-15T02:00:00Z", "v": 1.13, "lo": 0.0, "hi": 6.01}, {"t": "2026-11-15T03:00:00Z", "v": 0.99, "lo": 0.0, "hi": 5.86}, {"t": "2026-11-15T04:00:00Z", "v": 0.99, "lo": 0.0, "hi": 5.86}, {"t": "2026-11-15T05:00:00Z", "v": 1.01, "lo": 0.0, "hi": 5.88}, {"t": "2026-11-15T06:00:00Z", "v": 1.04, "lo": 0.0, "hi": 5.92}, {"t": "2026-11-15T07:00:00Z", "v": 1.55, "lo": 0.0, "hi": 6.43}, {"t": "2026-11-15T08:00:00Z", "v": 1.65, "lo": 0.0, "hi": 6.53}, {"t": "2026-11-15T09:00:00Z", "v": 5.61, "lo": 0.78, "hi": 10.48}, {"t": "2026-11-15T10:00:00Z", "v": 6.5, "lo": 1.68, "hi": 11.38}, {"t": "2026-11-15T11:00:00Z", "v": 7.51, "lo": 2.69, "hi": 12.39}, {"t": "2026-11-15T12:00:00Z", "v": 9.58, "lo": 4.76, "hi": 14.46}, {"t": "2026-11-15T13:00:00Z", "v": 11.7, "lo": 6.88, "hi": 16.58}, {"t": "2026-11-15T14:00:00Z", "v": 18.36, "lo": 13.54, "hi": 23.24}, {"t": "2026-11-15T15:00:00Z", "v": 17.33, "lo": 12.5, "hi": 22.21}, {"t": "2026-11-15T16:00:00Z", "v": 16.04, "lo": 11.21, "hi": 20.91}, {"t": "2026-11-15T17:00:00Z", "v": 11.34, "lo": 6.51, "hi": 16.21}, {"t": "2026-11-15T18:00:00Z", "v": 10.87, "lo": 6.05, "hi": 15.75}, {"t": "2026-11-15T19:00:00Z", "v": 8.45, "lo": 3.63, "hi": 13.33}, {"t": "2026-11-15T20:00:00Z", "v": 5.82, "lo": 0.99, "hi": 10.7}, {"t": "2026-11-15T21:00:00Z", "v": 3.71, "lo": 0.0, "hi": 8.59}, {"t": "2026-11-15T22:00:00Z", "v": 2.11, "lo": 0.0, "hi": 6.99}, {"t": "2026-11-15T23:00:00Z", "v": 1.47, "lo": 0.0, "hi": 6.34}, {"t": "2026-11-16T00:00:00Z", "v": 0.91, "lo": 0.0, "hi": 5.79}, {"t": "2026-11-16T01:00:00Z", "v": 0.88, "lo": 0.0, "hi": 5.76}, {"t": "2026-11-16T02:00:00Z", "v": 1.15, "lo": 0.0, "hi": 6.03}, {"t": "2026-11-16T03:00:00Z", "v": 1.15, "lo": 0.0, "hi": 6.03}, {"t": "2026-11-16T04:00:00Z", "v": 1.74, "lo": 0.0, "hi": 6.62}], "sunny-saturday-market": [{"t": "2026-11-14T05:00:00Z", "v": 1.17, "lo": 0.0, "hi": 6.04}, {"t": "2026-11-14T06:00:00Z", "v": 1.95, "lo": 0.0, "hi": 6.83}, {"t": "2026-11-14T07:00:00Z", "v": 2.94, "lo": 0.0, "hi": 7.81}, {"t": "2026-11-14T08:00:00Z", "v": 4.08, "lo": 0.0, "hi": 8.96}, {"t": "2026-11-14T09:00:00Z", "v": 7.63, "lo": 2.81, "hi": 12.51}, {"t": "2026-11-14T10:00:00Z", "v": 10.14, "lo": 5.32, "hi": 15.02}, {"t": "2026-11-14T11:00:00Z", "v": 11.16, "lo": 6.34, "hi": 16.04}, {"t": "2026-11-14T12:00:00Z", "v": 12.88, "lo": 8.05, "hi": 17.76}, {"t": "2026-11-14T13:00:00Z", "v": 12.02, "lo": 7.2, "hi": 16.9}, {"t": "2026-11-14T14:00:00Z", "v": 17.24, "lo": 12.42, "hi": 22.12}, {"t": "2026-11-14T15:00:00Z", "v": 15.8, "lo": 10.97, "hi": 20.67}, {"t": "2026-11-14T16:00:00Z", "v": 15.97, "lo": 11.15, "hi": 20.85}, {"t": "2026-11-14T17:00:00Z", "v": 15.77, "lo": 10.94, "hi": 20.65}, {"t": "2026-11-14T18:00:00Z", "v": 16.1, "lo": 11.27, "hi": 20.97}, {"t": "2026-11-14T19:00:00Z", "v": 17.49, "lo": 12.67, "hi": 22.37}, {"t": "2026-11-14T20:00:00Z", "v": 12.64, "lo": 7.81, "hi": 17.51}, {"t": "2026-11-14T21:00:00Z", "v": 14.52, "lo": 9.7, "hi": 19.4}, {"t": "2026-11-14T22:00:00Z", "v": 12.33, "lo": 7.5, "hi": 17.21}, {"t": "2026-11-14T23:00:00Z", "v": 9.87, "lo": 5.04, "hi": 14.74}, {"t": "2026-11-15T00:00:00Z", "v": 3.94, "lo": 0.0, "hi": 8.82}, {"t": "2026-11-15T01:00:00Z", "v": 1.34, "lo": 0.0, "hi": 6.22}, {"t": "2026-11-15T02:00:00Z", "v": 1.13, "lo": 0.0, "hi": 6.01}, {"t": "2026-11-15T03:00:00Z", "v": 0.99, "lo": 0.0, "hi": 5.86}, {"Line truncated
|
{"data_version": "2026-11-07T23:51:00Z", "model_version": "histgb-2026-11-14", "context_version": "2026-11-14T03:47:09Z", "generated_at": "2026-11-14T05:27:22Z", "forecast_start": "2026-11-14T06:00:00Z", "forecasts": {"density-01": {"48h": {"base": [{"t": "2026-11-14T06:00:00Z", "v": 1.95, "lo": 0.0, "hi": 6.83}, {"t": "2026-11-14T07:00:00Z", "v": 2.94, "lo": 0.0, "hi": 7.81}, {"t": "2026-11-14T08:00:00Z", "v": 4.08, "lo": 0.0, "hi": 8.96}, {"t": "2026-11-14T09:00:00Z", "v": 5.68, "lo": 0.85, "hi": 10.55}, {"t": "2026-11-14T10:00:00Z", "v": 8.5, "lo": 3.67, "hi": 13.38}, {"t": "2026-11-14T11:00:00Z", "v": 9.66, "lo": 4.84, "hi": 14.54}, {"t": "2026-11-14T12:00:00Z", "v": 11.39, "lo": 6.56, "hi": 16.26}, {"t": "2026-11-14T13:00:00Z", "v": 11.32, "lo": 6.5, "hi": 16.2}, {"t": "2026-11-14T14:00:00Z", "v": 12.63, "lo": 7.81, "hi": 17.51}, {"t": "2026-11-14T15:00:00Z", "v": 14.78, "lo": 9.96, "hi": 19.66}, {"t": "2026-11-14T16:00:00Z", "v": 14.21, "lo": 9.39, "hi": 19.09}, {"t": "2026-11-14T17:00:00Z", "v": 14.52, "lo": 9.7, "hi": 19.4}, {"t": "2026-11-14T18:00:00Z", "v": 15.67, "lo": 10.84, "hi": 20.55}, {"t": "2026-11-14T19:00:00Z", "v": 14.37, "lo": 9.55, "hi": 19.25}, {"t": "2026-11-14T20:00:00Z", "v": 12.64, "lo": 7.81, "hi": 17.51}, {"t": "2026-11-14T21:00:00Z", "v": 14.52, "lo": 9.7, "hi": 19.4}, {"t": "2026-11-14T22:00:00Z", "v": 12.33, "lo": 7.5, "hi": 17.21}, {"t": "2026-11-14T23:00:00Z", "v": 9.87, "lo": 5.04, "hi": 14.74}, {"t": "2026-11-15T00:00:00Z", "v": 3.94, "lo": 0.0, "hi": 8.82}, {"t": "2026-11-15T01:00:00Z", "v": 1.34, "lo": 0.0, "hi": 6.22}, {"t": "2026-11-15T02:00:00Z", "v": 1.13, "lo": 0.0, "hi": 6.01}, {"t": "2026-11-15T03:00:00Z", "v": 0.99, "lo": 0.0, "hi": 5.86}, {"t": "2026-11-15T04:00:00Z", "v": 0.99, "lo": 0.0, "hi": 5.86}, {"t": "2026-11-15T05:00:00Z", "v": 1.01, "lo": 0.0, "hi": 5.88}, {"t": "2026-11-15T06:00:00Z", "v": 1.04, "lo": 0.0, "hi": 5.92}, {"t": "2026-11-15T07:00:00Z", "v": 1.55, "lo": 0.0, "hi": 6.43}, {"t": "2026-11-15T08:00:00Z", "v": 1.65, "lo": 0.0, "hi": 6.53}, {"t": "2026-11-15T09:00:00Z", "v": 5.61, "lo": 0.78, "hi": 10.48}, {"t": "2026-11-15T10:00:00Z", "v": 6.5, "lo": 1.68, "hi": 11.38}, {"t": "2026-11-15T11:00:00Z", "v": 7.51, "lo": 2.69, "hi": 12.39}, {"t": "2026-11-15T12:00:00Z", "v": 9.58, "lo": 4.76, "hi": 14.46}, {"t": "2026-11-15T13:00:00Z", "v": 11.7, "lo": 6.88, "hi": 16.58}, {"t": "2026-11-15T14:00:00Z", "v": 18.36, "lo": 13.54, "hi": 23.24}, {"t": "2026-11-15T15:00:00Z", "v": 17.33, "lo": 12.5, "hi": 22.21}, {"t": "2026-11-15T16:00:00Z", "v": 16.04, "lo": 11.21, "hi": 20.91}, {"t": "2026-11-15T17:00:00Z", "v": 11.34, "lo": 6.51, "hi": 16.21}, {"t": "2026-11-15T18:00:00Z", "v": 10.87, "lo": 6.05, "hi": 15.75}, {"t": "2026-11-15T19:00:00Z", "v": 8.45, "lo": 3.63, "hi": 13.33}, {"t": "2026-11-15T20:00:00Z", "v": 5.82, "lo": 0.99, "hi": 10.7}, {"t": "2026-11-15T21:00:00Z", "v": 3.71, "lo": 0.0, "hi": 8.59}, {"t": "2026-11-15T22:00:00Z", "v": 2.11, "lo": 0.0, "hi": 6.99}, {"t": "2026-11-15T23:00:00Z", "v": 1.47, "lo": 0.0, "hi": 6.34}, {"t": "2026-11-16T00:00:00Z", "v": 0.91, "lo": 0.0, "hi": 5.79}, {"t": "2026-11-16T01:00:00Z", "v": 0.88, "lo": 0.0, "hi": 5.76}, {"t": "2026-11-16T02:00:00Z", "v": 1.15, "lo": 0.0, "hi": 6.03}, {"t": "2026-11-16T03:00:00Z", "v": 1.15, "lo": 0.0, "hi": 6.03}, {"t": "2026-11-16T04:00:00Z", "v": 1.74, "lo": 0.0, "hi": 6.62}, {"t": "2026-11-16T05:00:00Z", "v": 1.58, "lo": 0.0, "hi": 6.46}], "sunny-saturday-market": [{"t": "2026-11-14T06:00:00Z", "v": 1.95, "lo": 0.0, "hi": 6.83}, {"t": "2026-11-14T07:00:00Z", "v": 2.94, "lo": 0.0, "hi": 7.81}, {"t": "2026-11-14T08:00:00Z", "v": 4.08, "lo": 0.0, "hi": 8.96}, {"t": "2026-11-14T09:00:00Z", "v": 7.63, "lo": 2.81, "hi": 12.51}, {"t": "2026-11-14T10:00:00Z", "v": 10.14, "lo": 5.32, "hi": 15.02}, {"t": "2026-11-14T11:00:00Z", "v": 11.16, "lo": 6.34, "hi": 16.04}, {"t": "2026-11-14T12:00:00Z", "v": 12.88, "lo": 8.05, "hi": 17.76}, {"t": "2026-11-14T13:00:00Z", "v": 12.02, "lo": 7.2, "hi": 16.9}, {"t": "2026-11-14T14:00:00Z", "v": 17.24, "lo": 12.42, "hi": 22.12}, {"t": "2026-11-14T15:00:00Z", "v": 15.8, "lo": 10.97, "hi": 20.67}, {"t": "2026-11-14T16:00:00Z", "v": 15.97, "lo": 11.15, "hi": 20.85}, {"t": "2026-11-14T17:00:00Z", "v": 15.77, "lo": 10.94, "hi": 20.65}, {"t": "2026-11-14T18:00:00Z", "v": 16.1, "lo": 11.27, "hi": 20.97}, {"t": "2026-11-14T19:00:00Z", "v": 17.49, "lo": 12.67, "hi": 22.37}, {"t": "2026-11-14T20:00:00Z", "v": 12.64, "lo": 7.81, "hi": 17.51}, {"t": "2026-11-14T21:00:00Z", "v": 14.52, "lo": 9.7, "hi": 19.4}, {"t": "2026-11-14T22:00:00Z", "v": 12.33, "lo": 7.5, "hi": 17.21}, {"t": "2026-11-14T23:00:00Z", "v": 9.87, "lo": 5.04, "hi": 14.74}, {"t": "2026-11-15T00:00:00Z", "v": 3.94, "lo": 0.0, "hi": 8.82}, {"t": "2026-11-15T01:00:00Z", "v": 1.34, "lo": 0.0, "hi": 6.22}, {"t": "2026-11-15T02:00:00Z", "v": 1.13, "lo": 0.0, "hi": 6.01}, {"t": "2026-11-15T03:00:00Z", "v": 0.99, "lo": 0.0, "hi": 5.86}, {"t": "2026-11-15T04:00:00Z", "v": 0.99, "lo": 0.0, "hi": 5.86}, {"Line truncated
|
||||||
+207
-2
@@ -19,7 +19,7 @@
|
|||||||
"baseline_r2": 0.462,
|
"baseline_r2": 0.462,
|
||||||
"n_eval": 5350
|
"n_eval": 5350
|
||||||
},
|
},
|
||||||
"trained_at": "2026-11-14T04:03:37Z",
|
"trained_at": "2026-11-14T05:27:21Z",
|
||||||
"train_from": "2026-06-12T00:00:00Z",
|
"train_from": "2026-06-12T00:00:00Z",
|
||||||
"train_to": "2026-11-07T23:00:00Z",
|
"train_to": "2026-11-07T23:00:00Z",
|
||||||
"test_from": "2026-10-10T23:00:00Z",
|
"test_from": "2026-10-10T23:00:00Z",
|
||||||
@@ -90,11 +90,216 @@
|
|||||||
"n": 5350
|
"n": 5350
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
|
"per_sensor": [
|
||||||
|
{
|
||||||
|
"sensor": "density-01",
|
||||||
|
"name": "Density 01",
|
||||||
|
"n": 669,
|
||||||
|
"mae": 2.26,
|
||||||
|
"wape": 30.8,
|
||||||
|
"r2": 0.722,
|
||||||
|
"baseline_mae": 3.58,
|
||||||
|
"baseline_wape": 48.8,
|
||||||
|
"baseline_r2": 0.238,
|
||||||
|
"mae_improvement_pct": 36.8
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"sensor": "hn-goetzentuem-01",
|
||||||
|
"name": "Götzenturm",
|
||||||
|
"n": 669,
|
||||||
|
"mae": 2.4,
|
||||||
|
"wape": 26.9,
|
||||||
|
"r2": 0.718,
|
||||||
|
"baseline_mae": 3.84,
|
||||||
|
"baseline_wape": 43.0,
|
||||||
|
"baseline_r2": 0.281,
|
||||||
|
"mae_improvement_pct": 37.5
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"sensor": "hn-hagenbuchersee-01",
|
||||||
|
"name": "Hagenbuchersee 1",
|
||||||
|
"n": 669,
|
||||||
|
"mae": 2.47,
|
||||||
|
"wape": 38.6,
|
||||||
|
"r2": 0.602,
|
||||||
|
"baseline_mae": 4.01,
|
||||||
|
"baseline_wape": 62.8,
|
||||||
|
"baseline_r2": -0.212,
|
||||||
|
"mae_improvement_pct": 38.5
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"sensor": "hn-hagenbuchersee-02",
|
||||||
|
"name": "Hagenbuchersee 2",
|
||||||
|
"n": 669,
|
||||||
|
"mae": 4.12,
|
||||||
|
"wape": 31.1,
|
||||||
|
"r2": 0.731,
|
||||||
|
"baseline_mae": 6.49,
|
||||||
|
"baseline_wape": 48.9,
|
||||||
|
"baseline_r2": 0.337,
|
||||||
|
"mae_improvement_pct": 36.5
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"sensor": "hn-marahaus-01",
|
||||||
|
"name": "Marahaus",
|
||||||
|
"n": 669,
|
||||||
|
"mae": 2.67,
|
||||||
|
"wape": 29.8,
|
||||||
|
"r2": 0.671,
|
||||||
|
"baseline_mae": 4.57,
|
||||||
|
"baseline_wape": 51.0,
|
||||||
|
"baseline_r2": -0.045,
|
||||||
|
"mae_improvement_pct": 41.7
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"sensor": "hn-neckarufer-01",
|
||||||
|
"name": "Neckarufer",
|
||||||
|
"n": 668,
|
||||||
|
"mae": 3.63,
|
||||||
|
"wape": 28.2,
|
||||||
|
"r2": 0.695,
|
||||||
|
"baseline_mae": 5.89,
|
||||||
|
"baseline_wape": 45.8,
|
||||||
|
"baseline_r2": 0.11,
|
||||||
|
"mae_improvement_pct": 38.4
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"sensor": "hn-soleo-01",
|
||||||
|
"name": "Soleo",
|
||||||
|
"n": 669,
|
||||||
|
"mae": 3.19,
|
||||||
|
"wape": 20.2,
|
||||||
|
"r2": 0.548,
|
||||||
|
"baseline_mae": 4.4,
|
||||||
|
"baseline_wape": 28.0,
|
||||||
|
"baseline_r2": 0.163,
|
||||||
|
"mae_improvement_pct": 27.7
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"sensor": "hn-stadtgalerie-01",
|
||||||
|
"name": "Stadtgalerie",
|
||||||
|
"n": 668,
|
||||||
|
"mae": 5.83,
|
||||||
|
"wape": 25.3,
|
||||||
|
"r2": 0.765,
|
||||||
|
"baseline_mae": 10.39,
|
||||||
|
"baseline_wape": 45.1,
|
||||||
|
"baseline_r2": 0.48,
|
||||||
|
"mae_improvement_pct": 43.9
|
||||||
|
}
|
||||||
|
],
|
||||||
"limits": [
|
"limits": [
|
||||||
"estimated devices, not people",
|
"estimated devices, not people",
|
||||||
"8 sensors in a ~580 m inner-city cluster",
|
"8 sensors in a ~580 m inner-city cluster",
|
||||||
"5 months of multi-sensor data; no annual seasonality claim",
|
"5 months of multi-sensor data; no annual seasonality claim",
|
||||||
"observed weather in training, forecast weather in scenarios",
|
"observed weather in training, forecast weather in scenarios",
|
||||||
"holidays are context, not a large accuracy claim"
|
"holidays are context, not a large accuracy claim"
|
||||||
]
|
],
|
||||||
|
"small": {
|
||||||
|
"model_version": "logit-20261114-052723",
|
||||||
|
"trained_at": "2026-11-14T05:27:23Z",
|
||||||
|
"features": [
|
||||||
|
"hour",
|
||||||
|
"weekday",
|
||||||
|
"is_weekend",
|
||||||
|
"temp_c",
|
||||||
|
"precip_mm",
|
||||||
|
"is_holiday"
|
||||||
|
],
|
||||||
|
"metrics": {
|
||||||
|
"accuracy": 0.708,
|
||||||
|
"precision": 0.36,
|
||||||
|
"recall": 0.574,
|
||||||
|
"f1": 0.443,
|
||||||
|
"roc_auc": 0.726,
|
||||||
|
"n_train": 18242,
|
||||||
|
"n_test": 5710,
|
||||||
|
"positive_rate": 0.202,
|
||||||
|
"sensors": 8
|
||||||
|
},
|
||||||
|
"confusion_matrix": {
|
||||||
|
"labels": {
|
||||||
|
"0": "not busy",
|
||||||
|
"1": "busy"
|
||||||
|
},
|
||||||
|
"tn": 3377,
|
||||||
|
"fp": 1178,
|
||||||
|
"fn": 492,
|
||||||
|
"tp": 663
|
||||||
|
},
|
||||||
|
"per_sensor": [
|
||||||
|
{
|
||||||
|
"sensor": "density-01",
|
||||||
|
"n": 714,
|
||||||
|
"positive_rate": 0.111,
|
||||||
|
"accuracy": 0.761,
|
||||||
|
"precision": 0.3,
|
||||||
|
"recall": 0.873,
|
||||||
|
"f1": 0.447
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"sensor": "hn-goetzentuem-01",
|
||||||
|
"n": 714,
|
||||||
|
"positive_rate": 0.197,
|
||||||
|
"accuracy": 0.732,
|
||||||
|
"precision": 0.391,
|
||||||
|
"recall": 0.638,
|
||||||
|
"f1": 0.485
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"sensor": "hn-hagenbuchersee-01",
|
||||||
|
"n": 714,
|
||||||
|
"positive_rate": 0.049,
|
||||||
|
"accuracy": 0.724,
|
||||||
|
"precision": 0.148,
|
||||||
|
"recall": 0.971,
|
||||||
|
"f1": 0.257
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"sensor": "hn-hagenbuchersee-02",
|
||||||
|
"n": 714,
|
||||||
|
"positive_rate": 0.123,
|
||||||
|
"accuracy": 0.783,
|
||||||
|
"precision": 0.355,
|
||||||
|
"recall": 0.932,
|
||||||
|
"f1": 0.514
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"sensor": "hn-marahaus-01",
|
||||||
|
"n": 714,
|
||||||
|
"positive_rate": 0.055,
|
||||||
|
"accuracy": 0.71,
|
||||||
|
"precision": 0.135,
|
||||||
|
"recall": 0.795,
|
||||||
|
"f1": 0.23
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"sensor": "hn-neckarufer-01",
|
||||||
|
"n": 713,
|
||||||
|
"positive_rate": 0.09,
|
||||||
|
"accuracy": 0.764,
|
||||||
|
"precision": 0.274,
|
||||||
|
"recall": 0.984,
|
||||||
|
"f1": 0.429
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"sensor": "hn-soleo-01",
|
||||||
|
"n": 714,
|
||||||
|
"positive_rate": 0.704,
|
||||||
|
"accuracy": 0.581,
|
||||||
|
"precision": 0.943,
|
||||||
|
"recall": 0.431,
|
||||||
|
"f1": 0.592
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"sensor": "hn-stadtgalerie-01",
|
||||||
|
"n": 713,
|
||||||
|
"positive_rate": 0.289,
|
||||||
|
"accuracy": 0.604,
|
||||||
|
"precision": 0.335,
|
||||||
|
"recall": 0.374,
|
||||||
|
"f1": 0.353
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
}
|
}
|
||||||
Binary file not shown.
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"model_version": "logit-20261114-040339",
|
"model_version": "logit-20261114-052723",
|
||||||
"trained_at": "2026-11-14T04:03:39Z",
|
"trained_at": "2026-11-14T05:27:23Z",
|
||||||
"features": [
|
"features": [
|
||||||
"hour",
|
"hour",
|
||||||
"weekday",
|
"weekday",
|
||||||
@@ -33,5 +33,89 @@
|
|||||||
"n_test": 5710,
|
"n_test": 5710,
|
||||||
"positive_rate": 0.202,
|
"positive_rate": 0.202,
|
||||||
"sensors": 8
|
"sensors": 8
|
||||||
}
|
},
|
||||||
|
"confusion_matrix": {
|
||||||
|
"labels": {
|
||||||
|
"0": "not busy",
|
||||||
|
"1": "busy"
|
||||||
|
},
|
||||||
|
"tn": 3377,
|
||||||
|
"fp": 1178,
|
||||||
|
"fn": 492,
|
||||||
|
"tp": 663
|
||||||
|
},
|
||||||
|
"per_sensor": [
|
||||||
|
{
|
||||||
|
"sensor": "density-01",
|
||||||
|
"n": 714,
|
||||||
|
"positive_rate": 0.111,
|
||||||
|
"accuracy": 0.761,
|
||||||
|
"precision": 0.3,
|
||||||
|
"recall": 0.873,
|
||||||
|
"f1": 0.447
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"sensor": "hn-goetzentuem-01",
|
||||||
|
"n": 714,
|
||||||
|
"positive_rate": 0.197,
|
||||||
|
"accuracy": 0.732,
|
||||||
|
"precision": 0.391,
|
||||||
|
"recall": 0.638,
|
||||||
|
"f1": 0.485
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"sensor": "hn-hagenbuchersee-01",
|
||||||
|
"n": 714,
|
||||||
|
"positive_rate": 0.049,
|
||||||
|
"accuracy": 0.724,
|
||||||
|
"precision": 0.148,
|
||||||
|
"recall": 0.971,
|
||||||
|
"f1": 0.257
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"sensor": "hn-hagenbuchersee-02",
|
||||||
|
"n": 714,
|
||||||
|
"positive_rate": 0.123,
|
||||||
|
"accuracy": 0.783,
|
||||||
|
"precision": 0.355,
|
||||||
|
"recall": 0.932,
|
||||||
|
"f1": 0.514
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"sensor": "hn-marahaus-01",
|
||||||
|
"n": 714,
|
||||||
|
"positive_rate": 0.055,
|
||||||
|
"accuracy": 0.71,
|
||||||
|
"precision": 0.135,
|
||||||
|
"recall": 0.795,
|
||||||
|
"f1": 0.23
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"sensor": "hn-neckarufer-01",
|
||||||
|
"n": 713,
|
||||||
|
"positive_rate": 0.09,
|
||||||
|
"accuracy": 0.764,
|
||||||
|
"precision": 0.274,
|
||||||
|
"recall": 0.984,
|
||||||
|
"f1": 0.429
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"sensor": "hn-soleo-01",
|
||||||
|
"n": 714,
|
||||||
|
"positive_rate": 0.704,
|
||||||
|
"accuracy": 0.581,
|
||||||
|
"precision": 0.943,
|
||||||
|
"recall": 0.431,
|
||||||
|
"f1": 0.592
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"sensor": "hn-stadtgalerie-01",
|
||||||
|
"n": 713,
|
||||||
|
"positive_rate": 0.289,
|
||||||
|
"accuracy": 0.604,
|
||||||
|
"precision": 0.335,
|
||||||
|
"recall": 0.374,
|
||||||
|
"f1": 0.353
|
||||||
|
}
|
||||||
|
]
|
||||||
}
|
}
|
||||||
Binary file not shown.
@@ -17,6 +17,7 @@ import pandas as pd
|
|||||||
from sklearn.linear_model import LogisticRegression
|
from sklearn.linear_model import LogisticRegression
|
||||||
from sklearn.metrics import (
|
from sklearn.metrics import (
|
||||||
accuracy_score,
|
accuracy_score,
|
||||||
|
confusion_matrix,
|
||||||
f1_score,
|
f1_score,
|
||||||
precision_score,
|
precision_score,
|
||||||
recall_score,
|
recall_score,
|
||||||
@@ -254,6 +255,28 @@ def train_small_from_csv(
|
|||||||
except ValueError:
|
except ValueError:
|
||||||
roc_auc = None
|
roc_auc = None
|
||||||
|
|
||||||
|
tn, fp, fn, tp = confusion_matrix(truth, predicted, labels=[0, 1]).ravel()
|
||||||
|
scored = test.assign(_predicted=predicted)
|
||||||
|
per_sensor = []
|
||||||
|
for sensor, group in scored.groupby("sensor", observed=True):
|
||||||
|
sensor_truth = group["busy"].to_numpy(dtype=int)
|
||||||
|
sensor_pred = group["_predicted"].to_numpy(dtype=int)
|
||||||
|
per_sensor.append(
|
||||||
|
{
|
||||||
|
"sensor": str(sensor),
|
||||||
|
"n": len(group),
|
||||||
|
"positive_rate": round(float(sensor_truth.mean()), 3),
|
||||||
|
"accuracy": round(float(accuracy_score(sensor_truth, sensor_pred)), 3),
|
||||||
|
"precision": round(
|
||||||
|
float(precision_score(sensor_truth, sensor_pred, zero_division=0)), 3
|
||||||
|
),
|
||||||
|
"recall": round(
|
||||||
|
float(recall_score(sensor_truth, sensor_pred, zero_division=0)), 3
|
||||||
|
),
|
||||||
|
"f1": round(float(f1_score(sensor_truth, sensor_pred, zero_division=0)), 3),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
trained_at = _utc_now()
|
trained_at = _utc_now()
|
||||||
model_version = f"logit-{trained_at[:10].replace('-', '')}-{trained_at[11:19].replace(':', '')}"
|
model_version = f"logit-{trained_at[:10].replace('-', '')}-{trained_at[11:19].replace(':', '')}"
|
||||||
metrics_payload = {
|
metrics_payload = {
|
||||||
@@ -278,6 +301,14 @@ def train_small_from_csv(
|
|||||||
"sensors": int(hourly["sensor"].nunique()),
|
"sensors": int(hourly["sensor"].nunique()),
|
||||||
}
|
}
|
||||||
),
|
),
|
||||||
|
"confusion_matrix": {
|
||||||
|
"labels": {"0": "not busy", "1": "busy"},
|
||||||
|
"tn": int(tn),
|
||||||
|
"fp": int(fp),
|
||||||
|
"fn": int(fn),
|
||||||
|
"tp": int(tp),
|
||||||
|
},
|
||||||
|
"per_sensor": per_sensor,
|
||||||
}
|
}
|
||||||
bundle = {
|
bundle = {
|
||||||
"model": model,
|
"model": model,
|
||||||
|
|||||||
+42
-4
@@ -142,6 +142,34 @@ def train_bundle(
|
|||||||
baseline = by_name["seasonal_naive"]
|
baseline = by_name["seasonal_naive"]
|
||||||
headline = by_name["week_weather"]
|
headline = by_name["week_weather"]
|
||||||
|
|
||||||
|
# Per-sensor holdout breakdown for the paper; same rows as the headline.
|
||||||
|
scored = eval_frame.copy()
|
||||||
|
scored["_model"] = preds["week_weather"]
|
||||||
|
scored["_baseline"] = preds["seasonal_naive"]
|
||||||
|
per_sensor = []
|
||||||
|
for sensor, group in scored.groupby("sensor", observed=True):
|
||||||
|
model_row = metrics(group["y"], group["_model"])
|
||||||
|
baseline_row = metrics(group["y"], group["_baseline"])
|
||||||
|
improvement = (
|
||||||
|
100.0 * (baseline_row["mae"] - model_row["mae"]) / baseline_row["mae"]
|
||||||
|
if baseline_row["mae"]
|
||||||
|
else 0.0
|
||||||
|
)
|
||||||
|
per_sensor.append(
|
||||||
|
{
|
||||||
|
"sensor": str(sensor),
|
||||||
|
"name": config.DISPLAY_NAMES.get(str(sensor), str(sensor)),
|
||||||
|
"n": model_row["n"],
|
||||||
|
"mae": round(model_row["mae"], 2),
|
||||||
|
"wape": round(model_row["wape"], 1),
|
||||||
|
"r2": round(model_row["r2"], 3),
|
||||||
|
"baseline_mae": round(baseline_row["mae"], 2),
|
||||||
|
"baseline_wape": round(baseline_row["wape"], 1),
|
||||||
|
"baseline_r2": round(baseline_row["r2"], 3),
|
||||||
|
"mae_improvement_pct": round(improvement, 1),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
# Production model: refit on all observed rows after the honest holdout.
|
# Production model: refit on all observed rows after the honest holdout.
|
||||||
production = fit(features, config.FEATURE_WEEK_WEATHER)
|
production = fit(features, config.FEATURE_WEEK_WEATHER)
|
||||||
residual = residual_quantiles(eval_frame, preds["week_weather"])
|
residual = residual_quantiles(eval_frame, preds["week_weather"])
|
||||||
@@ -207,6 +235,7 @@ def train_bundle(
|
|||||||
"features": list(config.FEATURE_WEEK_WEATHER),
|
"features": list(config.FEATURE_WEEK_WEATHER),
|
||||||
"holiday_effect": holiday_effect,
|
"holiday_effect": holiday_effect,
|
||||||
"ablation": [_round_row(row) for row in results],
|
"ablation": [_round_row(row) for row in results],
|
||||||
|
"per_sensor": per_sensor,
|
||||||
"limits": list(config.LIMITS),
|
"limits": list(config.LIMITS),
|
||||||
}
|
}
|
||||||
return bundle, metrics_payload, results
|
return bundle, metrics_payload, results
|
||||||
@@ -267,10 +296,6 @@ def run_train() -> dict:
|
|||||||
bundle, metrics_payload, _ = train_bundle(hourly, features, archive, meta, audit)
|
bundle, metrics_payload, _ = train_bundle(hourly, features, archive, meta, audit)
|
||||||
joblib.dump(bundle, artifacts / "model.joblib")
|
joblib.dump(bundle, artifacts / "model.joblib")
|
||||||
|
|
||||||
(artifacts / "metrics.json").write_text(
|
|
||||||
json.dumps(metrics_payload, indent=2, ensure_ascii=False) + "\n"
|
|
||||||
)
|
|
||||||
|
|
||||||
start = pd.Timestamp.now(tz="UTC").ceil("h")
|
start = pd.Timestamp.now(tz="UTC").ceil("h")
|
||||||
forecasts = _export_forecasts(bundle, forecast, start)
|
forecasts = _export_forecasts(bundle, forecast, start)
|
||||||
(artifacts / "forecasts.json").write_text(
|
(artifacts / "forecasts.json").write_text(
|
||||||
@@ -289,6 +314,19 @@ def run_train() -> dict:
|
|||||||
json.dumps(small_metrics, indent=2, ensure_ascii=False) + "\n"
|
json.dumps(small_metrics, indent=2, ensure_ascii=False) + "\n"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# One self-contained file for the paper: ablation + per-sensor + small model.
|
||||||
|
metrics_payload["small"] = {
|
||||||
|
"model_version": small_metrics["model_version"],
|
||||||
|
"trained_at": small_metrics["trained_at"],
|
||||||
|
"features": small_metrics["features"],
|
||||||
|
"metrics": small_metrics["metrics"],
|
||||||
|
"confusion_matrix": small_metrics["confusion_matrix"],
|
||||||
|
"per_sensor": small_metrics["per_sensor"],
|
||||||
|
}
|
||||||
|
(artifacts / "metrics.json").write_text(
|
||||||
|
json.dumps(metrics_payload, indent=2, ensure_ascii=False) + "\n"
|
||||||
|
)
|
||||||
|
|
||||||
log.info(
|
log.info(
|
||||||
"wrote metrics.json, forecasts.json (%s..%s), model.joblib, small_model.joblib",
|
"wrote metrics.json, forecasts.json (%s..%s), model.joblib, small_model.joblib",
|
||||||
start,
|
start,
|
||||||
|
|||||||
@@ -67,6 +67,24 @@ def test_metrics_and_audit():
|
|||||||
assert metrics["metrics"]["wape"] < metrics["metrics"]["baseline_wape"]
|
assert metrics["metrics"]["wape"] < metrics["metrics"]["baseline_wape"]
|
||||||
assert len(metrics["features"]) == 10
|
assert len(metrics["features"]) == 10
|
||||||
assert len(metrics["limits"]) >= 4
|
assert len(metrics["limits"]) >= 4
|
||||||
|
assert len(metrics["per_sensor"]) == 8
|
||||||
|
assert sum(row["n"] for row in metrics["per_sensor"]) == metrics["metrics"]["n_eval"]
|
||||||
|
assert {row["sensor"] for row in metrics["per_sensor"]} == set(config.USABLE_SENSORS)
|
||||||
|
for row in metrics["per_sensor"]:
|
||||||
|
assert row["mae"] >= 0
|
||||||
|
assert row["baseline_mae"] >= 0
|
||||||
|
assert 0 <= row["wape"] < 100
|
||||||
|
assert row["mae_improvement_pct"] > 0
|
||||||
|
|
||||||
|
small = _load("small_metrics.json")
|
||||||
|
matrix = small["confusion_matrix"]
|
||||||
|
total = matrix["tn"] + matrix["fp"] + matrix["fn"] + matrix["tp"]
|
||||||
|
assert total == small["metrics"]["n_test"]
|
||||||
|
assert sum(row["n"] for row in small["per_sensor"]) == total
|
||||||
|
recomputed = (matrix["tn"] + matrix["tp"]) / total
|
||||||
|
assert abs(recomputed - small["metrics"]["accuracy"]) < 0.001
|
||||||
|
assert metrics["small"]["metrics"] == small["metrics"]
|
||||||
|
assert set(matrix) == {"labels", "tn", "fp", "fn", "tp"}
|
||||||
|
|
||||||
audit = _load("data_audit.json")
|
audit = _load("data_audit.json")
|
||||||
assert audit["all_checks_ok"] is True
|
assert audit["all_checks_ok"] is True
|
||||||
|
|||||||
Reference in new issue
Block a user