feat: per-sensor holdout metrics and small-model confusion matrix
- train.py emits per-sensor MAE/WAPE/R2 vs the seasonal-naive baseline - small.py emits the full confusion matrix and per-sensor classifier metrics - metrics.json now carries the small-model summary for paper support - regenerate baked artifacts; tests assert the tables are internally consistent
This commit is contained in:
1 parent
4ea7212b70
commit
000aaed7a5
8 files changed
+386
-10
No files matched your search
+207
-2
@@ -19,7 +19,7 @@
|
||||
"baseline_r2": 0.462,
|
||||
"n_eval": 5350
|
||||
},
|
||||
"trained_at": "2026-11-14T04:03:37Z",
|
||||
"trained_at": "2026-11-14T05:27:21Z",
|
||||
"train_from": "2026-06-12T00:00:00Z",
|
||||
"train_to": "2026-11-07T23:00:00Z",
|
||||
"test_from": "2026-10-10T23:00:00Z",
|
||||
@@ -90,11 +90,216 @@
|
||||
"n": 5350
|
||||
}
|
||||
],
|
||||
"per_sensor": [
|
||||
{
|
||||
"sensor": "density-01",
|
||||
"name": "Density 01",
|
||||
"n": 669,
|
||||
"mae": 2.26,
|
||||
"wape": 30.8,
|
||||
"r2": 0.722,
|
||||
"baseline_mae": 3.58,
|
||||
"baseline_wape": 48.8,
|
||||
"baseline_r2": 0.238,
|
||||
"mae_improvement_pct": 36.8
|
||||
},
|
||||
{
|
||||
"sensor": "hn-goetzentuem-01",
|
||||
"name": "Götzenturm",
|
||||
"n": 669,
|
||||
"mae": 2.4,
|
||||
"wape": 26.9,
|
||||
"r2": 0.718,
|
||||
"baseline_mae": 3.84,
|
||||
"baseline_wape": 43.0,
|
||||
"baseline_r2": 0.281,
|
||||
"mae_improvement_pct": 37.5
|
||||
},
|
||||
{
|
||||
"sensor": "hn-hagenbuchersee-01",
|
||||
"name": "Hagenbuchersee 1",
|
||||
"n": 669,
|
||||
"mae": 2.47,
|
||||
"wape": 38.6,
|
||||
"r2": 0.602,
|
||||
"baseline_mae": 4.01,
|
||||
"baseline_wape": 62.8,
|
||||
"baseline_r2": -0.212,
|
||||
"mae_improvement_pct": 38.5
|
||||
},
|
||||
{
|
||||
"sensor": "hn-hagenbuchersee-02",
|
||||
"name": "Hagenbuchersee 2",
|
||||
"n": 669,
|
||||
"mae": 4.12,
|
||||
"wape": 31.1,
|
||||
"r2": 0.731,
|
||||
"baseline_mae": 6.49,
|
||||
"baseline_wape": 48.9,
|
||||
"baseline_r2": 0.337,
|
||||
"mae_improvement_pct": 36.5
|
||||
},
|
||||
{
|
||||
"sensor": "hn-marahaus-01",
|
||||
"name": "Marahaus",
|
||||
"n": 669,
|
||||
"mae": 2.67,
|
||||
"wape": 29.8,
|
||||
"r2": 0.671,
|
||||
"baseline_mae": 4.57,
|
||||
"baseline_wape": 51.0,
|
||||
"baseline_r2": -0.045,
|
||||
"mae_improvement_pct": 41.7
|
||||
},
|
||||
{
|
||||
"sensor": "hn-neckarufer-01",
|
||||
"name": "Neckarufer",
|
||||
"n": 668,
|
||||
"mae": 3.63,
|
||||
"wape": 28.2,
|
||||
"r2": 0.695,
|
||||
"baseline_mae": 5.89,
|
||||
"baseline_wape": 45.8,
|
||||
"baseline_r2": 0.11,
|
||||
"mae_improvement_pct": 38.4
|
||||
},
|
||||
{
|
||||
"sensor": "hn-soleo-01",
|
||||
"name": "Soleo",
|
||||
"n": 669,
|
||||
"mae": 3.19,
|
||||
"wape": 20.2,
|
||||
"r2": 0.548,
|
||||
"baseline_mae": 4.4,
|
||||
"baseline_wape": 28.0,
|
||||
"baseline_r2": 0.163,
|
||||
"mae_improvement_pct": 27.7
|
||||
},
|
||||
{
|
||||
"sensor": "hn-stadtgalerie-01",
|
||||
"name": "Stadtgalerie",
|
||||
"n": 668,
|
||||
"mae": 5.83,
|
||||
"wape": 25.3,
|
||||
"r2": 0.765,
|
||||
"baseline_mae": 10.39,
|
||||
"baseline_wape": 45.1,
|
||||
"baseline_r2": 0.48,
|
||||
"mae_improvement_pct": 43.9
|
||||
}
|
||||
],
|
||||
"limits": [
|
||||
"estimated devices, not people",
|
||||
"8 sensors in a ~580 m inner-city cluster",
|
||||
"5 months of multi-sensor data; no annual seasonality claim",
|
||||
"observed weather in training, forecast weather in scenarios",
|
||||
"holidays are context, not a large accuracy claim"
|
||||
]
|
||||
],
|
||||
"small": {
|
||||
"model_version": "logit-20261114-052723",
|
||||
"trained_at": "2026-11-14T05:27:23Z",
|
||||
"features": [
|
||||
"hour",
|
||||
"weekday",
|
||||
"is_weekend",
|
||||
"temp_c",
|
||||
"precip_mm",
|
||||
"is_holiday"
|
||||
],
|
||||
"metrics": {
|
||||
"accuracy": 0.708,
|
||||
"precision": 0.36,
|
||||
"recall": 0.574,
|
||||
"f1": 0.443,
|
||||
"roc_auc": 0.726,
|
||||
"n_train": 18242,
|
||||
"n_test": 5710,
|
||||
"positive_rate": 0.202,
|
||||
"sensors": 8
|
||||
},
|
||||
"confusion_matrix": {
|
||||
"labels": {
|
||||
"0": "not busy",
|
||||
"1": "busy"
|
||||
},
|
||||
"tn": 3377,
|
||||
"fp": 1178,
|
||||
"fn": 492,
|
||||
"tp": 663
|
||||
},
|
||||
"per_sensor": [
|
||||
{
|
||||
"sensor": "density-01",
|
||||
"n": 714,
|
||||
"positive_rate": 0.111,
|
||||
"accuracy": 0.761,
|
||||
"precision": 0.3,
|
||||
"recall": 0.873,
|
||||
"f1": 0.447
|
||||
},
|
||||
{
|
||||
"sensor": "hn-goetzentuem-01",
|
||||
"n": 714,
|
||||
"positive_rate": 0.197,
|
||||
"accuracy": 0.732,
|
||||
"precision": 0.391,
|
||||
"recall": 0.638,
|
||||
"f1": 0.485
|
||||
},
|
||||
{
|
||||
"sensor": "hn-hagenbuchersee-01",
|
||||
"n": 714,
|
||||
"positive_rate": 0.049,
|
||||
"accuracy": 0.724,
|
||||
"precision": 0.148,
|
||||
"recall": 0.971,
|
||||
"f1": 0.257
|
||||
},
|
||||
{
|
||||
"sensor": "hn-hagenbuchersee-02",
|
||||
"n": 714,
|
||||
"positive_rate": 0.123,
|
||||
"accuracy": 0.783,
|
||||
"precision": 0.355,
|
||||
"recall": 0.932,
|
||||
"f1": 0.514
|
||||
},
|
||||
{
|
||||
"sensor": "hn-marahaus-01",
|
||||
"n": 714,
|
||||
"positive_rate": 0.055,
|
||||
"accuracy": 0.71,
|
||||
"precision": 0.135,
|
||||
"recall": 0.795,
|
||||
"f1": 0.23
|
||||
},
|
||||
{
|
||||
"sensor": "hn-neckarufer-01",
|
||||
"n": 713,
|
||||
"positive_rate": 0.09,
|
||||
"accuracy": 0.764,
|
||||
"precision": 0.274,
|
||||
"recall": 0.984,
|
||||
"f1": 0.429
|
||||
},
|
||||
{
|
||||
"sensor": "hn-soleo-01",
|
||||
"n": 714,
|
||||
"positive_rate": 0.704,
|
||||
"accuracy": 0.581,
|
||||
"precision": 0.943,
|
||||
"recall": 0.431,
|
||||
"f1": 0.592
|
||||
},
|
||||
{
|
||||
"sensor": "hn-stadtgalerie-01",
|
||||
"n": 713,
|
||||
"positive_rate": 0.289,
|
||||
"accuracy": 0.604,
|
||||
"precision": 0.335,
|
||||
"recall": 0.374,
|
||||
"f1": 0.353
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user