Files
Noel Sabia 000aaed7a5 feat: per-sensor holdout metrics and small-model confusion matrix
- train.py emits per-sensor MAE/WAPE/R2 vs the seasonal-naive baseline
- small.py emits the full confusion matrix and per-sensor classifier metrics
- metrics.json now carries the small-model summary for paper support
- regenerate baked artifacts; tests assert the tables are internally consistent
2026-11-14 06:35:14 +01:00

306 lines
6.8 KiB
JSON

{
"data_version": "2026-11-07T23:51:00Z",
"model_version": "histgb-2026-11-14",
"context_version": "2026-11-14T03:47:09Z",
"coverage": {
"sensors": 8,
"from": "2026-06-12T00:00:00Z",
"to": "2026-11-07T23:51:00Z",
"cadence_min": 15
},
"metrics": {
"model": "weather-7d",
"mae": 3.32,
"wape": 27.5,
"r2": 0.777,
"baseline": "seasonal_naive",
"baseline_mae": 5.4,
"baseline_wape": 44.7,
"baseline_r2": 0.462,
"n_eval": 5350
},
"trained_at": "2026-11-14T05:27:21Z",
"train_from": "2026-06-12T00:00:00Z",
"train_to": "2026-11-07T23:00:00Z",
"test_from": "2026-10-10T23:00:00Z",
"test_to": "2026-11-07T23:00:00Z",
"features": [
"hour",
"weekday",
"month",
"is_weekend",
"is_holiday",
"sensor_code",
"lag_week",
"seasonal_mean_4w",
"temp_c",
"precip_mm"
],
"holiday_effect": {
"mae_delta": -0.12,
"note": "Holiday flag alone changes MAE by this amount on the holdout; weather is the main driver, holidays are context."
},
"ablation": [
{
"name": "seasonal_naive",
"description": "exact 168-row lag (same observed hour last week)",
"mae": 5.4,
"wape": 44.7,
"r2": 0.462,
"n": 5350
},
{
"name": "day_no_weather",
"description": "calendar + lags 24/48/168 + 24h/168h rolling means",
"mae": 3.96,
"wape": 32.8,
"r2": 0.706,
"n": 5350
},
{
"name": "day_holiday",
"description": "day-ahead + Baden-Württemberg holiday flag",
"mae": 3.84,
"wape": 31.8,
"r2": 0.732,
"n": 5350
},
{
"name": "day_weather",
"description": "day-ahead + temperature + rain (observed weather)",
"mae": 3.36,
"wape": 27.8,
"r2": 0.763,
"n": 5350
},
{
"name": "week_no_weather",
"description": "7-day-safe calendar + previous same weekday/hour values",
"mae": 4.08,
"wape": 33.8,
"r2": 0.697,
"n": 5350
},
{
"name": "week_weather",
"description": "7-day-safe + temperature + rain (production model)",
"mae": 3.32,
"wape": 27.5,
"r2": 0.777,
"n": 5350
}
],
"per_sensor": [
{
"sensor": "density-01",
"name": "Density 01",
"n": 669,
"mae": 2.26,
"wape": 30.8,
"r2": 0.722,
"baseline_mae": 3.58,
"baseline_wape": 48.8,
"baseline_r2": 0.238,
"mae_improvement_pct": 36.8
},
{
"sensor": "hn-goetzentuem-01",
"name": "Götzenturm",
"n": 669,
"mae": 2.4,
"wape": 26.9,
"r2": 0.718,
"baseline_mae": 3.84,
"baseline_wape": 43.0,
"baseline_r2": 0.281,
"mae_improvement_pct": 37.5
},
{
"sensor": "hn-hagenbuchersee-01",
"name": "Hagenbuchersee 1",
"n": 669,
"mae": 2.47,
"wape": 38.6,
"r2": 0.602,
"baseline_mae": 4.01,
"baseline_wape": 62.8,
"baseline_r2": -0.212,
"mae_improvement_pct": 38.5
},
{
"sensor": "hn-hagenbuchersee-02",
"name": "Hagenbuchersee 2",
"n": 669,
"mae": 4.12,
"wape": 31.1,
"r2": 0.731,
"baseline_mae": 6.49,
"baseline_wape": 48.9,
"baseline_r2": 0.337,
"mae_improvement_pct": 36.5
},
{
"sensor": "hn-marahaus-01",
"name": "Marahaus",
"n": 669,
"mae": 2.67,
"wape": 29.8,
"r2": 0.671,
"baseline_mae": 4.57,
"baseline_wape": 51.0,
"baseline_r2": -0.045,
"mae_improvement_pct": 41.7
},
{
"sensor": "hn-neckarufer-01",
"name": "Neckarufer",
"n": 668,
"mae": 3.63,
"wape": 28.2,
"r2": 0.695,
"baseline_mae": 5.89,
"baseline_wape": 45.8,
"baseline_r2": 0.11,
"mae_improvement_pct": 38.4
},
{
"sensor": "hn-soleo-01",
"name": "Soleo",
"n": 669,
"mae": 3.19,
"wape": 20.2,
"r2": 0.548,
"baseline_mae": 4.4,
"baseline_wape": 28.0,
"baseline_r2": 0.163,
"mae_improvement_pct": 27.7
},
{
"sensor": "hn-stadtgalerie-01",
"name": "Stadtgalerie",
"n": 668,
"mae": 5.83,
"wape": 25.3,
"r2": 0.765,
"baseline_mae": 10.39,
"baseline_wape": 45.1,
"baseline_r2": 0.48,
"mae_improvement_pct": 43.9
}
],
"limits": [
"estimated devices, not people",
"8 sensors in a ~580 m inner-city cluster",
"5 months of multi-sensor data; no annual seasonality claim",
"observed weather in training, forecast weather in scenarios",
"holidays are context, not a large accuracy claim"
],
"small": {
"model_version": "logit-20261114-052723",
"trained_at": "2026-11-14T05:27:23Z",
"features": [
"hour",
"weekday",
"is_weekend",
"temp_c",
"precip_mm",
"is_holiday"
],
"metrics": {
"accuracy": 0.708,
"precision": 0.36,
"recall": 0.574,
"f1": 0.443,
"roc_auc": 0.726,
"n_train": 18242,
"n_test": 5710,
"positive_rate": 0.202,
"sensors": 8
},
"confusion_matrix": {
"labels": {
"0": "not busy",
"1": "busy"
},
"tn": 3377,
"fp": 1178,
"fn": 492,
"tp": 663
},
"per_sensor": [
{
"sensor": "density-01",
"n": 714,
"positive_rate": 0.111,
"accuracy": 0.761,
"precision": 0.3,
"recall": 0.873,
"f1": 0.447
},
{
"sensor": "hn-goetzentuem-01",
"n": 714,
"positive_rate": 0.197,
"accuracy": 0.732,
"precision": 0.391,
"recall": 0.638,
"f1": 0.485
},
{
"sensor": "hn-hagenbuchersee-01",
"n": 714,
"positive_rate": 0.049,
"accuracy": 0.724,
"precision": 0.148,
"recall": 0.971,
"f1": 0.257
},
{
"sensor": "hn-hagenbuchersee-02",
"n": 714,
"positive_rate": 0.123,
"accuracy": 0.783,
"precision": 0.355,
"recall": 0.932,
"f1": 0.514
},
{
"sensor": "hn-marahaus-01",
"n": 714,
"positive_rate": 0.055,
"accuracy": 0.71,
"precision": 0.135,
"recall": 0.795,
"f1": 0.23
},
{
"sensor": "hn-neckarufer-01",
"n": 713,
"positive_rate": 0.09,
"accuracy": 0.764,
"precision": 0.274,
"recall": 0.984,
"f1": 0.429
},
{
"sensor": "hn-soleo-01",
"n": 714,
"positive_rate": 0.704,
"accuracy": 0.581,
"precision": 0.943,
"recall": 0.431,
"f1": 0.592
},
{
"sensor": "hn-stadtgalerie-01",
"n": 713,
"positive_rate": 0.289,
"accuracy": 0.604,
"precision": 0.335,
"recall": 0.374,
"f1": 0.353
}
]
}
}