feat: per-sensor holdout metrics and small-model confusion matrix

- train.py emits per-sensor MAE/WAPE/R2 vs the seasonal-naive baseline
- small.py emits the full confusion matrix and per-sensor classifier metrics
- metrics.json now carries the small-model summary for paper support
- regenerate baked artifacts; tests assert the tables are internally consistent
This commit is contained in:
Noel Sabia committed 2026-11-14 06:35:14 +01:00
1 parent 4ea7212b70
commit 000aaed7a5
8 files changed
+386 -10

No files matched your search

+31
View File
@@ -17,6 +17,7 @@ import pandas as pd
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import (
accuracy_score,
confusion_matrix,
f1_score,
precision_score,
recall_score,
@@ -254,6 +255,28 @@ def train_small_from_csv(
except ValueError:
roc_auc = None
tn, fp, fn, tp = confusion_matrix(truth, predicted, labels=[0, 1]).ravel()
scored = test.assign(_predicted=predicted)
per_sensor = []
for sensor, group in scored.groupby("sensor", observed=True):
sensor_truth = group["busy"].to_numpy(dtype=int)
sensor_pred = group["_predicted"].to_numpy(dtype=int)
per_sensor.append(
{
"sensor": str(sensor),
"n": len(group),
"positive_rate": round(float(sensor_truth.mean()), 3),
"accuracy": round(float(accuracy_score(sensor_truth, sensor_pred)), 3),
"precision": round(
float(precision_score(sensor_truth, sensor_pred, zero_division=0)), 3
),
"recall": round(
float(recall_score(sensor_truth, sensor_pred, zero_division=0)), 3
),
"f1": round(float(f1_score(sensor_truth, sensor_pred, zero_division=0)), 3),
}
)
trained_at = _utc_now()
model_version = f"logit-{trained_at[:10].replace('-', '')}-{trained_at[11:19].replace(':', '')}"
metrics_payload = {
@@ -278,6 +301,14 @@ def train_small_from_csv(
"sensors": int(hourly["sensor"].nunique()),
}
),
"confusion_matrix": {
"labels": {"0": "not busy", "1": "busy"},
"tn": int(tn),
"fp": int(fp),
"fn": int(fn),
"tp": int(tp),
},
"per_sensor": per_sensor,
}
bundle = {
"model": model,
+42 -4
View File
@@ -142,6 +142,34 @@ def train_bundle(
baseline = by_name["seasonal_naive"]
headline = by_name["week_weather"]
# Per-sensor holdout breakdown for the paper; same rows as the headline.
scored = eval_frame.copy()
scored["_model"] = preds["week_weather"]
scored["_baseline"] = preds["seasonal_naive"]
per_sensor = []
for sensor, group in scored.groupby("sensor", observed=True):
model_row = metrics(group["y"], group["_model"])
baseline_row = metrics(group["y"], group["_baseline"])
improvement = (
100.0 * (baseline_row["mae"] - model_row["mae"]) / baseline_row["mae"]
if baseline_row["mae"]
else 0.0
)
per_sensor.append(
{
"sensor": str(sensor),
"name": config.DISPLAY_NAMES.get(str(sensor), str(sensor)),
"n": model_row["n"],
"mae": round(model_row["mae"], 2),
"wape": round(model_row["wape"], 1),
"r2": round(model_row["r2"], 3),
"baseline_mae": round(baseline_row["mae"], 2),
"baseline_wape": round(baseline_row["wape"], 1),
"baseline_r2": round(baseline_row["r2"], 3),
"mae_improvement_pct": round(improvement, 1),
}
)
# Production model: refit on all observed rows after the honest holdout.
production = fit(features, config.FEATURE_WEEK_WEATHER)
residual = residual_quantiles(eval_frame, preds["week_weather"])
@@ -207,6 +235,7 @@ def train_bundle(
"features": list(config.FEATURE_WEEK_WEATHER),
"holiday_effect": holiday_effect,
"ablation": [_round_row(row) for row in results],
"per_sensor": per_sensor,
"limits": list(config.LIMITS),
}
return bundle, metrics_payload, results
@@ -267,10 +296,6 @@ def run_train() -> dict:
bundle, metrics_payload, _ = train_bundle(hourly, features, archive, meta, audit)
joblib.dump(bundle, artifacts / "model.joblib")
(artifacts / "metrics.json").write_text(
json.dumps(metrics_payload, indent=2, ensure_ascii=False) + "\n"
)
start = pd.Timestamp.now(tz="UTC").ceil("h")
forecasts = _export_forecasts(bundle, forecast, start)
(artifacts / "forecasts.json").write_text(
@@ -289,6 +314,19 @@ def run_train() -> dict:
json.dumps(small_metrics, indent=2, ensure_ascii=False) + "\n"
)
# One self-contained file for the paper: ablation + per-sensor + small model.
metrics_payload["small"] = {
"model_version": small_metrics["model_version"],
"trained_at": small_metrics["trained_at"],
"features": small_metrics["features"],
"metrics": small_metrics["metrics"],
"confusion_matrix": small_metrics["confusion_matrix"],
"per_sensor": small_metrics["per_sensor"],
}
(artifacts / "metrics.json").write_text(
json.dumps(metrics_payload, indent=2, ensure_ascii=False) + "\n"
)
log.info(
"wrote metrics.json, forecasts.json (%s..%s), model.joblib, small_model.joblib",
start,