feat: per-sensor holdout metrics and small-model confusion matrix
- train.py emits per-sensor MAE/WAPE/R2 vs the seasonal-naive baseline - small.py emits the full confusion matrix and per-sensor classifier metrics - metrics.json now carries the small-model summary for paper support - regenerate baked artifacts; tests assert the tables are internally consistent
This commit is contained in:
1 parent
4ea7212b70
commit
000aaed7a5
8 files changed
+386
-10
No files matched your search
@@ -17,6 +17,7 @@ import pandas as pd
|
||||
from sklearn.linear_model import LogisticRegression
|
||||
from sklearn.metrics import (
|
||||
accuracy_score,
|
||||
confusion_matrix,
|
||||
f1_score,
|
||||
precision_score,
|
||||
recall_score,
|
||||
@@ -254,6 +255,28 @@ def train_small_from_csv(
|
||||
except ValueError:
|
||||
roc_auc = None
|
||||
|
||||
tn, fp, fn, tp = confusion_matrix(truth, predicted, labels=[0, 1]).ravel()
|
||||
scored = test.assign(_predicted=predicted)
|
||||
per_sensor = []
|
||||
for sensor, group in scored.groupby("sensor", observed=True):
|
||||
sensor_truth = group["busy"].to_numpy(dtype=int)
|
||||
sensor_pred = group["_predicted"].to_numpy(dtype=int)
|
||||
per_sensor.append(
|
||||
{
|
||||
"sensor": str(sensor),
|
||||
"n": len(group),
|
||||
"positive_rate": round(float(sensor_truth.mean()), 3),
|
||||
"accuracy": round(float(accuracy_score(sensor_truth, sensor_pred)), 3),
|
||||
"precision": round(
|
||||
float(precision_score(sensor_truth, sensor_pred, zero_division=0)), 3
|
||||
),
|
||||
"recall": round(
|
||||
float(recall_score(sensor_truth, sensor_pred, zero_division=0)), 3
|
||||
),
|
||||
"f1": round(float(f1_score(sensor_truth, sensor_pred, zero_division=0)), 3),
|
||||
}
|
||||
)
|
||||
|
||||
trained_at = _utc_now()
|
||||
model_version = f"logit-{trained_at[:10].replace('-', '')}-{trained_at[11:19].replace(':', '')}"
|
||||
metrics_payload = {
|
||||
@@ -278,6 +301,14 @@ def train_small_from_csv(
|
||||
"sensors": int(hourly["sensor"].nunique()),
|
||||
}
|
||||
),
|
||||
"confusion_matrix": {
|
||||
"labels": {"0": "not busy", "1": "busy"},
|
||||
"tn": int(tn),
|
||||
"fp": int(fp),
|
||||
"fn": int(fn),
|
||||
"tp": int(tp),
|
||||
},
|
||||
"per_sensor": per_sensor,
|
||||
}
|
||||
bundle = {
|
||||
"model": model,
|
||||
|
||||
+42
-4
@@ -142,6 +142,34 @@ def train_bundle(
|
||||
baseline = by_name["seasonal_naive"]
|
||||
headline = by_name["week_weather"]
|
||||
|
||||
# Per-sensor holdout breakdown for the paper; same rows as the headline.
|
||||
scored = eval_frame.copy()
|
||||
scored["_model"] = preds["week_weather"]
|
||||
scored["_baseline"] = preds["seasonal_naive"]
|
||||
per_sensor = []
|
||||
for sensor, group in scored.groupby("sensor", observed=True):
|
||||
model_row = metrics(group["y"], group["_model"])
|
||||
baseline_row = metrics(group["y"], group["_baseline"])
|
||||
improvement = (
|
||||
100.0 * (baseline_row["mae"] - model_row["mae"]) / baseline_row["mae"]
|
||||
if baseline_row["mae"]
|
||||
else 0.0
|
||||
)
|
||||
per_sensor.append(
|
||||
{
|
||||
"sensor": str(sensor),
|
||||
"name": config.DISPLAY_NAMES.get(str(sensor), str(sensor)),
|
||||
"n": model_row["n"],
|
||||
"mae": round(model_row["mae"], 2),
|
||||
"wape": round(model_row["wape"], 1),
|
||||
"r2": round(model_row["r2"], 3),
|
||||
"baseline_mae": round(baseline_row["mae"], 2),
|
||||
"baseline_wape": round(baseline_row["wape"], 1),
|
||||
"baseline_r2": round(baseline_row["r2"], 3),
|
||||
"mae_improvement_pct": round(improvement, 1),
|
||||
}
|
||||
)
|
||||
|
||||
# Production model: refit on all observed rows after the honest holdout.
|
||||
production = fit(features, config.FEATURE_WEEK_WEATHER)
|
||||
residual = residual_quantiles(eval_frame, preds["week_weather"])
|
||||
@@ -207,6 +235,7 @@ def train_bundle(
|
||||
"features": list(config.FEATURE_WEEK_WEATHER),
|
||||
"holiday_effect": holiday_effect,
|
||||
"ablation": [_round_row(row) for row in results],
|
||||
"per_sensor": per_sensor,
|
||||
"limits": list(config.LIMITS),
|
||||
}
|
||||
return bundle, metrics_payload, results
|
||||
@@ -267,10 +296,6 @@ def run_train() -> dict:
|
||||
bundle, metrics_payload, _ = train_bundle(hourly, features, archive, meta, audit)
|
||||
joblib.dump(bundle, artifacts / "model.joblib")
|
||||
|
||||
(artifacts / "metrics.json").write_text(
|
||||
json.dumps(metrics_payload, indent=2, ensure_ascii=False) + "\n"
|
||||
)
|
||||
|
||||
start = pd.Timestamp.now(tz="UTC").ceil("h")
|
||||
forecasts = _export_forecasts(bundle, forecast, start)
|
||||
(artifacts / "forecasts.json").write_text(
|
||||
@@ -289,6 +314,19 @@ def run_train() -> dict:
|
||||
json.dumps(small_metrics, indent=2, ensure_ascii=False) + "\n"
|
||||
)
|
||||
|
||||
# One self-contained file for the paper: ablation + per-sensor + small model.
|
||||
metrics_payload["small"] = {
|
||||
"model_version": small_metrics["model_version"],
|
||||
"trained_at": small_metrics["trained_at"],
|
||||
"features": small_metrics["features"],
|
||||
"metrics": small_metrics["metrics"],
|
||||
"confusion_matrix": small_metrics["confusion_matrix"],
|
||||
"per_sensor": small_metrics["per_sensor"],
|
||||
}
|
||||
(artifacts / "metrics.json").write_text(
|
||||
json.dumps(metrics_payload, indent=2, ensure_ascii=False) + "\n"
|
||||
)
|
||||
|
||||
log.info(
|
||||
"wrote metrics.json, forecasts.json (%s..%s), model.joblib, small_model.joblib",
|
||||
start,
|
||||
|
||||
Reference in new issue
Block a user