Files
ML-py/scripts/plot_metrics.py
Noel Sabia 2bcf66c550 feat: repeatable small-model, figures, and training e2e scripts
- make small trains from a CSV and prints metrics + confusion matrix
- make figures renders dependency-free SVGs from metrics.json
- scripts/e2e_train.py boots a real server and checks snapshot, robustness,
  auth, one-job-at-a-time, and both training jobs with hot-swap
- CI/ruff/compileall now include scripts/
2026-11-14 06:35:19 +01:00

263 lines
9.4 KiB
Python

#!/usr/bin/env python3
"""Render dependency-free SVG figures from ``artifacts/metrics.json``.
The paper figures are generated from the measured holdout metrics, never
hand-written. Run ``make figures`` after ``make train``.
"""
from __future__ import annotations
import argparse
import json
from pathlib import Path
from xml.sax.saxutils import escape
from pulsecast import config
MODEL_COLOR = "#2563eb"
BASELINE_COLOR = "#94a3b8"
PRODUCTION_COLOR = "#0f766e"
TEXT_COLOR = "#0f172a"
GRID_COLOR = "#e2e8f0"
def _svg_document(
title: str,
subtitle: str,
body: list[str],
width: int,
height: int,
footer: str = "Source: artifacts/metrics.json · PulseCast 28-day time-based holdout",
) -> str:
return "\n".join(
[
f'<svg xmlns="http://www.w3.org/2000/svg" width="{width}" height="{height}" '
f'viewBox="0 0 {width} {height}" role="img" aria-label="{escape(title)}">',
'<rect width="100%" height="100%" fill="#ffffff"/>',
'<g font-family="-apple-system, Segoe UI, Helvetica, Arial, sans-serif">',
f'<text x="24" y="34" font-size="20" font-weight="600" fill="{TEXT_COLOR}">'
f"{escape(title)}</text>",
f'<text x="24" y="56" font-size="13" fill="#475569">{escape(subtitle)}</text>',
*body,
f'<text x="24" y="{height - 12}" font-size="11" fill="#64748b">'
f"{escape(footer)}</text>",
"</g>",
"</svg>",
"",
]
)
def _bar_chart(
title: str,
subtitle: str,
rows: list[dict],
*,
value_key: str = "mae",
secondary_key: str | None = None,
secondary_label: str | None = None,
primary_label: str = "MAE",
production: str | None = None,
) -> str:
width = 960
left = 230
right = 96
top = 86
row_height = 38
height = top + row_height * len(rows) + 56
plot_width = width - left - right
values = [row[value_key] for row in rows]
if secondary_key:
values += [row[secondary_key] for row in rows]
maximum = max(values) * 1.08 if values else 1.0
body: list[str] = []
for tick in range(5):
x = left + plot_width * tick / 4
value = maximum * tick / 4
body.append(
f'<line x1="{x:.1f}" y1="{top - 18}" x2="{x:.1f}" y2="{top + row_height * len(rows)}" '
f'stroke="{GRID_COLOR}" stroke-width="1"/>'
)
body.append(
f'<text x="{x:.1f}" y="{top - 24}" font-size="11" fill="#64748b" '
f'text-anchor="middle">{value:.1f}</text>'
)
legend_y = top - 34
if secondary_key:
body.append(
f'<rect x="{left}" y="{legend_y}" width="10" height="10" fill="{BASELINE_COLOR}"/>'
f'<text x="{left + 16}" y="{legend_y + 9}" font-size="12" fill="#475569">'
f"{escape(secondary_label or 'baseline')}</text>"
)
body.append(
f'<rect x="{left + 140}" y="{legend_y}" width="10" height="10" fill="{MODEL_COLOR}"/>'
f'<text x="{left + 156}" y="{legend_y + 9}" font-size="12" fill="#475569">'
f"{escape(primary_label)}</text>"
)
for index, row in enumerate(rows):
y = top + index * row_height
label = str(row.get("name") or row.get("sensor"))
body.append(
f'<text x="{left - 12}" y="{y + 20}" font-size="12" fill="{TEXT_COLOR}" '
f'text-anchor="end">{escape(label)}</text>'
)
if secondary_key:
baseline_width = plot_width * row[secondary_key] / maximum
body.append(
f'<rect x="{left}" y="{y + 3}" width="{baseline_width:.1f}" height="12" '
f'fill="{BASELINE_COLOR}"/>'
)
body.append(
f'<text x="{left + baseline_width + 6:.1f}" y="{y + 13}" font-size="11" '
f'fill="#64748b">{row[secondary_key]:.2f}</text>'
)
model_width = plot_width * row[value_key] / maximum
body.append(
f'<rect x="{left}" y="{y + 18}" width="{model_width:.1f}" height="12" '
f'fill="{MODEL_COLOR}"/>'
)
body.append(
f'<text x="{left + model_width + 6:.1f}" y="{y + 28}" font-size="11" '
f'fill="{TEXT_COLOR}">{row[value_key]:.2f}</text>'
)
else:
color = PRODUCTION_COLOR if production and row.get("name") == production else MODEL_COLOR
bar_width = plot_width * row[value_key] / maximum
body.append(
f'<rect x="{left}" y="{y + 6}" width="{bar_width:.1f}" height="22" fill="{color}"/>'
)
body.append(
f'<text x="{left + bar_width + 6:.1f}" y="{y + 21}" font-size="12" '
f'fill="{TEXT_COLOR}">{row[value_key]:.2f}</text>'
)
return _svg_document(title, subtitle, body, width, height)
def _confusion_svg(small: dict) -> str:
matrix = small["confusion_matrix"]
metrics = small["metrics"]
width, height = 760, 520
left, top, cell = 200, 130, 190
values = {
(0, 0): ("TN", matrix["tn"]),
(0, 1): ("FP", matrix["fp"]),
(1, 0): ("FN", matrix["fn"]),
(1, 1): ("TP", matrix["tp"]),
}
maximum = max(value for _, value in values.values())
body: list[str] = []
body.append(
f'<text x="{left + cell / 2}" y="{top - 16}" font-size="13" fill="#475569" '
f'text-anchor="middle">Predicted not busy</text>'
)
body.append(
f'<text x="{left + cell + cell / 2}" y="{top - 16}" font-size="13" fill="#475569" '
f'text-anchor="middle">Predicted busy</text>'
)
row_labels = ("Actually\nnot busy", "Actually\nbusy")
for row, label in enumerate(row_labels):
y = top + row * cell
body.append(
f'<text x="{left - 18}" y="{y + cell / 2}" font-size="13" fill="#475569" '
f'text-anchor="end">{escape(label).replace(chr(10), " ")}</text>'
)
for (row, column), (code, value) in values.items():
x = left + column * cell
y = top + row * cell
intensity = 0.12 + 0.88 * (value / maximum)
red = round(219 + (29 - 219) * intensity)
green = round(234 + (78 - 234) * intensity)
blue = round(254 + (216 - 254) * intensity)
fill = f"#{red:02x}{green:02x}{blue:02x}"
text_color = "#ffffff" if intensity > 0.55 else TEXT_COLOR
row_total = matrix["tn"] + matrix["fp"] if row == 0 else matrix["fn"] + matrix["tp"]
share = 100.0 * value / row_total if row_total else 0.0
body.append(
f'<rect x="{x}" y="{y}" width="{cell}" height="{cell}" fill="{fill}" '
f'stroke="#ffffff" stroke-width="2"/>'
)
body.append(
f'<text x="{x + cell / 2}" y="{y + cell / 2 - 8}" font-size="30" '
f'font-weight="600" fill="{text_color}" text-anchor="middle">{value:,}</text>'
)
body.append(
f'<text x="{x + cell / 2}" y="{y + cell / 2 + 20}" font-size="13" '
f'fill="{text_color}" text-anchor="middle">{code} · {share:.1f}% of row</text>'
)
subtitle = (
f"n={metrics['n_test']:,} · accuracy {metrics['accuracy']:.3f} · "
f"precision {metrics['precision']:.3f} · recall {metrics['recall']:.3f} · "
f"F1 {metrics['f1']:.3f} · ROC-AUC {metrics['roc_auc']:.3f}"
)
return _svg_document(
"Small busy-hour classifier: holdout confusion matrix",
subtitle,
body,
width,
height,
footer="Source: artifacts/small_metrics.json · chronological 80/20 split",
)
def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
"--artifacts",
type=Path,
default=config.artifacts_dir(),
help="artifact directory containing metrics.json (default: %(default)s)",
)
parser.add_argument(
"--out",
type=Path,
default=Path(__file__).resolve().parents[1] / "docs" / "figures",
help="output directory for SVG files (default: %(default)s)",
)
args = parser.parse_args(argv)
metrics = json.loads((args.artifacts / "metrics.json").read_text())
holdout = f"Holdout {metrics['test_from']} .. {metrics['test_to']}, n={metrics['metrics']['n_eval']}"
args.out.mkdir(parents=True, exist_ok=True)
ablation = _bar_chart(
"Ablation: holdout MAE by feature set",
holdout,
[
{
"name": row["name"].replace("_", " "),
"mae": row["mae"],
}
for row in metrics["ablation"]
],
production="week weather",
)
ablation_path = args.out / "ablation_mae.svg"
ablation_path.write_text(ablation)
per_sensor = _bar_chart(
"Per-sensor holdout MAE: production model vs seasonal naive",
holdout,
metrics["per_sensor"],
secondary_key="baseline_mae",
secondary_label="seasonal naive",
primary_label="weather-7d",
)
per_sensor_path = args.out / "per_sensor_mae.svg"
per_sensor_path.write_text(per_sensor)
confusion_path = args.out / "small_model_confusion.svg"
confusion_path.write_text(_confusion_svg(metrics["small"]))
print(f"wrote {ablation_path}")
print(f"wrote {per_sensor_path}")
print(f"wrote {confusion_path}")
return 0
if __name__ == "__main__":
raise SystemExit(main())