feat: repeatable small-model, figures, and training e2e scripts
- make small trains from a CSV and prints metrics + confusion matrix - make figures renders dependency-free SVGs from metrics.json - scripts/e2e_train.py boots a real server and checks snapshot, robustness, auth, one-job-at-a-time, and both training jobs with hot-swap - CI/ruff/compileall now include scripts/
This commit is contained in:
1 parent
000aaed7a5
commit
2bcf66c550
5 files changed
+952
-6
No files matched your search
@@ -0,0 +1,262 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Render dependency-free SVG figures from ``artifacts/metrics.json``.
|
||||
|
||||
The paper figures are generated from the measured holdout metrics, never
|
||||
hand-written. Run ``make figures`` after ``make train``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
from xml.sax.saxutils import escape
|
||||
|
||||
from pulsecast import config
|
||||
|
||||
MODEL_COLOR = "#2563eb"
|
||||
BASELINE_COLOR = "#94a3b8"
|
||||
PRODUCTION_COLOR = "#0f766e"
|
||||
TEXT_COLOR = "#0f172a"
|
||||
GRID_COLOR = "#e2e8f0"
|
||||
|
||||
|
||||
def _svg_document(
|
||||
title: str,
|
||||
subtitle: str,
|
||||
body: list[str],
|
||||
width: int,
|
||||
height: int,
|
||||
footer: str = "Source: artifacts/metrics.json · PulseCast 28-day time-based holdout",
|
||||
) -> str:
|
||||
return "\n".join(
|
||||
[
|
||||
f'<svg xmlns="http://www.w3.org/2000/svg" width="{width}" height="{height}" '
|
||||
f'viewBox="0 0 {width} {height}" role="img" aria-label="{escape(title)}">',
|
||||
'<rect width="100%" height="100%" fill="#ffffff"/>',
|
||||
'<g font-family="-apple-system, Segoe UI, Helvetica, Arial, sans-serif">',
|
||||
f'<text x="24" y="34" font-size="20" font-weight="600" fill="{TEXT_COLOR}">'
|
||||
f"{escape(title)}</text>",
|
||||
f'<text x="24" y="56" font-size="13" fill="#475569">{escape(subtitle)}</text>',
|
||||
*body,
|
||||
f'<text x="24" y="{height - 12}" font-size="11" fill="#64748b">'
|
||||
f"{escape(footer)}</text>",
|
||||
"</g>",
|
||||
"</svg>",
|
||||
"",
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def _bar_chart(
|
||||
title: str,
|
||||
subtitle: str,
|
||||
rows: list[dict],
|
||||
*,
|
||||
value_key: str = "mae",
|
||||
secondary_key: str | None = None,
|
||||
secondary_label: str | None = None,
|
||||
primary_label: str = "MAE",
|
||||
production: str | None = None,
|
||||
) -> str:
|
||||
width = 960
|
||||
left = 230
|
||||
right = 96
|
||||
top = 86
|
||||
row_height = 38
|
||||
height = top + row_height * len(rows) + 56
|
||||
plot_width = width - left - right
|
||||
values = [row[value_key] for row in rows]
|
||||
if secondary_key:
|
||||
values += [row[secondary_key] for row in rows]
|
||||
maximum = max(values) * 1.08 if values else 1.0
|
||||
|
||||
body: list[str] = []
|
||||
for tick in range(5):
|
||||
x = left + plot_width * tick / 4
|
||||
value = maximum * tick / 4
|
||||
body.append(
|
||||
f'<line x1="{x:.1f}" y1="{top - 18}" x2="{x:.1f}" y2="{top + row_height * len(rows)}" '
|
||||
f'stroke="{GRID_COLOR}" stroke-width="1"/>'
|
||||
)
|
||||
body.append(
|
||||
f'<text x="{x:.1f}" y="{top - 24}" font-size="11" fill="#64748b" '
|
||||
f'text-anchor="middle">{value:.1f}</text>'
|
||||
)
|
||||
|
||||
legend_y = top - 34
|
||||
if secondary_key:
|
||||
body.append(
|
||||
f'<rect x="{left}" y="{legend_y}" width="10" height="10" fill="{BASELINE_COLOR}"/>'
|
||||
f'<text x="{left + 16}" y="{legend_y + 9}" font-size="12" fill="#475569">'
|
||||
f"{escape(secondary_label or 'baseline')}</text>"
|
||||
)
|
||||
body.append(
|
||||
f'<rect x="{left + 140}" y="{legend_y}" width="10" height="10" fill="{MODEL_COLOR}"/>'
|
||||
f'<text x="{left + 156}" y="{legend_y + 9}" font-size="12" fill="#475569">'
|
||||
f"{escape(primary_label)}</text>"
|
||||
)
|
||||
|
||||
for index, row in enumerate(rows):
|
||||
y = top + index * row_height
|
||||
label = str(row.get("name") or row.get("sensor"))
|
||||
body.append(
|
||||
f'<text x="{left - 12}" y="{y + 20}" font-size="12" fill="{TEXT_COLOR}" '
|
||||
f'text-anchor="end">{escape(label)}</text>'
|
||||
)
|
||||
if secondary_key:
|
||||
baseline_width = plot_width * row[secondary_key] / maximum
|
||||
body.append(
|
||||
f'<rect x="{left}" y="{y + 3}" width="{baseline_width:.1f}" height="12" '
|
||||
f'fill="{BASELINE_COLOR}"/>'
|
||||
)
|
||||
body.append(
|
||||
f'<text x="{left + baseline_width + 6:.1f}" y="{y + 13}" font-size="11" '
|
||||
f'fill="#64748b">{row[secondary_key]:.2f}</text>'
|
||||
)
|
||||
model_width = plot_width * row[value_key] / maximum
|
||||
body.append(
|
||||
f'<rect x="{left}" y="{y + 18}" width="{model_width:.1f}" height="12" '
|
||||
f'fill="{MODEL_COLOR}"/>'
|
||||
)
|
||||
body.append(
|
||||
f'<text x="{left + model_width + 6:.1f}" y="{y + 28}" font-size="11" '
|
||||
f'fill="{TEXT_COLOR}">{row[value_key]:.2f}</text>'
|
||||
)
|
||||
else:
|
||||
color = PRODUCTION_COLOR if production and row.get("name") == production else MODEL_COLOR
|
||||
bar_width = plot_width * row[value_key] / maximum
|
||||
body.append(
|
||||
f'<rect x="{left}" y="{y + 6}" width="{bar_width:.1f}" height="22" fill="{color}"/>'
|
||||
)
|
||||
body.append(
|
||||
f'<text x="{left + bar_width + 6:.1f}" y="{y + 21}" font-size="12" '
|
||||
f'fill="{TEXT_COLOR}">{row[value_key]:.2f}</text>'
|
||||
)
|
||||
|
||||
return _svg_document(title, subtitle, body, width, height)
|
||||
|
||||
|
||||
def _confusion_svg(small: dict) -> str:
|
||||
matrix = small["confusion_matrix"]
|
||||
metrics = small["metrics"]
|
||||
width, height = 760, 520
|
||||
left, top, cell = 200, 130, 190
|
||||
values = {
|
||||
(0, 0): ("TN", matrix["tn"]),
|
||||
(0, 1): ("FP", matrix["fp"]),
|
||||
(1, 0): ("FN", matrix["fn"]),
|
||||
(1, 1): ("TP", matrix["tp"]),
|
||||
}
|
||||
maximum = max(value for _, value in values.values())
|
||||
body: list[str] = []
|
||||
body.append(
|
||||
f'<text x="{left + cell / 2}" y="{top - 16}" font-size="13" fill="#475569" '
|
||||
f'text-anchor="middle">Predicted not busy</text>'
|
||||
)
|
||||
body.append(
|
||||
f'<text x="{left + cell + cell / 2}" y="{top - 16}" font-size="13" fill="#475569" '
|
||||
f'text-anchor="middle">Predicted busy</text>'
|
||||
)
|
||||
row_labels = ("Actually\nnot busy", "Actually\nbusy")
|
||||
for row, label in enumerate(row_labels):
|
||||
y = top + row * cell
|
||||
body.append(
|
||||
f'<text x="{left - 18}" y="{y + cell / 2}" font-size="13" fill="#475569" '
|
||||
f'text-anchor="end">{escape(label).replace(chr(10), " ")}</text>'
|
||||
)
|
||||
for (row, column), (code, value) in values.items():
|
||||
x = left + column * cell
|
||||
y = top + row * cell
|
||||
intensity = 0.12 + 0.88 * (value / maximum)
|
||||
red = round(219 + (29 - 219) * intensity)
|
||||
green = round(234 + (78 - 234) * intensity)
|
||||
blue = round(254 + (216 - 254) * intensity)
|
||||
fill = f"#{red:02x}{green:02x}{blue:02x}"
|
||||
text_color = "#ffffff" if intensity > 0.55 else TEXT_COLOR
|
||||
row_total = matrix["tn"] + matrix["fp"] if row == 0 else matrix["fn"] + matrix["tp"]
|
||||
share = 100.0 * value / row_total if row_total else 0.0
|
||||
body.append(
|
||||
f'<rect x="{x}" y="{y}" width="{cell}" height="{cell}" fill="{fill}" '
|
||||
f'stroke="#ffffff" stroke-width="2"/>'
|
||||
)
|
||||
body.append(
|
||||
f'<text x="{x + cell / 2}" y="{y + cell / 2 - 8}" font-size="30" '
|
||||
f'font-weight="600" fill="{text_color}" text-anchor="middle">{value:,}</text>'
|
||||
)
|
||||
body.append(
|
||||
f'<text x="{x + cell / 2}" y="{y + cell / 2 + 20}" font-size="13" '
|
||||
f'fill="{text_color}" text-anchor="middle">{code} · {share:.1f}% of row</text>'
|
||||
)
|
||||
subtitle = (
|
||||
f"n={metrics['n_test']:,} · accuracy {metrics['accuracy']:.3f} · "
|
||||
f"precision {metrics['precision']:.3f} · recall {metrics['recall']:.3f} · "
|
||||
f"F1 {metrics['f1']:.3f} · ROC-AUC {metrics['roc_auc']:.3f}"
|
||||
)
|
||||
return _svg_document(
|
||||
"Small busy-hour classifier: holdout confusion matrix",
|
||||
subtitle,
|
||||
body,
|
||||
width,
|
||||
height,
|
||||
footer="Source: artifacts/small_metrics.json · chronological 80/20 split",
|
||||
)
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument(
|
||||
"--artifacts",
|
||||
type=Path,
|
||||
default=config.artifacts_dir(),
|
||||
help="artifact directory containing metrics.json (default: %(default)s)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--out",
|
||||
type=Path,
|
||||
default=Path(__file__).resolve().parents[1] / "docs" / "figures",
|
||||
help="output directory for SVG files (default: %(default)s)",
|
||||
)
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
metrics = json.loads((args.artifacts / "metrics.json").read_text())
|
||||
holdout = f"Holdout {metrics['test_from']} .. {metrics['test_to']}, n={metrics['metrics']['n_eval']}"
|
||||
args.out.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
ablation = _bar_chart(
|
||||
"Ablation: holdout MAE by feature set",
|
||||
holdout,
|
||||
[
|
||||
{
|
||||
"name": row["name"].replace("_", " "),
|
||||
"mae": row["mae"],
|
||||
}
|
||||
for row in metrics["ablation"]
|
||||
],
|
||||
production="week weather",
|
||||
)
|
||||
ablation_path = args.out / "ablation_mae.svg"
|
||||
ablation_path.write_text(ablation)
|
||||
|
||||
per_sensor = _bar_chart(
|
||||
"Per-sensor holdout MAE: production model vs seasonal naive",
|
||||
holdout,
|
||||
metrics["per_sensor"],
|
||||
secondary_key="baseline_mae",
|
||||
secondary_label="seasonal naive",
|
||||
primary_label="weather-7d",
|
||||
)
|
||||
per_sensor_path = args.out / "per_sensor_mae.svg"
|
||||
per_sensor_path.write_text(per_sensor)
|
||||
|
||||
confusion_path = args.out / "small_model_confusion.svg"
|
||||
confusion_path.write_text(_confusion_svg(metrics["small"]))
|
||||
|
||||
print(f"wrote {ablation_path}")
|
||||
print(f"wrote {per_sensor_path}")
|
||||
print(f"wrote {confusion_path}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in new issue
Block a user