- make small trains from a CSV and prints metrics + confusion matrix - make figures renders dependency-free SVGs from metrics.json - scripts/e2e_train.py boots a real server and checks snapshot, robustness, auth, one-job-at-a-time, and both training jobs with hot-swap - CI/ruff/compileall now include scripts/
263 lines
9.4 KiB
Python
263 lines
9.4 KiB
Python
#!/usr/bin/env python3
|
|
"""Render dependency-free SVG figures from ``artifacts/metrics.json``.
|
|
|
|
The paper figures are generated from the measured holdout metrics, never
|
|
hand-written. Run ``make figures`` after ``make train``.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
from pathlib import Path
|
|
from xml.sax.saxutils import escape
|
|
|
|
from pulsecast import config
|
|
|
|
MODEL_COLOR = "#2563eb"
|
|
BASELINE_COLOR = "#94a3b8"
|
|
PRODUCTION_COLOR = "#0f766e"
|
|
TEXT_COLOR = "#0f172a"
|
|
GRID_COLOR = "#e2e8f0"
|
|
|
|
|
|
def _svg_document(
|
|
title: str,
|
|
subtitle: str,
|
|
body: list[str],
|
|
width: int,
|
|
height: int,
|
|
footer: str = "Source: artifacts/metrics.json · PulseCast 28-day time-based holdout",
|
|
) -> str:
|
|
return "\n".join(
|
|
[
|
|
f'<svg xmlns="http://www.w3.org/2000/svg" width="{width}" height="{height}" '
|
|
f'viewBox="0 0 {width} {height}" role="img" aria-label="{escape(title)}">',
|
|
'<rect width="100%" height="100%" fill="#ffffff"/>',
|
|
'<g font-family="-apple-system, Segoe UI, Helvetica, Arial, sans-serif">',
|
|
f'<text x="24" y="34" font-size="20" font-weight="600" fill="{TEXT_COLOR}">'
|
|
f"{escape(title)}</text>",
|
|
f'<text x="24" y="56" font-size="13" fill="#475569">{escape(subtitle)}</text>',
|
|
*body,
|
|
f'<text x="24" y="{height - 12}" font-size="11" fill="#64748b">'
|
|
f"{escape(footer)}</text>",
|
|
"</g>",
|
|
"</svg>",
|
|
"",
|
|
]
|
|
)
|
|
|
|
|
|
def _bar_chart(
|
|
title: str,
|
|
subtitle: str,
|
|
rows: list[dict],
|
|
*,
|
|
value_key: str = "mae",
|
|
secondary_key: str | None = None,
|
|
secondary_label: str | None = None,
|
|
primary_label: str = "MAE",
|
|
production: str | None = None,
|
|
) -> str:
|
|
width = 960
|
|
left = 230
|
|
right = 96
|
|
top = 86
|
|
row_height = 38
|
|
height = top + row_height * len(rows) + 56
|
|
plot_width = width - left - right
|
|
values = [row[value_key] for row in rows]
|
|
if secondary_key:
|
|
values += [row[secondary_key] for row in rows]
|
|
maximum = max(values) * 1.08 if values else 1.0
|
|
|
|
body: list[str] = []
|
|
for tick in range(5):
|
|
x = left + plot_width * tick / 4
|
|
value = maximum * tick / 4
|
|
body.append(
|
|
f'<line x1="{x:.1f}" y1="{top - 18}" x2="{x:.1f}" y2="{top + row_height * len(rows)}" '
|
|
f'stroke="{GRID_COLOR}" stroke-width="1"/>'
|
|
)
|
|
body.append(
|
|
f'<text x="{x:.1f}" y="{top - 24}" font-size="11" fill="#64748b" '
|
|
f'text-anchor="middle">{value:.1f}</text>'
|
|
)
|
|
|
|
legend_y = top - 34
|
|
if secondary_key:
|
|
body.append(
|
|
f'<rect x="{left}" y="{legend_y}" width="10" height="10" fill="{BASELINE_COLOR}"/>'
|
|
f'<text x="{left + 16}" y="{legend_y + 9}" font-size="12" fill="#475569">'
|
|
f"{escape(secondary_label or 'baseline')}</text>"
|
|
)
|
|
body.append(
|
|
f'<rect x="{left + 140}" y="{legend_y}" width="10" height="10" fill="{MODEL_COLOR}"/>'
|
|
f'<text x="{left + 156}" y="{legend_y + 9}" font-size="12" fill="#475569">'
|
|
f"{escape(primary_label)}</text>"
|
|
)
|
|
|
|
for index, row in enumerate(rows):
|
|
y = top + index * row_height
|
|
label = str(row.get("name") or row.get("sensor"))
|
|
body.append(
|
|
f'<text x="{left - 12}" y="{y + 20}" font-size="12" fill="{TEXT_COLOR}" '
|
|
f'text-anchor="end">{escape(label)}</text>'
|
|
)
|
|
if secondary_key:
|
|
baseline_width = plot_width * row[secondary_key] / maximum
|
|
body.append(
|
|
f'<rect x="{left}" y="{y + 3}" width="{baseline_width:.1f}" height="12" '
|
|
f'fill="{BASELINE_COLOR}"/>'
|
|
)
|
|
body.append(
|
|
f'<text x="{left + baseline_width + 6:.1f}" y="{y + 13}" font-size="11" '
|
|
f'fill="#64748b">{row[secondary_key]:.2f}</text>'
|
|
)
|
|
model_width = plot_width * row[value_key] / maximum
|
|
body.append(
|
|
f'<rect x="{left}" y="{y + 18}" width="{model_width:.1f}" height="12" '
|
|
f'fill="{MODEL_COLOR}"/>'
|
|
)
|
|
body.append(
|
|
f'<text x="{left + model_width + 6:.1f}" y="{y + 28}" font-size="11" '
|
|
f'fill="{TEXT_COLOR}">{row[value_key]:.2f}</text>'
|
|
)
|
|
else:
|
|
color = PRODUCTION_COLOR if production and row.get("name") == production else MODEL_COLOR
|
|
bar_width = plot_width * row[value_key] / maximum
|
|
body.append(
|
|
f'<rect x="{left}" y="{y + 6}" width="{bar_width:.1f}" height="22" fill="{color}"/>'
|
|
)
|
|
body.append(
|
|
f'<text x="{left + bar_width + 6:.1f}" y="{y + 21}" font-size="12" '
|
|
f'fill="{TEXT_COLOR}">{row[value_key]:.2f}</text>'
|
|
)
|
|
|
|
return _svg_document(title, subtitle, body, width, height)
|
|
|
|
|
|
def _confusion_svg(small: dict) -> str:
|
|
matrix = small["confusion_matrix"]
|
|
metrics = small["metrics"]
|
|
width, height = 760, 520
|
|
left, top, cell = 200, 130, 190
|
|
values = {
|
|
(0, 0): ("TN", matrix["tn"]),
|
|
(0, 1): ("FP", matrix["fp"]),
|
|
(1, 0): ("FN", matrix["fn"]),
|
|
(1, 1): ("TP", matrix["tp"]),
|
|
}
|
|
maximum = max(value for _, value in values.values())
|
|
body: list[str] = []
|
|
body.append(
|
|
f'<text x="{left + cell / 2}" y="{top - 16}" font-size="13" fill="#475569" '
|
|
f'text-anchor="middle">Predicted not busy</text>'
|
|
)
|
|
body.append(
|
|
f'<text x="{left + cell + cell / 2}" y="{top - 16}" font-size="13" fill="#475569" '
|
|
f'text-anchor="middle">Predicted busy</text>'
|
|
)
|
|
row_labels = ("Actually\nnot busy", "Actually\nbusy")
|
|
for row, label in enumerate(row_labels):
|
|
y = top + row * cell
|
|
body.append(
|
|
f'<text x="{left - 18}" y="{y + cell / 2}" font-size="13" fill="#475569" '
|
|
f'text-anchor="end">{escape(label).replace(chr(10), " ")}</text>'
|
|
)
|
|
for (row, column), (code, value) in values.items():
|
|
x = left + column * cell
|
|
y = top + row * cell
|
|
intensity = 0.12 + 0.88 * (value / maximum)
|
|
red = round(219 + (29 - 219) * intensity)
|
|
green = round(234 + (78 - 234) * intensity)
|
|
blue = round(254 + (216 - 254) * intensity)
|
|
fill = f"#{red:02x}{green:02x}{blue:02x}"
|
|
text_color = "#ffffff" if intensity > 0.55 else TEXT_COLOR
|
|
row_total = matrix["tn"] + matrix["fp"] if row == 0 else matrix["fn"] + matrix["tp"]
|
|
share = 100.0 * value / row_total if row_total else 0.0
|
|
body.append(
|
|
f'<rect x="{x}" y="{y}" width="{cell}" height="{cell}" fill="{fill}" '
|
|
f'stroke="#ffffff" stroke-width="2"/>'
|
|
)
|
|
body.append(
|
|
f'<text x="{x + cell / 2}" y="{y + cell / 2 - 8}" font-size="30" '
|
|
f'font-weight="600" fill="{text_color}" text-anchor="middle">{value:,}</text>'
|
|
)
|
|
body.append(
|
|
f'<text x="{x + cell / 2}" y="{y + cell / 2 + 20}" font-size="13" '
|
|
f'fill="{text_color}" text-anchor="middle">{code} · {share:.1f}% of row</text>'
|
|
)
|
|
subtitle = (
|
|
f"n={metrics['n_test']:,} · accuracy {metrics['accuracy']:.3f} · "
|
|
f"precision {metrics['precision']:.3f} · recall {metrics['recall']:.3f} · "
|
|
f"F1 {metrics['f1']:.3f} · ROC-AUC {metrics['roc_auc']:.3f}"
|
|
)
|
|
return _svg_document(
|
|
"Small busy-hour classifier: holdout confusion matrix",
|
|
subtitle,
|
|
body,
|
|
width,
|
|
height,
|
|
footer="Source: artifacts/small_metrics.json · chronological 80/20 split",
|
|
)
|
|
|
|
|
|
def main(argv: list[str] | None = None) -> int:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument(
|
|
"--artifacts",
|
|
type=Path,
|
|
default=config.artifacts_dir(),
|
|
help="artifact directory containing metrics.json (default: %(default)s)",
|
|
)
|
|
parser.add_argument(
|
|
"--out",
|
|
type=Path,
|
|
default=Path(__file__).resolve().parents[1] / "docs" / "figures",
|
|
help="output directory for SVG files (default: %(default)s)",
|
|
)
|
|
args = parser.parse_args(argv)
|
|
|
|
metrics = json.loads((args.artifacts / "metrics.json").read_text())
|
|
holdout = f"Holdout {metrics['test_from']} .. {metrics['test_to']}, n={metrics['metrics']['n_eval']}"
|
|
args.out.mkdir(parents=True, exist_ok=True)
|
|
|
|
ablation = _bar_chart(
|
|
"Ablation: holdout MAE by feature set",
|
|
holdout,
|
|
[
|
|
{
|
|
"name": row["name"].replace("_", " "),
|
|
"mae": row["mae"],
|
|
}
|
|
for row in metrics["ablation"]
|
|
],
|
|
production="week weather",
|
|
)
|
|
ablation_path = args.out / "ablation_mae.svg"
|
|
ablation_path.write_text(ablation)
|
|
|
|
per_sensor = _bar_chart(
|
|
"Per-sensor holdout MAE: production model vs seasonal naive",
|
|
holdout,
|
|
metrics["per_sensor"],
|
|
secondary_key="baseline_mae",
|
|
secondary_label="seasonal naive",
|
|
primary_label="weather-7d",
|
|
)
|
|
per_sensor_path = args.out / "per_sensor_mae.svg"
|
|
per_sensor_path.write_text(per_sensor)
|
|
|
|
confusion_path = args.out / "small_model_confusion.svg"
|
|
confusion_path.write_text(_confusion_svg(metrics["small"]))
|
|
|
|
print(f"wrote {ablation_path}")
|
|
print(f"wrote {per_sensor_path}")
|
|
print(f"wrote {confusion_path}")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|