feat: repeatable small-model, figures, and training e2e scripts

- make small trains from a CSV and prints metrics + confusion matrix
- make figures renders dependency-free SVGs from metrics.json
- scripts/e2e_train.py boots a real server and checks snapshot, robustness,
  auth, one-job-at-a-time, and both training jobs with hot-swap
- CI/ruff/compileall now include scripts/
This commit is contained in:
Noel Sabia committed 2026-11-14 06:35:19 +01:00
1 parent 000aaed7a5
commit 2bcf66c550
5 files changed
+952 -6

No files matched your search

+2 -2
View File
@@ -32,10 +32,10 @@ jobs:
run: uv sync --frozen
- name: Static analysis (ruff)
run: uv run ruff check src tests main.py
run: uv run ruff check src tests scripts main.py
- name: Byte-compile
run: uv run python -m compileall src
run: uv run python -m compileall src scripts
- name: Tests
run: uv run pytest -q
+12 -4
View File
@@ -1,11 +1,13 @@
UV ?= uv
.PHONY: help data train test lint check serve docker clean
.PHONY: help data train small figures test lint check serve docker clean
help:
@printf '%s\n' \
'make data parse/clean the raw CSV, join weather + holidays, export JSON' \
'make train fit the 28-day holdout model, export metrics/forecasts/model' \
'make small train + print the busy-hour classifier metrics from the raw CSV' \
'make figures regenerate docs/figures/*.svg from artifacts/metrics.json' \
'make test run pytest' \
'make lint run ruff static checks' \
'make check CI gate: sync --frozen, ruff, compileall, pytest' \
@@ -19,16 +21,22 @@ data:
train:
$(UV) run python -m pulsecast.train
small:
$(UV) run python scripts/train_small.py
figures:
$(UV) run python scripts/plot_metrics.py
test:
$(UV) run pytest
lint:
$(UV) run ruff check src tests main.py
$(UV) run ruff check src tests scripts main.py
check:
$(UV) sync --frozen
$(UV) run ruff check src tests main.py
$(UV) run python -m compileall src
$(UV) run ruff check src tests scripts main.py
$(UV) run python -m compileall src scripts
$(UV) run pytest -q
serve:
+523
View File
@@ -0,0 +1,523 @@
#!/usr/bin/env python3
"""End-to-end acceptance run against a real FastAPI process on port 18000.
Starts uvicorn with ``CONTROL_TOKEN``, then exercises robustness and the full
training lifecycle:
- /healthz, /readyz, /meta, /bootstrap (8 sensors, hourly series, context)
- /predict for every sensor x horizon x scenario (128 calls)
- /simulate for every scenario with all sensors
- auth, unknown job, unknown model, malformed CSV, one-job-at-a-time
- POST /train model=small on a real CSV subset; poll to success; hot-swap
- POST /train model=forecast on the full CSV; poll to success; hot-swap
- /predict still works after each swap
Exit code is 0 only when every check passes. Figures printed to stdout are the
measured numbers; ``--report`` writes the same result as JSON.
"""
from __future__ import annotations
import argparse
import json
import os
import shutil
import socket
import subprocess
import sys
import tempfile
import time
from datetime import UTC, datetime
from pathlib import Path
import httpx
import pandas as pd
from pulsecast import config
HORIZON_SIZES = {"15m": 1, "1h": 4, "48h": 48, "7d": 168}
SCENARIOS = tuple(config.SCENARIOS)
SENSORS = tuple(sorted(config.USABLE_SENSORS))
class CheckFailed(RuntimeError):
pass
class Runner:
def __init__(self) -> None:
self.checks = 0
self.results: dict = {}
def check(self, condition: bool, message: str) -> None:
self.checks += 1
if not condition:
raise CheckFailed(message)
print(f" [ok] {message}")
def _utc_now() -> str:
return datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ")
def _parse_ts(value: str) -> pd.Timestamp:
return pd.Timestamp(value)
def _seconds(start: str | None, end: str | None) -> float | None:
if not start or not end:
return None
return round((_parse_ts(end) - _parse_ts(start)).total_seconds(), 3)
def _wait_http(client: httpx.Client, url: str, timeout: float = 30.0) -> None:
deadline = time.monotonic() + timeout
last: Exception | None = None
while time.monotonic() < deadline:
try:
response = client.get(url, timeout=2.0)
if response.status_code == 200:
return
except Exception as exc: # noqa: BLE001 - startup race
last = exc
time.sleep(0.1)
raise CheckFailed(f"server did not answer {url} within {timeout}s: {last}")
def _wait_job(
client: httpx.Client,
job_id: str,
headers: dict[str, str],
*,
timeout: float = 180.0,
state: str | None = None,
) -> dict:
deadline = time.monotonic() + timeout
last_status = None
while time.monotonic() < deadline:
response = client.get(f"/train/{job_id}", headers=headers)
if response.status_code != 200:
raise CheckFailed(f"GET /train/{job_id} -> {response.status_code}: {response.text}")
job = response.json()
last_status = job["status"]
if state is not None and last_status == state:
return job
if state is None and last_status in {"succeeded", "failed"}:
return job
time.sleep(0.05)
raise CheckFailed(f"job {job_id} did not reach {state or 'terminal'} (last={last_status})")
def _job_headers(token: str) -> dict[str, str]:
return {"Authorization": f"Bearer {token}"}
def _submit(
client: httpx.Client, headers: dict[str, str], path: Path, model: str
) -> httpx.Response:
return client.post(
"/train",
headers=headers,
files={"file": (path.name, path.read_bytes(), "text/csv")},
data={"model": model},
timeout=120.0,
)
def _assert_points(points: list[dict], expected: int, label: str) -> None:
if len(points) != expected:
raise CheckFailed(f"{label}: expected {expected} points, got {len(points)}")
previous = None
for point in points:
if set(point) != {"t", "v", "lo", "hi"}:
raise CheckFailed(f"{label}: unexpected point keys {sorted(point)}")
if not point["t"].endswith("Z"):
raise CheckFailed(f"{label}: timestamp is not UTC RFC 3339: {point['t']}")
if point["v"] < 0 or point["lo"] > point["v"] or point["v"] > point["hi"]:
raise CheckFailed(f"{label}: invalid band {point}")
stamp = _parse_ts(point["t"])
if previous is not None and stamp <= previous:
raise CheckFailed(f"{label}: timestamps are not increasing at {point['t']}")
previous = stamp
def _versions(payload: dict) -> dict:
return {
key: payload.get(key)
for key in ("data_version", "model_version", "context_version")
}
def _build_subset(raw: Path, destination: Path, days: int = 60) -> tuple[Path, dict]:
frame = pd.read_csv(raw)
frame = frame[frame["type"].astype(str) == "densityData"].copy()
frame["time"] = pd.to_datetime(frame["time"], utc=True)
end = frame["time"].max()
cut = end - pd.Timedelta(days=days)
frame = frame[frame["time"] >= cut][["time", "name", "value"]]
frame["time"] = frame["time"].dt.strftime("%Y-%m-%dT%H:%M:%SZ")
frame.to_csv(destination, index=False)
return destination, {
"rows": len(frame),
"sensors": int(frame["name"].nunique()),
"from": frame["time"].min(),
"to": frame["time"].max(),
"days": days,
}
def _port_free(port: int) -> bool:
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
sock.settimeout(0.5)
return sock.connect_ex(("127.0.0.1", port)) != 0
def run(args: argparse.Namespace, runner: Runner) -> dict:
raw = Path(args.csv)
if not raw.exists():
raise CheckFailed(f"CSV not found: {raw}")
if not _port_free(args.port):
raise CheckFailed(f"port {args.port} is already in use")
workdir = Path(tempfile.mkdtemp(prefix="pulsecast-e2e-"))
models_dir = workdir / "models"
uploads_dir = workdir / "uploads"
subset_path, subset_info = _build_subset(raw, workdir / "small-subset.csv", args.days)
garbage_path = workdir / "garbage.csv"
garbage_path.write_text("not,a,pulsecast,file\n1,2,3,4\n")
env = os.environ.copy()
env.update(
CONTROL_TOKEN=args.token,
PULSECAST_MODELS=str(models_dir),
PULSECAST_UPLOADS=str(uploads_dir),
)
log_path = workdir / "server.log"
log_handle = log_path.open("wb")
command = [
sys.executable,
"-m",
"uvicorn",
"pulsecast.server:app",
"--host",
"127.0.0.1",
"--port",
str(args.port),
"--log-level",
"warning",
]
process = subprocess.Popen(command, env=env, stdout=log_handle, stderr=subprocess.STDOUT)
report: dict = {
"started_at": _utc_now(),
"base_url": f"http://127.0.0.1:{args.port}",
"subset": subset_info,
"raw_csv_bytes": raw.stat().st_size,
}
try:
with httpx.Client(base_url=f"http://127.0.0.1:{args.port}", timeout=120.0) as client:
_wait_http(client, "/healthz")
headers = _job_headers(args.token)
print("Boot and snapshot")
health = client.get("/healthz").json()
runner.check(health == {"ok": True}, "/healthz returns {ok:true}")
ready = client.get("/readyz")
runner.check(ready.status_code == 200 and ready.json()["model"] is True, "/readyz 200 model=true")
meta_before = client.get("/meta").json()
runner.check(meta_before["coverage"]["sensors"] == 8, "/meta covers 8 sensors")
runner.check(
meta_before["metrics"]["mae"] < meta_before["metrics"]["baseline_mae"],
"baked model beats seasonal naive on the holdout",
)
bootstrap = client.get("/bootstrap").json()
runner.check(len(bootstrap["sensors"]) == 8, "/bootstrap returns 8 sensors")
runner.check(
set(bootstrap["series"]) == set(SENSORS),
"/bootstrap series covers all 8 sensors",
)
series_points = sum(len(points) for points in bootstrap["series"].values())
runner.check(series_points > 20_000, f"/bootstrap carries {series_points} hourly points")
context = bootstrap["context"]
runner.check(
len(context["weather"]["forecast"]) == 168,
"/bootstrap context has 168 forecast weather hours",
)
runner.check(bool(context["holidays"]), "/bootstrap context has holidays")
runner.check(
_versions(bootstrap) == _versions(meta_before),
"/bootstrap versions match /meta",
)
report["versions_before"] = _versions(meta_before)
report["bootstrap"] = {"sensors": 8, "series_points": series_points}
print("Robustness: /predict every sensor x horizon x scenario")
predict_calls = 0
for sensor in SENSORS:
for horizon, count in HORIZON_SIZES.items():
for scenario in SCENARIOS:
response = client.post(
"/predict",
json={"sensor": sensor, "horizon": horizon, "scenario": scenario},
)
if response.status_code != 200:
raise CheckFailed(
f"/predict {sensor}/{horizon}/{scenario} -> "
f"{response.status_code}: {response.text}"
)
payload = response.json()
_assert_points(
payload["points"],
count,
f"/predict {sensor}/{horizon}/{scenario}",
)
predict_calls += 1
runner.check(predict_calls == 128, "128 /predict combinations returned valid points")
print("Robustness: /simulate every scenario, all sensors")
simulate_calls = 0
for scenario in SCENARIOS:
response = client.post(
"/simulate", json={"scenario": scenario, "sensors": "all"}
)
if response.status_code != 200:
raise CheckFailed(f"/simulate {scenario} -> {response.status_code}: {response.text}")
payload = response.json()
if set(payload["forecasts"]) != set(SENSORS):
raise CheckFailed(f"/simulate {scenario}: sensor set mismatch")
for sensor, by_horizon in payload["forecasts"].items():
if set(by_horizon) != set(HORIZON_SIZES):
raise CheckFailed(f"/simulate {scenario}/{sensor}: horizon set mismatch")
for horizon, count in HORIZON_SIZES.items():
_assert_points(
by_horizon[horizon],
count,
f"/simulate {scenario}/{sensor}/{horizon}",
)
simulate_calls += 1
runner.check(simulate_calls == 4, "4 /simulate scenarios x 8 sensors x 4 horizons")
cold = client.post(
"/predict",
json={"sensor": SENSORS[0], "horizon": "48h", "weather": {"temp_c": -8.0, "precip_mm": 6.0}},
).json()["points"]
warm = client.post(
"/predict",
json={"sensor": SENSORS[0], "horizon": "48h", "weather": {"temp_c": 30.0, "precip_mm": 0.0}},
).json()["points"]
runner.check(cold != warm, "weather override changes the forecast")
print("Error handling")
no_token = client.post("/predict", json={"sensor": "nope"})
runner.check(no_token.status_code == 404, "/predict unknown sensor -> 404")
unauthorized = _submit(client, {}, subset_path, "small")
runner.check(
unauthorized.status_code == 401
and unauthorized.json()["error"]["code"] == "unauthorized",
"/train without token -> 401 unauthorized",
)
bad_token = _submit(client, _job_headers("wrong-token"), subset_path, "small")
runner.check(bad_token.status_code == 401, "/train with wrong token -> 401")
unknown_job = client.get("/train/nope", headers=headers)
runner.check(
unknown_job.status_code == 404
and unknown_job.json()["error"]["code"] == "not_found",
"/train/nope -> 404 not_found",
)
bad_model = _submit(client, headers, subset_path, "giant")
runner.check(
bad_model.status_code == 400
and bad_model.json()["error"]["code"] == "bad_request",
"/train invalid model -> 400 bad_request",
)
missing_path = client.post(
"/train", headers=headers, json={"path": "/tmp/no-such-pulsecast.csv", "model": "small"}
)
runner.check(missing_path.status_code == 400, "/train missing JSON path -> 400")
print("Small model: real CSV subset through the API")
started = time.monotonic()
submitted = _submit(client, headers, subset_path, "small")
if submitted.status_code != 202:
raise CheckFailed(f"/train small -> {submitted.status_code}: {submitted.text}")
small_submitted = submitted.json()
runner.check(
small_submitted["status"] == "queued" and small_submitted["model"] == "small",
f"/train small queued as job {small_submitted['job_id']}",
)
small_job = _wait_job(client, small_submitted["job_id"], headers)
wall = round(time.monotonic() - started, 3)
if small_job["status"] != "succeeded":
raise CheckFailed(f"small job failed: {small_job}")
small_metrics = small_job["metrics"]
for key in ("accuracy", "precision", "recall", "f1", "roc_auc", "n_train", "n_test", "positive_rate"):
if key not in small_metrics:
raise CheckFailed(f"small metrics missing {key}: {small_metrics}")
train_seconds = _seconds(small_job["started_at"], small_job["finished_at"])
runner.check(
small_job["model_version"].startswith("logit-"),
f"small model_version {small_job['model_version']}",
)
runner.check(
wall < 5.0,
f"small job took {wall}s wall time including upload (contract < 5 s; "
f"server timestamps have 1 s resolution, train_seconds={train_seconds})",
)
runner.check(small_metrics["accuracy"] > 0.6, f"small accuracy {small_metrics['accuracy']}")
runner.check(
(models_dir / "small_latest.joblib").exists(),
"small model persisted to PULSECAST_MODELS",
)
meta_after_small = client.get("/meta").json()
runner.check(
meta_after_small["small_model"]["model_version"] == small_job["model_version"],
"small model hot-swapped into /meta",
)
runner.check(
meta_after_small["small_model"]["metrics"]["accuracy"] == small_metrics["accuracy"],
"hot-swapped small metrics match the job",
)
predict_after_small = client.post(
"/predict", json={"sensor": SENSORS[0], "horizon": "7d", "scenario": "base"}
)
runner.check(
predict_after_small.status_code == 200
and len(predict_after_small.json()["points"]) == 168,
"/predict still works after the small swap",
)
report["jobs"] = {
"small": {
"job": small_job,
"wall_seconds": wall,
"train_seconds": train_seconds,
"input": subset_info,
}
}
print("Forecast model: full CSV, hot-swap, 409 while busy")
forecast_started = time.monotonic()
forecast_response = _submit(client, headers, raw, "forecast")
if forecast_response.status_code != 202:
raise CheckFailed(
f"/train forecast -> {forecast_response.status_code}: {forecast_response.text}"
)
forecast_submitted = forecast_response.json()
runner.check(
forecast_submitted["model"] == "forecast",
f"/train forecast queued as job {forecast_submitted['job_id']}",
)
_wait_job(client, forecast_submitted["job_id"], headers, state="running", timeout=30)
busy = _submit(client, headers, subset_path, "small")
runner.check(
busy.status_code == 409 and busy.json()["error"]["code"] == "conflict",
"second /train while forecast is running -> 409 conflict",
)
forecast_job = _wait_job(client, forecast_submitted["job_id"], headers, timeout=300)
forecast_wall = round(time.monotonic() - forecast_started, 3)
if forecast_job["status"] != "succeeded":
raise CheckFailed(f"forecast job failed: {forecast_job}")
forecast_metrics = forecast_job["metrics"]
for key in ("mae", "wape", "r2", "baseline_mae", "n_eval"):
if key not in forecast_metrics:
raise CheckFailed(f"forecast metrics missing {key}: {forecast_metrics}")
runner.check(
forecast_job["model_version"].startswith("histgb-"),
f"forecast model_version {forecast_job['model_version']}",
)
runner.check(
forecast_metrics["mae"] < forecast_metrics["baseline_mae"],
f"retrained forecast MAE {forecast_metrics['mae']} < baseline {forecast_metrics['baseline_mae']}",
)
runner.check(
(models_dir / "forecast_latest.joblib").exists(),
"forecast model persisted to PULSECAST_MODELS",
)
meta_after_forecast = client.get("/meta").json()
runner.check(
meta_after_forecast["model_version"] == forecast_job["model_version"]
and meta_after_forecast["model_source"] == "retrained",
"forecast model hot-swapped into /meta as retrained",
)
predict_after_forecast = client.post(
"/predict", json={"sensor": SENSORS[-1], "horizon": "48h", "scenario": "rainy-saturday"}
)
runner.check(
predict_after_forecast.status_code == 200
and predict_after_forecast.json()["model_version"] == forecast_job["model_version"],
"/predict uses the retrained forecast model",
)
runner.check(client.get("/readyz").status_code == 200, "/readyz stays 200 after hot-swap")
report["jobs"]["forecast"] = {
"job": forecast_job,
"wall_seconds": forecast_wall,
"input": {"rows": "full challenge CSV", "bytes": raw.stat().st_size},
}
print("Malformed CSV fails the job (not the process)")
bad_csv = _submit(client, headers, garbage_path, "small")
runner.check(bad_csv.status_code == 202, "malformed CSV is accepted as a job")
bad_job = _wait_job(client, bad_csv.json()["job_id"], headers)
runner.check(
bad_job["status"] == "failed" and bool(bad_job["error"]),
f"malformed CSV job failed with {bad_job['error']!r}",
)
runner.check(
client.get("/healthz").status_code == 200,
"/healthz still 200 after a failed job",
)
report["versions_after_small"] = _versions(meta_after_small)
report["versions_after_forecast"] = _versions(meta_after_forecast)
report["checks"] = runner.checks
report["ok"] = True
finally:
process.terminate()
try:
process.wait(timeout=10)
except subprocess.TimeoutExpired:
process.kill()
process.wait(timeout=5)
log_handle.close()
if process.returncode not in (0, -15):
print(f"server exited with {process.returncode}; log: {log_path}", file=sys.stderr)
print(log_path.read_text(errors="replace")[-4000:], file=sys.stderr)
report["server_log_tail"] = log_path.read_text(errors="replace")[-1000:]
if args.keep:
print(f"kept workdir {workdir}")
else:
shutil.rmtree(workdir, ignore_errors=True)
report["finished_at"] = _utc_now()
return report
def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument("--port", type=int, default=18000)
parser.add_argument("--token", default=os.environ.get("CONTROL_TOKEN") or "e2e-local-token")
parser.add_argument("--csv", default=str(config.raw_csv_path()))
parser.add_argument("--days", type=int, default=60, help="days in the small-model CSV subset")
parser.add_argument("--report", type=Path, help="write the JSON result here")
parser.add_argument("--keep", action="store_true", help="keep the temporary workdir")
args = parser.parse_args(argv)
runner = Runner()
print(f"PulseCast end-to-end check on port {args.port}")
started = time.monotonic()
try:
report = run(args, runner)
except CheckFailed as exc:
print(f"\nFAIL after {runner.checks} checks: {exc}", file=sys.stderr)
return 1
elapsed = round(time.monotonic() - started, 1)
report["elapsed_seconds"] = elapsed
print(f"\nPASS: {runner.checks} checks in {elapsed}s")
if args.report:
args.report.parent.mkdir(parents=True, exist_ok=True)
args.report.write_text(json.dumps(report, indent=2, ensure_ascii=False) + "\n")
print(f"report: {args.report}")
return 0
if __name__ == "__main__":
raise SystemExit(main())
+262
View File
@@ -0,0 +1,262 @@
#!/usr/bin/env python3
"""Render dependency-free SVG figures from ``artifacts/metrics.json``.
The paper figures are generated from the measured holdout metrics, never
hand-written. Run ``make figures`` after ``make train``.
"""
from __future__ import annotations
import argparse
import json
from pathlib import Path
from xml.sax.saxutils import escape
from pulsecast import config
MODEL_COLOR = "#2563eb"
BASELINE_COLOR = "#94a3b8"
PRODUCTION_COLOR = "#0f766e"
TEXT_COLOR = "#0f172a"
GRID_COLOR = "#e2e8f0"
def _svg_document(
title: str,
subtitle: str,
body: list[str],
width: int,
height: int,
footer: str = "Source: artifacts/metrics.json · PulseCast 28-day time-based holdout",
) -> str:
return "\n".join(
[
f'<svg xmlns="http://www.w3.org/2000/svg" width="{width}" height="{height}" '
f'viewBox="0 0 {width} {height}" role="img" aria-label="{escape(title)}">',
'<rect width="100%" height="100%" fill="#ffffff"/>',
'<g font-family="-apple-system, Segoe UI, Helvetica, Arial, sans-serif">',
f'<text x="24" y="34" font-size="20" font-weight="600" fill="{TEXT_COLOR}">'
f"{escape(title)}</text>",
f'<text x="24" y="56" font-size="13" fill="#475569">{escape(subtitle)}</text>',
*body,
f'<text x="24" y="{height - 12}" font-size="11" fill="#64748b">'
f"{escape(footer)}</text>",
"</g>",
"</svg>",
"",
]
)
def _bar_chart(
title: str,
subtitle: str,
rows: list[dict],
*,
value_key: str = "mae",
secondary_key: str | None = None,
secondary_label: str | None = None,
primary_label: str = "MAE",
production: str | None = None,
) -> str:
width = 960
left = 230
right = 96
top = 86
row_height = 38
height = top + row_height * len(rows) + 56
plot_width = width - left - right
values = [row[value_key] for row in rows]
if secondary_key:
values += [row[secondary_key] for row in rows]
maximum = max(values) * 1.08 if values else 1.0
body: list[str] = []
for tick in range(5):
x = left + plot_width * tick / 4
value = maximum * tick / 4
body.append(
f'<line x1="{x:.1f}" y1="{top - 18}" x2="{x:.1f}" y2="{top + row_height * len(rows)}" '
f'stroke="{GRID_COLOR}" stroke-width="1"/>'
)
body.append(
f'<text x="{x:.1f}" y="{top - 24}" font-size="11" fill="#64748b" '
f'text-anchor="middle">{value:.1f}</text>'
)
legend_y = top - 34
if secondary_key:
body.append(
f'<rect x="{left}" y="{legend_y}" width="10" height="10" fill="{BASELINE_COLOR}"/>'
f'<text x="{left + 16}" y="{legend_y + 9}" font-size="12" fill="#475569">'
f"{escape(secondary_label or 'baseline')}</text>"
)
body.append(
f'<rect x="{left + 140}" y="{legend_y}" width="10" height="10" fill="{MODEL_COLOR}"/>'
f'<text x="{left + 156}" y="{legend_y + 9}" font-size="12" fill="#475569">'
f"{escape(primary_label)}</text>"
)
for index, row in enumerate(rows):
y = top + index * row_height
label = str(row.get("name") or row.get("sensor"))
body.append(
f'<text x="{left - 12}" y="{y + 20}" font-size="12" fill="{TEXT_COLOR}" '
f'text-anchor="end">{escape(label)}</text>'
)
if secondary_key:
baseline_width = plot_width * row[secondary_key] / maximum
body.append(
f'<rect x="{left}" y="{y + 3}" width="{baseline_width:.1f}" height="12" '
f'fill="{BASELINE_COLOR}"/>'
)
body.append(
f'<text x="{left + baseline_width + 6:.1f}" y="{y + 13}" font-size="11" '
f'fill="#64748b">{row[secondary_key]:.2f}</text>'
)
model_width = plot_width * row[value_key] / maximum
body.append(
f'<rect x="{left}" y="{y + 18}" width="{model_width:.1f}" height="12" '
f'fill="{MODEL_COLOR}"/>'
)
body.append(
f'<text x="{left + model_width + 6:.1f}" y="{y + 28}" font-size="11" '
f'fill="{TEXT_COLOR}">{row[value_key]:.2f}</text>'
)
else:
color = PRODUCTION_COLOR if production and row.get("name") == production else MODEL_COLOR
bar_width = plot_width * row[value_key] / maximum
body.append(
f'<rect x="{left}" y="{y + 6}" width="{bar_width:.1f}" height="22" fill="{color}"/>'
)
body.append(
f'<text x="{left + bar_width + 6:.1f}" y="{y + 21}" font-size="12" '
f'fill="{TEXT_COLOR}">{row[value_key]:.2f}</text>'
)
return _svg_document(title, subtitle, body, width, height)
def _confusion_svg(small: dict) -> str:
matrix = small["confusion_matrix"]
metrics = small["metrics"]
width, height = 760, 520
left, top, cell = 200, 130, 190
values = {
(0, 0): ("TN", matrix["tn"]),
(0, 1): ("FP", matrix["fp"]),
(1, 0): ("FN", matrix["fn"]),
(1, 1): ("TP", matrix["tp"]),
}
maximum = max(value for _, value in values.values())
body: list[str] = []
body.append(
f'<text x="{left + cell / 2}" y="{top - 16}" font-size="13" fill="#475569" '
f'text-anchor="middle">Predicted not busy</text>'
)
body.append(
f'<text x="{left + cell + cell / 2}" y="{top - 16}" font-size="13" fill="#475569" '
f'text-anchor="middle">Predicted busy</text>'
)
row_labels = ("Actually\nnot busy", "Actually\nbusy")
for row, label in enumerate(row_labels):
y = top + row * cell
body.append(
f'<text x="{left - 18}" y="{y + cell / 2}" font-size="13" fill="#475569" '
f'text-anchor="end">{escape(label).replace(chr(10), " ")}</text>'
)
for (row, column), (code, value) in values.items():
x = left + column * cell
y = top + row * cell
intensity = 0.12 + 0.88 * (value / maximum)
red = round(219 + (29 - 219) * intensity)
green = round(234 + (78 - 234) * intensity)
blue = round(254 + (216 - 254) * intensity)
fill = f"#{red:02x}{green:02x}{blue:02x}"
text_color = "#ffffff" if intensity > 0.55 else TEXT_COLOR
row_total = matrix["tn"] + matrix["fp"] if row == 0 else matrix["fn"] + matrix["tp"]
share = 100.0 * value / row_total if row_total else 0.0
body.append(
f'<rect x="{x}" y="{y}" width="{cell}" height="{cell}" fill="{fill}" '
f'stroke="#ffffff" stroke-width="2"/>'
)
body.append(
f'<text x="{x + cell / 2}" y="{y + cell / 2 - 8}" font-size="30" '
f'font-weight="600" fill="{text_color}" text-anchor="middle">{value:,}</text>'
)
body.append(
f'<text x="{x + cell / 2}" y="{y + cell / 2 + 20}" font-size="13" '
f'fill="{text_color}" text-anchor="middle">{code} · {share:.1f}% of row</text>'
)
subtitle = (
f"n={metrics['n_test']:,} · accuracy {metrics['accuracy']:.3f} · "
f"precision {metrics['precision']:.3f} · recall {metrics['recall']:.3f} · "
f"F1 {metrics['f1']:.3f} · ROC-AUC {metrics['roc_auc']:.3f}"
)
return _svg_document(
"Small busy-hour classifier: holdout confusion matrix",
subtitle,
body,
width,
height,
footer="Source: artifacts/small_metrics.json · chronological 80/20 split",
)
def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
"--artifacts",
type=Path,
default=config.artifacts_dir(),
help="artifact directory containing metrics.json (default: %(default)s)",
)
parser.add_argument(
"--out",
type=Path,
default=Path(__file__).resolve().parents[1] / "docs" / "figures",
help="output directory for SVG files (default: %(default)s)",
)
args = parser.parse_args(argv)
metrics = json.loads((args.artifacts / "metrics.json").read_text())
holdout = f"Holdout {metrics['test_from']} .. {metrics['test_to']}, n={metrics['metrics']['n_eval']}"
args.out.mkdir(parents=True, exist_ok=True)
ablation = _bar_chart(
"Ablation: holdout MAE by feature set",
holdout,
[
{
"name": row["name"].replace("_", " "),
"mae": row["mae"],
}
for row in metrics["ablation"]
],
production="week weather",
)
ablation_path = args.out / "ablation_mae.svg"
ablation_path.write_text(ablation)
per_sensor = _bar_chart(
"Per-sensor holdout MAE: production model vs seasonal naive",
holdout,
metrics["per_sensor"],
secondary_key="baseline_mae",
secondary_label="seasonal naive",
primary_label="weather-7d",
)
per_sensor_path = args.out / "per_sensor_mae.svg"
per_sensor_path.write_text(per_sensor)
confusion_path = args.out / "small_model_confusion.svg"
confusion_path.write_text(_confusion_svg(metrics["small"]))
print(f"wrote {ablation_path}")
print(f"wrote {per_sensor_path}")
print(f"wrote {confusion_path}")
return 0
if __name__ == "__main__":
raise SystemExit(main())
+153
View File
@@ -0,0 +1,153 @@
#!/usr/bin/env python3
"""Train and evaluate the small "busy hour" classifier from a CSV.
This is the repeatable artifact behind ``make small`` and the runtime
``model=small`` training job. It prints every number reported in
``docs/small-model.md``; ``--save`` bakes the bundle and metrics into
``artifacts/`` exactly like ``make train`` does.
Examples:
uv run python scripts/train_small.py
uv run python scripts/train_small.py --csv new-data.csv --all
uv run python scripts/train_small.py --save --json
"""
from __future__ import annotations
import argparse
import json
import sys
import time
from pathlib import Path
import joblib
import pandas as pd
from pulsecast import config
from pulsecast.small import load_hourly_csv, train_small_from_csv
def _parse_args(argv: list[str] | None = None) -> argparse.Namespace:
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument(
"csv_positional",
nargs="?",
metavar="CSV",
help=f"CSV to train from (default: {config.raw_csv_path()})",
)
parser.add_argument(
"--csv",
dest="csv_option",
help="CSV to train from; overrides the positional argument",
)
parser.add_argument(
"--start",
default=config.MULTI_SENSOR_START,
help=(
"drop rows before this UTC timestamp (default: %(default)s, the "
"8-sensor challenge window); ignored with --all"
),
)
parser.add_argument(
"--all",
action="store_true",
help="use every row in the CSV, including density-01's older history",
)
parser.add_argument(
"--save",
action="store_true",
help="write artifacts/small_model.joblib and artifacts/small_metrics.json",
)
parser.add_argument(
"--json",
action="store_true",
help="print the full metrics payload as JSON after the summary",
)
return parser.parse_args(argv)
def _confusion_lines(matrix: dict) -> list[str]:
tn, fp, fn, tp = matrix["tn"], matrix["fp"], matrix["fn"], matrix["tp"]
total = tn + fp + fn + tp
specificity = tn / (tn + fp) if tn + fp else 0.0
negative_predictive = tn / (tn + fn) if tn + fn else 0.0
return [
"Confusion matrix (rows = truth, columns = prediction)",
f" not busy: {tn:5d} {fp:5d}",
f" busy : {fn:5d} {tp:5d}",
f" n={total} specificity={specificity:.3f} NPV={negative_predictive:.3f}",
]
def main(argv: list[str] | None = None) -> int:
args = _parse_args(argv)
path = Path(args.csv_option or args.csv_positional or config.raw_csv_path())
if not path.exists():
print(f"CSV not found: {path}", file=sys.stderr)
return 2
start = None if args.all else pd.Timestamp(args.start)
started = time.perf_counter()
try:
bundle, payload = train_small_from_csv(path, start=start)
except (ValueError, FileNotFoundError) as exc:
print(f"training failed: {exc}", file=sys.stderr)
return 1
elapsed = time.perf_counter() - started
hourly = load_hourly_csv(path)
if start is not None:
hourly = hourly[hourly["t"] >= start]
metrics = payload["metrics"]
print(f"CSV {path}")
print(f"Rows {len(hourly):,} hourly rows, {metrics['sensors']} sensor(s)")
print(f"Window {payload['train_from']} .. {payload['test_to']}")
print(f"Split (time) train {payload['train_from']} .. {payload['train_to']}"
f" ({metrics['n_train']:,} rows)")
print(f" test {payload['test_from']} .. {payload['test_to']}"
f" ({metrics['n_test']:,} rows)")
print("Target busy = hourly density > sensor p75 (training window only)")
print(f"Features {', '.join(payload['features'])}")
print("Model standardized logistic regression (class_weight=balanced)")
print(f"Model version {payload['model_version']}")
print()
print("Metrics")
print(f" accuracy {metrics['accuracy']:.3f}")
print(f" precision {metrics['precision']:.3f}")
print(f" recall {metrics['recall']:.3f}")
print(f" f1 {metrics['f1']:.3f}")
print(f" roc_auc {metrics['roc_auc'] if metrics['roc_auc'] is not None else 'n/a'}")
print(f" base rate {metrics['positive_rate']:.3f}")
print()
print("\n".join(_confusion_lines(payload["confusion_matrix"])))
print()
print("Per sensor (holdout)")
print(f" {'sensor':<24} {'n':>6} {'pos':>6} {'acc':>6} {'prec':>6} {'rec':>6} {'f1':>6}")
for row in payload.get("per_sensor", []):
print(
f" {row['sensor']:<24} {row['n']:>6} {row['positive_rate']:>6.3f}"
f" {row['accuracy']:>6.3f} {row['precision']:>6.3f}"
f" {row['recall']:>6.3f} {row['f1']:>6.3f}"
)
print()
print(f"Fit time {elapsed:.2f} s (contract: < 5 s)")
if args.save:
artifacts = config.artifacts_dir()
artifacts.mkdir(parents=True, exist_ok=True)
joblib.dump(bundle, artifacts / "small_model.joblib")
(artifacts / "small_metrics.json").write_text(
json.dumps(payload, indent=2, ensure_ascii=False) + "\n"
)
print(f"Saved {artifacts / 'small_model.joblib'}")
print(f" {artifacts / 'small_metrics.json'}")
if args.json:
print()
print(json.dumps(payload, indent=2, ensure_ascii=False))
return 0
if __name__ == "__main__":
raise SystemExit(main())