From 2bcf66c55093e8b8b7b7cbb5cc3b6b7a42c540ff Mon Sep 17 00:00:00 2001 From: Noel Sabia Date: Sat, 14 Nov 2026 06:35:19 +0100 Subject: [PATCH] feat: repeatable small-model, figures, and training e2e scripts - make small trains from a CSV and prints metrics + confusion matrix - make figures renders dependency-free SVGs from metrics.json - scripts/e2e_train.py boots a real server and checks snapshot, robustness, auth, one-job-at-a-time, and both training jobs with hot-swap - CI/ruff/compileall now include scripts/ --- .gitea/workflows/ci.yml | 4 +- Makefile | 16 +- scripts/e2e_train.py | 523 ++++++++++++++++++++++++++++++++++++++++ scripts/plot_metrics.py | 262 ++++++++++++++++++++ scripts/train_small.py | 153 ++++++++++++ 5 files changed, 952 insertions(+), 6 deletions(-) create mode 100644 scripts/e2e_train.py create mode 100644 scripts/plot_metrics.py create mode 100644 scripts/train_small.py diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index 230f40f..1706dd4 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -32,10 +32,10 @@ jobs: run: uv sync --frozen - name: Static analysis (ruff) - run: uv run ruff check src tests main.py + run: uv run ruff check src tests scripts main.py - name: Byte-compile - run: uv run python -m compileall src + run: uv run python -m compileall src scripts - name: Tests run: uv run pytest -q diff --git a/Makefile b/Makefile index 02d5e44..3f1608b 100644 --- a/Makefile +++ b/Makefile @@ -1,11 +1,13 @@ UV ?= uv -.PHONY: help data train test lint check serve docker clean +.PHONY: help data train small figures test lint check serve docker clean help: @printf '%s\n' \ 'make data parse/clean the raw CSV, join weather + holidays, export JSON' \ 'make train fit the 28-day holdout model, export metrics/forecasts/model' \ + 'make small train + print the busy-hour classifier metrics from the raw CSV' \ + 'make figures regenerate docs/figures/*.svg from artifacts/metrics.json' \ 'make test run pytest' \ 'make lint run ruff static checks' \ 'make check CI gate: sync --frozen, ruff, compileall, pytest' \ @@ -19,16 +21,22 @@ data: train: $(UV) run python -m pulsecast.train +small: + $(UV) run python scripts/train_small.py + +figures: + $(UV) run python scripts/plot_metrics.py + test: $(UV) run pytest lint: - $(UV) run ruff check src tests main.py + $(UV) run ruff check src tests scripts main.py check: $(UV) sync --frozen - $(UV) run ruff check src tests main.py - $(UV) run python -m compileall src + $(UV) run ruff check src tests scripts main.py + $(UV) run python -m compileall src scripts $(UV) run pytest -q serve: diff --git a/scripts/e2e_train.py b/scripts/e2e_train.py new file mode 100644 index 0000000..3b988e0 --- /dev/null +++ b/scripts/e2e_train.py @@ -0,0 +1,523 @@ +#!/usr/bin/env python3 +"""End-to-end acceptance run against a real FastAPI process on port 18000. + +Starts uvicorn with ``CONTROL_TOKEN``, then exercises robustness and the full +training lifecycle: + +- /healthz, /readyz, /meta, /bootstrap (8 sensors, hourly series, context) +- /predict for every sensor x horizon x scenario (128 calls) +- /simulate for every scenario with all sensors +- auth, unknown job, unknown model, malformed CSV, one-job-at-a-time +- POST /train model=small on a real CSV subset; poll to success; hot-swap +- POST /train model=forecast on the full CSV; poll to success; hot-swap +- /predict still works after each swap + +Exit code is 0 only when every check passes. Figures printed to stdout are the +measured numbers; ``--report`` writes the same result as JSON. +""" + +from __future__ import annotations + +import argparse +import json +import os +import shutil +import socket +import subprocess +import sys +import tempfile +import time +from datetime import UTC, datetime +from pathlib import Path + +import httpx +import pandas as pd + +from pulsecast import config + +HORIZON_SIZES = {"15m": 1, "1h": 4, "48h": 48, "7d": 168} +SCENARIOS = tuple(config.SCENARIOS) +SENSORS = tuple(sorted(config.USABLE_SENSORS)) + + +class CheckFailed(RuntimeError): + pass + + +class Runner: + def __init__(self) -> None: + self.checks = 0 + self.results: dict = {} + + def check(self, condition: bool, message: str) -> None: + self.checks += 1 + if not condition: + raise CheckFailed(message) + print(f" [ok] {message}") + + +def _utc_now() -> str: + return datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def _parse_ts(value: str) -> pd.Timestamp: + return pd.Timestamp(value) + + +def _seconds(start: str | None, end: str | None) -> float | None: + if not start or not end: + return None + return round((_parse_ts(end) - _parse_ts(start)).total_seconds(), 3) + + +def _wait_http(client: httpx.Client, url: str, timeout: float = 30.0) -> None: + deadline = time.monotonic() + timeout + last: Exception | None = None + while time.monotonic() < deadline: + try: + response = client.get(url, timeout=2.0) + if response.status_code == 200: + return + except Exception as exc: # noqa: BLE001 - startup race + last = exc + time.sleep(0.1) + raise CheckFailed(f"server did not answer {url} within {timeout}s: {last}") + + +def _wait_job( + client: httpx.Client, + job_id: str, + headers: dict[str, str], + *, + timeout: float = 180.0, + state: str | None = None, +) -> dict: + deadline = time.monotonic() + timeout + last_status = None + while time.monotonic() < deadline: + response = client.get(f"/train/{job_id}", headers=headers) + if response.status_code != 200: + raise CheckFailed(f"GET /train/{job_id} -> {response.status_code}: {response.text}") + job = response.json() + last_status = job["status"] + if state is not None and last_status == state: + return job + if state is None and last_status in {"succeeded", "failed"}: + return job + time.sleep(0.05) + raise CheckFailed(f"job {job_id} did not reach {state or 'terminal'} (last={last_status})") + + +def _job_headers(token: str) -> dict[str, str]: + return {"Authorization": f"Bearer {token}"} + + +def _submit( + client: httpx.Client, headers: dict[str, str], path: Path, model: str +) -> httpx.Response: + return client.post( + "/train", + headers=headers, + files={"file": (path.name, path.read_bytes(), "text/csv")}, + data={"model": model}, + timeout=120.0, + ) + + +def _assert_points(points: list[dict], expected: int, label: str) -> None: + if len(points) != expected: + raise CheckFailed(f"{label}: expected {expected} points, got {len(points)}") + previous = None + for point in points: + if set(point) != {"t", "v", "lo", "hi"}: + raise CheckFailed(f"{label}: unexpected point keys {sorted(point)}") + if not point["t"].endswith("Z"): + raise CheckFailed(f"{label}: timestamp is not UTC RFC 3339: {point['t']}") + if point["v"] < 0 or point["lo"] > point["v"] or point["v"] > point["hi"]: + raise CheckFailed(f"{label}: invalid band {point}") + stamp = _parse_ts(point["t"]) + if previous is not None and stamp <= previous: + raise CheckFailed(f"{label}: timestamps are not increasing at {point['t']}") + previous = stamp + + +def _versions(payload: dict) -> dict: + return { + key: payload.get(key) + for key in ("data_version", "model_version", "context_version") + } + + +def _build_subset(raw: Path, destination: Path, days: int = 60) -> tuple[Path, dict]: + frame = pd.read_csv(raw) + frame = frame[frame["type"].astype(str) == "densityData"].copy() + frame["time"] = pd.to_datetime(frame["time"], utc=True) + end = frame["time"].max() + cut = end - pd.Timedelta(days=days) + frame = frame[frame["time"] >= cut][["time", "name", "value"]] + frame["time"] = frame["time"].dt.strftime("%Y-%m-%dT%H:%M:%SZ") + frame.to_csv(destination, index=False) + return destination, { + "rows": len(frame), + "sensors": int(frame["name"].nunique()), + "from": frame["time"].min(), + "to": frame["time"].max(), + "days": days, + } + + +def _port_free(port: int) -> bool: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + sock.settimeout(0.5) + return sock.connect_ex(("127.0.0.1", port)) != 0 + + +def run(args: argparse.Namespace, runner: Runner) -> dict: + raw = Path(args.csv) + if not raw.exists(): + raise CheckFailed(f"CSV not found: {raw}") + if not _port_free(args.port): + raise CheckFailed(f"port {args.port} is already in use") + + workdir = Path(tempfile.mkdtemp(prefix="pulsecast-e2e-")) + models_dir = workdir / "models" + uploads_dir = workdir / "uploads" + subset_path, subset_info = _build_subset(raw, workdir / "small-subset.csv", args.days) + garbage_path = workdir / "garbage.csv" + garbage_path.write_text("not,a,pulsecast,file\n1,2,3,4\n") + + env = os.environ.copy() + env.update( + CONTROL_TOKEN=args.token, + PULSECAST_MODELS=str(models_dir), + PULSECAST_UPLOADS=str(uploads_dir), + ) + log_path = workdir / "server.log" + log_handle = log_path.open("wb") + command = [ + sys.executable, + "-m", + "uvicorn", + "pulsecast.server:app", + "--host", + "127.0.0.1", + "--port", + str(args.port), + "--log-level", + "warning", + ] + process = subprocess.Popen(command, env=env, stdout=log_handle, stderr=subprocess.STDOUT) + report: dict = { + "started_at": _utc_now(), + "base_url": f"http://127.0.0.1:{args.port}", + "subset": subset_info, + "raw_csv_bytes": raw.stat().st_size, + } + + try: + with httpx.Client(base_url=f"http://127.0.0.1:{args.port}", timeout=120.0) as client: + _wait_http(client, "/healthz") + headers = _job_headers(args.token) + + print("Boot and snapshot") + health = client.get("/healthz").json() + runner.check(health == {"ok": True}, "/healthz returns {ok:true}") + ready = client.get("/readyz") + runner.check(ready.status_code == 200 and ready.json()["model"] is True, "/readyz 200 model=true") + meta_before = client.get("/meta").json() + runner.check(meta_before["coverage"]["sensors"] == 8, "/meta covers 8 sensors") + runner.check( + meta_before["metrics"]["mae"] < meta_before["metrics"]["baseline_mae"], + "baked model beats seasonal naive on the holdout", + ) + bootstrap = client.get("/bootstrap").json() + runner.check(len(bootstrap["sensors"]) == 8, "/bootstrap returns 8 sensors") + runner.check( + set(bootstrap["series"]) == set(SENSORS), + "/bootstrap series covers all 8 sensors", + ) + series_points = sum(len(points) for points in bootstrap["series"].values()) + runner.check(series_points > 20_000, f"/bootstrap carries {series_points} hourly points") + context = bootstrap["context"] + runner.check( + len(context["weather"]["forecast"]) == 168, + "/bootstrap context has 168 forecast weather hours", + ) + runner.check(bool(context["holidays"]), "/bootstrap context has holidays") + runner.check( + _versions(bootstrap) == _versions(meta_before), + "/bootstrap versions match /meta", + ) + report["versions_before"] = _versions(meta_before) + report["bootstrap"] = {"sensors": 8, "series_points": series_points} + + print("Robustness: /predict every sensor x horizon x scenario") + predict_calls = 0 + for sensor in SENSORS: + for horizon, count in HORIZON_SIZES.items(): + for scenario in SCENARIOS: + response = client.post( + "/predict", + json={"sensor": sensor, "horizon": horizon, "scenario": scenario}, + ) + if response.status_code != 200: + raise CheckFailed( + f"/predict {sensor}/{horizon}/{scenario} -> " + f"{response.status_code}: {response.text}" + ) + payload = response.json() + _assert_points( + payload["points"], + count, + f"/predict {sensor}/{horizon}/{scenario}", + ) + predict_calls += 1 + runner.check(predict_calls == 128, "128 /predict combinations returned valid points") + + print("Robustness: /simulate every scenario, all sensors") + simulate_calls = 0 + for scenario in SCENARIOS: + response = client.post( + "/simulate", json={"scenario": scenario, "sensors": "all"} + ) + if response.status_code != 200: + raise CheckFailed(f"/simulate {scenario} -> {response.status_code}: {response.text}") + payload = response.json() + if set(payload["forecasts"]) != set(SENSORS): + raise CheckFailed(f"/simulate {scenario}: sensor set mismatch") + for sensor, by_horizon in payload["forecasts"].items(): + if set(by_horizon) != set(HORIZON_SIZES): + raise CheckFailed(f"/simulate {scenario}/{sensor}: horizon set mismatch") + for horizon, count in HORIZON_SIZES.items(): + _assert_points( + by_horizon[horizon], + count, + f"/simulate {scenario}/{sensor}/{horizon}", + ) + simulate_calls += 1 + runner.check(simulate_calls == 4, "4 /simulate scenarios x 8 sensors x 4 horizons") + + cold = client.post( + "/predict", + json={"sensor": SENSORS[0], "horizon": "48h", "weather": {"temp_c": -8.0, "precip_mm": 6.0}}, + ).json()["points"] + warm = client.post( + "/predict", + json={"sensor": SENSORS[0], "horizon": "48h", "weather": {"temp_c": 30.0, "precip_mm": 0.0}}, + ).json()["points"] + runner.check(cold != warm, "weather override changes the forecast") + + print("Error handling") + no_token = client.post("/predict", json={"sensor": "nope"}) + runner.check(no_token.status_code == 404, "/predict unknown sensor -> 404") + unauthorized = _submit(client, {}, subset_path, "small") + runner.check( + unauthorized.status_code == 401 + and unauthorized.json()["error"]["code"] == "unauthorized", + "/train without token -> 401 unauthorized", + ) + bad_token = _submit(client, _job_headers("wrong-token"), subset_path, "small") + runner.check(bad_token.status_code == 401, "/train with wrong token -> 401") + unknown_job = client.get("/train/nope", headers=headers) + runner.check( + unknown_job.status_code == 404 + and unknown_job.json()["error"]["code"] == "not_found", + "/train/nope -> 404 not_found", + ) + bad_model = _submit(client, headers, subset_path, "giant") + runner.check( + bad_model.status_code == 400 + and bad_model.json()["error"]["code"] == "bad_request", + "/train invalid model -> 400 bad_request", + ) + missing_path = client.post( + "/train", headers=headers, json={"path": "/tmp/no-such-pulsecast.csv", "model": "small"} + ) + runner.check(missing_path.status_code == 400, "/train missing JSON path -> 400") + + print("Small model: real CSV subset through the API") + started = time.monotonic() + submitted = _submit(client, headers, subset_path, "small") + if submitted.status_code != 202: + raise CheckFailed(f"/train small -> {submitted.status_code}: {submitted.text}") + small_submitted = submitted.json() + runner.check( + small_submitted["status"] == "queued" and small_submitted["model"] == "small", + f"/train small queued as job {small_submitted['job_id']}", + ) + small_job = _wait_job(client, small_submitted["job_id"], headers) + wall = round(time.monotonic() - started, 3) + if small_job["status"] != "succeeded": + raise CheckFailed(f"small job failed: {small_job}") + small_metrics = small_job["metrics"] + for key in ("accuracy", "precision", "recall", "f1", "roc_auc", "n_train", "n_test", "positive_rate"): + if key not in small_metrics: + raise CheckFailed(f"small metrics missing {key}: {small_metrics}") + train_seconds = _seconds(small_job["started_at"], small_job["finished_at"]) + runner.check( + small_job["model_version"].startswith("logit-"), + f"small model_version {small_job['model_version']}", + ) + runner.check( + wall < 5.0, + f"small job took {wall}s wall time including upload (contract < 5 s; " + f"server timestamps have 1 s resolution, train_seconds={train_seconds})", + ) + runner.check(small_metrics["accuracy"] > 0.6, f"small accuracy {small_metrics['accuracy']}") + runner.check( + (models_dir / "small_latest.joblib").exists(), + "small model persisted to PULSECAST_MODELS", + ) + meta_after_small = client.get("/meta").json() + runner.check( + meta_after_small["small_model"]["model_version"] == small_job["model_version"], + "small model hot-swapped into /meta", + ) + runner.check( + meta_after_small["small_model"]["metrics"]["accuracy"] == small_metrics["accuracy"], + "hot-swapped small metrics match the job", + ) + predict_after_small = client.post( + "/predict", json={"sensor": SENSORS[0], "horizon": "7d", "scenario": "base"} + ) + runner.check( + predict_after_small.status_code == 200 + and len(predict_after_small.json()["points"]) == 168, + "/predict still works after the small swap", + ) + report["jobs"] = { + "small": { + "job": small_job, + "wall_seconds": wall, + "train_seconds": train_seconds, + "input": subset_info, + } + } + + print("Forecast model: full CSV, hot-swap, 409 while busy") + forecast_started = time.monotonic() + forecast_response = _submit(client, headers, raw, "forecast") + if forecast_response.status_code != 202: + raise CheckFailed( + f"/train forecast -> {forecast_response.status_code}: {forecast_response.text}" + ) + forecast_submitted = forecast_response.json() + runner.check( + forecast_submitted["model"] == "forecast", + f"/train forecast queued as job {forecast_submitted['job_id']}", + ) + _wait_job(client, forecast_submitted["job_id"], headers, state="running", timeout=30) + busy = _submit(client, headers, subset_path, "small") + runner.check( + busy.status_code == 409 and busy.json()["error"]["code"] == "conflict", + "second /train while forecast is running -> 409 conflict", + ) + forecast_job = _wait_job(client, forecast_submitted["job_id"], headers, timeout=300) + forecast_wall = round(time.monotonic() - forecast_started, 3) + if forecast_job["status"] != "succeeded": + raise CheckFailed(f"forecast job failed: {forecast_job}") + forecast_metrics = forecast_job["metrics"] + for key in ("mae", "wape", "r2", "baseline_mae", "n_eval"): + if key not in forecast_metrics: + raise CheckFailed(f"forecast metrics missing {key}: {forecast_metrics}") + runner.check( + forecast_job["model_version"].startswith("histgb-"), + f"forecast model_version {forecast_job['model_version']}", + ) + runner.check( + forecast_metrics["mae"] < forecast_metrics["baseline_mae"], + f"retrained forecast MAE {forecast_metrics['mae']} < baseline {forecast_metrics['baseline_mae']}", + ) + runner.check( + (models_dir / "forecast_latest.joblib").exists(), + "forecast model persisted to PULSECAST_MODELS", + ) + meta_after_forecast = client.get("/meta").json() + runner.check( + meta_after_forecast["model_version"] == forecast_job["model_version"] + and meta_after_forecast["model_source"] == "retrained", + "forecast model hot-swapped into /meta as retrained", + ) + predict_after_forecast = client.post( + "/predict", json={"sensor": SENSORS[-1], "horizon": "48h", "scenario": "rainy-saturday"} + ) + runner.check( + predict_after_forecast.status_code == 200 + and predict_after_forecast.json()["model_version"] == forecast_job["model_version"], + "/predict uses the retrained forecast model", + ) + runner.check(client.get("/readyz").status_code == 200, "/readyz stays 200 after hot-swap") + report["jobs"]["forecast"] = { + "job": forecast_job, + "wall_seconds": forecast_wall, + "input": {"rows": "full challenge CSV", "bytes": raw.stat().st_size}, + } + + print("Malformed CSV fails the job (not the process)") + bad_csv = _submit(client, headers, garbage_path, "small") + runner.check(bad_csv.status_code == 202, "malformed CSV is accepted as a job") + bad_job = _wait_job(client, bad_csv.json()["job_id"], headers) + runner.check( + bad_job["status"] == "failed" and bool(bad_job["error"]), + f"malformed CSV job failed with {bad_job['error']!r}", + ) + runner.check( + client.get("/healthz").status_code == 200, + "/healthz still 200 after a failed job", + ) + + report["versions_after_small"] = _versions(meta_after_small) + report["versions_after_forecast"] = _versions(meta_after_forecast) + report["checks"] = runner.checks + report["ok"] = True + finally: + process.terminate() + try: + process.wait(timeout=10) + except subprocess.TimeoutExpired: + process.kill() + process.wait(timeout=5) + log_handle.close() + if process.returncode not in (0, -15): + print(f"server exited with {process.returncode}; log: {log_path}", file=sys.stderr) + print(log_path.read_text(errors="replace")[-4000:], file=sys.stderr) + report["server_log_tail"] = log_path.read_text(errors="replace")[-1000:] + if args.keep: + print(f"kept workdir {workdir}") + else: + shutil.rmtree(workdir, ignore_errors=True) + + report["finished_at"] = _utc_now() + return report + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--port", type=int, default=18000) + parser.add_argument("--token", default=os.environ.get("CONTROL_TOKEN") or "e2e-local-token") + parser.add_argument("--csv", default=str(config.raw_csv_path())) + parser.add_argument("--days", type=int, default=60, help="days in the small-model CSV subset") + parser.add_argument("--report", type=Path, help="write the JSON result here") + parser.add_argument("--keep", action="store_true", help="keep the temporary workdir") + args = parser.parse_args(argv) + + runner = Runner() + print(f"PulseCast end-to-end check on port {args.port}") + started = time.monotonic() + try: + report = run(args, runner) + except CheckFailed as exc: + print(f"\nFAIL after {runner.checks} checks: {exc}", file=sys.stderr) + return 1 + elapsed = round(time.monotonic() - started, 1) + report["elapsed_seconds"] = elapsed + print(f"\nPASS: {runner.checks} checks in {elapsed}s") + if args.report: + args.report.parent.mkdir(parents=True, exist_ok=True) + args.report.write_text(json.dumps(report, indent=2, ensure_ascii=False) + "\n") + print(f"report: {args.report}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/plot_metrics.py b/scripts/plot_metrics.py new file mode 100644 index 0000000..d9a4350 --- /dev/null +++ b/scripts/plot_metrics.py @@ -0,0 +1,262 @@ +#!/usr/bin/env python3 +"""Render dependency-free SVG figures from ``artifacts/metrics.json``. + +The paper figures are generated from the measured holdout metrics, never +hand-written. Run ``make figures`` after ``make train``. +""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path +from xml.sax.saxutils import escape + +from pulsecast import config + +MODEL_COLOR = "#2563eb" +BASELINE_COLOR = "#94a3b8" +PRODUCTION_COLOR = "#0f766e" +TEXT_COLOR = "#0f172a" +GRID_COLOR = "#e2e8f0" + + +def _svg_document( + title: str, + subtitle: str, + body: list[str], + width: int, + height: int, + footer: str = "Source: artifacts/metrics.json · PulseCast 28-day time-based holdout", +) -> str: + return "\n".join( + [ + f'', + '', + '', + f'' + f"{escape(title)}", + f'{escape(subtitle)}', + *body, + f'' + f"{escape(footer)}", + "", + "", + "", + ] + ) + + +def _bar_chart( + title: str, + subtitle: str, + rows: list[dict], + *, + value_key: str = "mae", + secondary_key: str | None = None, + secondary_label: str | None = None, + primary_label: str = "MAE", + production: str | None = None, +) -> str: + width = 960 + left = 230 + right = 96 + top = 86 + row_height = 38 + height = top + row_height * len(rows) + 56 + plot_width = width - left - right + values = [row[value_key] for row in rows] + if secondary_key: + values += [row[secondary_key] for row in rows] + maximum = max(values) * 1.08 if values else 1.0 + + body: list[str] = [] + for tick in range(5): + x = left + plot_width * tick / 4 + value = maximum * tick / 4 + body.append( + f'' + ) + body.append( + f'{value:.1f}' + ) + + legend_y = top - 34 + if secondary_key: + body.append( + f'' + f'' + f"{escape(secondary_label or 'baseline')}" + ) + body.append( + f'' + f'' + f"{escape(primary_label)}" + ) + + for index, row in enumerate(rows): + y = top + index * row_height + label = str(row.get("name") or row.get("sensor")) + body.append( + f'{escape(label)}' + ) + if secondary_key: + baseline_width = plot_width * row[secondary_key] / maximum + body.append( + f'' + ) + body.append( + f'{row[secondary_key]:.2f}' + ) + model_width = plot_width * row[value_key] / maximum + body.append( + f'' + ) + body.append( + f'{row[value_key]:.2f}' + ) + else: + color = PRODUCTION_COLOR if production and row.get("name") == production else MODEL_COLOR + bar_width = plot_width * row[value_key] / maximum + body.append( + f'' + ) + body.append( + f'{row[value_key]:.2f}' + ) + + return _svg_document(title, subtitle, body, width, height) + + +def _confusion_svg(small: dict) -> str: + matrix = small["confusion_matrix"] + metrics = small["metrics"] + width, height = 760, 520 + left, top, cell = 200, 130, 190 + values = { + (0, 0): ("TN", matrix["tn"]), + (0, 1): ("FP", matrix["fp"]), + (1, 0): ("FN", matrix["fn"]), + (1, 1): ("TP", matrix["tp"]), + } + maximum = max(value for _, value in values.values()) + body: list[str] = [] + body.append( + f'Predicted not busy' + ) + body.append( + f'Predicted busy' + ) + row_labels = ("Actually\nnot busy", "Actually\nbusy") + for row, label in enumerate(row_labels): + y = top + row * cell + body.append( + f'{escape(label).replace(chr(10), " ")}' + ) + for (row, column), (code, value) in values.items(): + x = left + column * cell + y = top + row * cell + intensity = 0.12 + 0.88 * (value / maximum) + red = round(219 + (29 - 219) * intensity) + green = round(234 + (78 - 234) * intensity) + blue = round(254 + (216 - 254) * intensity) + fill = f"#{red:02x}{green:02x}{blue:02x}" + text_color = "#ffffff" if intensity > 0.55 else TEXT_COLOR + row_total = matrix["tn"] + matrix["fp"] if row == 0 else matrix["fn"] + matrix["tp"] + share = 100.0 * value / row_total if row_total else 0.0 + body.append( + f'' + ) + body.append( + f'{value:,}' + ) + body.append( + f'{code} · {share:.1f}% of row' + ) + subtitle = ( + f"n={metrics['n_test']:,} · accuracy {metrics['accuracy']:.3f} · " + f"precision {metrics['precision']:.3f} · recall {metrics['recall']:.3f} · " + f"F1 {metrics['f1']:.3f} · ROC-AUC {metrics['roc_auc']:.3f}" + ) + return _svg_document( + "Small busy-hour classifier: holdout confusion matrix", + subtitle, + body, + width, + height, + footer="Source: artifacts/small_metrics.json · chronological 80/20 split", + ) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--artifacts", + type=Path, + default=config.artifacts_dir(), + help="artifact directory containing metrics.json (default: %(default)s)", + ) + parser.add_argument( + "--out", + type=Path, + default=Path(__file__).resolve().parents[1] / "docs" / "figures", + help="output directory for SVG files (default: %(default)s)", + ) + args = parser.parse_args(argv) + + metrics = json.loads((args.artifacts / "metrics.json").read_text()) + holdout = f"Holdout {metrics['test_from']} .. {metrics['test_to']}, n={metrics['metrics']['n_eval']}" + args.out.mkdir(parents=True, exist_ok=True) + + ablation = _bar_chart( + "Ablation: holdout MAE by feature set", + holdout, + [ + { + "name": row["name"].replace("_", " "), + "mae": row["mae"], + } + for row in metrics["ablation"] + ], + production="week weather", + ) + ablation_path = args.out / "ablation_mae.svg" + ablation_path.write_text(ablation) + + per_sensor = _bar_chart( + "Per-sensor holdout MAE: production model vs seasonal naive", + holdout, + metrics["per_sensor"], + secondary_key="baseline_mae", + secondary_label="seasonal naive", + primary_label="weather-7d", + ) + per_sensor_path = args.out / "per_sensor_mae.svg" + per_sensor_path.write_text(per_sensor) + + confusion_path = args.out / "small_model_confusion.svg" + confusion_path.write_text(_confusion_svg(metrics["small"])) + + print(f"wrote {ablation_path}") + print(f"wrote {per_sensor_path}") + print(f"wrote {confusion_path}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/train_small.py b/scripts/train_small.py new file mode 100644 index 0000000..9add34e --- /dev/null +++ b/scripts/train_small.py @@ -0,0 +1,153 @@ +#!/usr/bin/env python3 +"""Train and evaluate the small "busy hour" classifier from a CSV. + +This is the repeatable artifact behind ``make small`` and the runtime +``model=small`` training job. It prints every number reported in +``docs/small-model.md``; ``--save`` bakes the bundle and metrics into +``artifacts/`` exactly like ``make train`` does. + +Examples: + uv run python scripts/train_small.py + uv run python scripts/train_small.py --csv new-data.csv --all + uv run python scripts/train_small.py --save --json +""" + +from __future__ import annotations + +import argparse +import json +import sys +import time +from pathlib import Path + +import joblib +import pandas as pd + +from pulsecast import config +from pulsecast.small import load_hourly_csv, train_small_from_csv + + +def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument( + "csv_positional", + nargs="?", + metavar="CSV", + help=f"CSV to train from (default: {config.raw_csv_path()})", + ) + parser.add_argument( + "--csv", + dest="csv_option", + help="CSV to train from; overrides the positional argument", + ) + parser.add_argument( + "--start", + default=config.MULTI_SENSOR_START, + help=( + "drop rows before this UTC timestamp (default: %(default)s, the " + "8-sensor challenge window); ignored with --all" + ), + ) + parser.add_argument( + "--all", + action="store_true", + help="use every row in the CSV, including density-01's older history", + ) + parser.add_argument( + "--save", + action="store_true", + help="write artifacts/small_model.joblib and artifacts/small_metrics.json", + ) + parser.add_argument( + "--json", + action="store_true", + help="print the full metrics payload as JSON after the summary", + ) + return parser.parse_args(argv) + + +def _confusion_lines(matrix: dict) -> list[str]: + tn, fp, fn, tp = matrix["tn"], matrix["fp"], matrix["fn"], matrix["tp"] + total = tn + fp + fn + tp + specificity = tn / (tn + fp) if tn + fp else 0.0 + negative_predictive = tn / (tn + fn) if tn + fn else 0.0 + return [ + "Confusion matrix (rows = truth, columns = prediction)", + f" not busy: {tn:5d} {fp:5d}", + f" busy : {fn:5d} {tp:5d}", + f" n={total} specificity={specificity:.3f} NPV={negative_predictive:.3f}", + ] + + +def main(argv: list[str] | None = None) -> int: + args = _parse_args(argv) + path = Path(args.csv_option or args.csv_positional or config.raw_csv_path()) + if not path.exists(): + print(f"CSV not found: {path}", file=sys.stderr) + return 2 + + start = None if args.all else pd.Timestamp(args.start) + started = time.perf_counter() + try: + bundle, payload = train_small_from_csv(path, start=start) + except (ValueError, FileNotFoundError) as exc: + print(f"training failed: {exc}", file=sys.stderr) + return 1 + elapsed = time.perf_counter() - started + + hourly = load_hourly_csv(path) + if start is not None: + hourly = hourly[hourly["t"] >= start] + metrics = payload["metrics"] + + print(f"CSV {path}") + print(f"Rows {len(hourly):,} hourly rows, {metrics['sensors']} sensor(s)") + print(f"Window {payload['train_from']} .. {payload['test_to']}") + print(f"Split (time) train {payload['train_from']} .. {payload['train_to']}" + f" ({metrics['n_train']:,} rows)") + print(f" test {payload['test_from']} .. {payload['test_to']}" + f" ({metrics['n_test']:,} rows)") + print("Target busy = hourly density > sensor p75 (training window only)") + print(f"Features {', '.join(payload['features'])}") + print("Model standardized logistic regression (class_weight=balanced)") + print(f"Model version {payload['model_version']}") + print() + print("Metrics") + print(f" accuracy {metrics['accuracy']:.3f}") + print(f" precision {metrics['precision']:.3f}") + print(f" recall {metrics['recall']:.3f}") + print(f" f1 {metrics['f1']:.3f}") + print(f" roc_auc {metrics['roc_auc'] if metrics['roc_auc'] is not None else 'n/a'}") + print(f" base rate {metrics['positive_rate']:.3f}") + print() + print("\n".join(_confusion_lines(payload["confusion_matrix"]))) + print() + print("Per sensor (holdout)") + print(f" {'sensor':<24} {'n':>6} {'pos':>6} {'acc':>6} {'prec':>6} {'rec':>6} {'f1':>6}") + for row in payload.get("per_sensor", []): + print( + f" {row['sensor']:<24} {row['n']:>6} {row['positive_rate']:>6.3f}" + f" {row['accuracy']:>6.3f} {row['precision']:>6.3f}" + f" {row['recall']:>6.3f} {row['f1']:>6.3f}" + ) + print() + print(f"Fit time {elapsed:.2f} s (contract: < 5 s)") + + if args.save: + artifacts = config.artifacts_dir() + artifacts.mkdir(parents=True, exist_ok=True) + joblib.dump(bundle, artifacts / "small_model.joblib") + (artifacts / "small_metrics.json").write_text( + json.dumps(payload, indent=2, ensure_ascii=False) + "\n" + ) + print(f"Saved {artifacts / 'small_model.joblib'}") + print(f" {artifacts / 'small_metrics.json'}") + + if args.json: + print() + print(json.dumps(payload, indent=2, ensure_ascii=False)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())