feat: repeatable small-model, figures, and training e2e scripts
- make small trains from a CSV and prints metrics + confusion matrix - make figures renders dependency-free SVGs from metrics.json - scripts/e2e_train.py boots a real server and checks snapshot, robustness, auth, one-job-at-a-time, and both training jobs with hot-swap - CI/ruff/compileall now include scripts/
This commit is contained in:
1 parent
000aaed7a5
commit
2bcf66c550
5 files changed
+952
-6
No files matched your search
@@ -32,10 +32,10 @@ jobs:
|
||||
run: uv sync --frozen
|
||||
|
||||
- name: Static analysis (ruff)
|
||||
run: uv run ruff check src tests main.py
|
||||
run: uv run ruff check src tests scripts main.py
|
||||
|
||||
- name: Byte-compile
|
||||
run: uv run python -m compileall src
|
||||
run: uv run python -m compileall src scripts
|
||||
|
||||
- name: Tests
|
||||
run: uv run pytest -q
|
||||
@@ -1,11 +1,13 @@
|
||||
UV ?= uv
|
||||
|
||||
.PHONY: help data train test lint check serve docker clean
|
||||
.PHONY: help data train small figures test lint check serve docker clean
|
||||
|
||||
help:
|
||||
@printf '%s\n' \
|
||||
'make data parse/clean the raw CSV, join weather + holidays, export JSON' \
|
||||
'make train fit the 28-day holdout model, export metrics/forecasts/model' \
|
||||
'make small train + print the busy-hour classifier metrics from the raw CSV' \
|
||||
'make figures regenerate docs/figures/*.svg from artifacts/metrics.json' \
|
||||
'make test run pytest' \
|
||||
'make lint run ruff static checks' \
|
||||
'make check CI gate: sync --frozen, ruff, compileall, pytest' \
|
||||
@@ -19,16 +21,22 @@ data:
|
||||
train:
|
||||
$(UV) run python -m pulsecast.train
|
||||
|
||||
small:
|
||||
$(UV) run python scripts/train_small.py
|
||||
|
||||
figures:
|
||||
$(UV) run python scripts/plot_metrics.py
|
||||
|
||||
test:
|
||||
$(UV) run pytest
|
||||
|
||||
lint:
|
||||
$(UV) run ruff check src tests main.py
|
||||
$(UV) run ruff check src tests scripts main.py
|
||||
|
||||
check:
|
||||
$(UV) sync --frozen
|
||||
$(UV) run ruff check src tests main.py
|
||||
$(UV) run python -m compileall src
|
||||
$(UV) run ruff check src tests scripts main.py
|
||||
$(UV) run python -m compileall src scripts
|
||||
$(UV) run pytest -q
|
||||
|
||||
serve:
|
||||
|
||||
@@ -0,0 +1,523 @@
|
||||
#!/usr/bin/env python3
|
||||
"""End-to-end acceptance run against a real FastAPI process on port 18000.
|
||||
|
||||
Starts uvicorn with ``CONTROL_TOKEN``, then exercises robustness and the full
|
||||
training lifecycle:
|
||||
|
||||
- /healthz, /readyz, /meta, /bootstrap (8 sensors, hourly series, context)
|
||||
- /predict for every sensor x horizon x scenario (128 calls)
|
||||
- /simulate for every scenario with all sensors
|
||||
- auth, unknown job, unknown model, malformed CSV, one-job-at-a-time
|
||||
- POST /train model=small on a real CSV subset; poll to success; hot-swap
|
||||
- POST /train model=forecast on the full CSV; poll to success; hot-swap
|
||||
- /predict still works after each swap
|
||||
|
||||
Exit code is 0 only when every check passes. Figures printed to stdout are the
|
||||
measured numbers; ``--report`` writes the same result as JSON.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
|
||||
import httpx
|
||||
import pandas as pd
|
||||
|
||||
from pulsecast import config
|
||||
|
||||
HORIZON_SIZES = {"15m": 1, "1h": 4, "48h": 48, "7d": 168}
|
||||
SCENARIOS = tuple(config.SCENARIOS)
|
||||
SENSORS = tuple(sorted(config.USABLE_SENSORS))
|
||||
|
||||
|
||||
class CheckFailed(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
class Runner:
|
||||
def __init__(self) -> None:
|
||||
self.checks = 0
|
||||
self.results: dict = {}
|
||||
|
||||
def check(self, condition: bool, message: str) -> None:
|
||||
self.checks += 1
|
||||
if not condition:
|
||||
raise CheckFailed(message)
|
||||
print(f" [ok] {message}")
|
||||
|
||||
|
||||
def _utc_now() -> str:
|
||||
return datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
|
||||
|
||||
def _parse_ts(value: str) -> pd.Timestamp:
|
||||
return pd.Timestamp(value)
|
||||
|
||||
|
||||
def _seconds(start: str | None, end: str | None) -> float | None:
|
||||
if not start or not end:
|
||||
return None
|
||||
return round((_parse_ts(end) - _parse_ts(start)).total_seconds(), 3)
|
||||
|
||||
|
||||
def _wait_http(client: httpx.Client, url: str, timeout: float = 30.0) -> None:
|
||||
deadline = time.monotonic() + timeout
|
||||
last: Exception | None = None
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
response = client.get(url, timeout=2.0)
|
||||
if response.status_code == 200:
|
||||
return
|
||||
except Exception as exc: # noqa: BLE001 - startup race
|
||||
last = exc
|
||||
time.sleep(0.1)
|
||||
raise CheckFailed(f"server did not answer {url} within {timeout}s: {last}")
|
||||
|
||||
|
||||
def _wait_job(
|
||||
client: httpx.Client,
|
||||
job_id: str,
|
||||
headers: dict[str, str],
|
||||
*,
|
||||
timeout: float = 180.0,
|
||||
state: str | None = None,
|
||||
) -> dict:
|
||||
deadline = time.monotonic() + timeout
|
||||
last_status = None
|
||||
while time.monotonic() < deadline:
|
||||
response = client.get(f"/train/{job_id}", headers=headers)
|
||||
if response.status_code != 200:
|
||||
raise CheckFailed(f"GET /train/{job_id} -> {response.status_code}: {response.text}")
|
||||
job = response.json()
|
||||
last_status = job["status"]
|
||||
if state is not None and last_status == state:
|
||||
return job
|
||||
if state is None and last_status in {"succeeded", "failed"}:
|
||||
return job
|
||||
time.sleep(0.05)
|
||||
raise CheckFailed(f"job {job_id} did not reach {state or 'terminal'} (last={last_status})")
|
||||
|
||||
|
||||
def _job_headers(token: str) -> dict[str, str]:
|
||||
return {"Authorization": f"Bearer {token}"}
|
||||
|
||||
|
||||
def _submit(
|
||||
client: httpx.Client, headers: dict[str, str], path: Path, model: str
|
||||
) -> httpx.Response:
|
||||
return client.post(
|
||||
"/train",
|
||||
headers=headers,
|
||||
files={"file": (path.name, path.read_bytes(), "text/csv")},
|
||||
data={"model": model},
|
||||
timeout=120.0,
|
||||
)
|
||||
|
||||
|
||||
def _assert_points(points: list[dict], expected: int, label: str) -> None:
|
||||
if len(points) != expected:
|
||||
raise CheckFailed(f"{label}: expected {expected} points, got {len(points)}")
|
||||
previous = None
|
||||
for point in points:
|
||||
if set(point) != {"t", "v", "lo", "hi"}:
|
||||
raise CheckFailed(f"{label}: unexpected point keys {sorted(point)}")
|
||||
if not point["t"].endswith("Z"):
|
||||
raise CheckFailed(f"{label}: timestamp is not UTC RFC 3339: {point['t']}")
|
||||
if point["v"] < 0 or point["lo"] > point["v"] or point["v"] > point["hi"]:
|
||||
raise CheckFailed(f"{label}: invalid band {point}")
|
||||
stamp = _parse_ts(point["t"])
|
||||
if previous is not None and stamp <= previous:
|
||||
raise CheckFailed(f"{label}: timestamps are not increasing at {point['t']}")
|
||||
previous = stamp
|
||||
|
||||
|
||||
def _versions(payload: dict) -> dict:
|
||||
return {
|
||||
key: payload.get(key)
|
||||
for key in ("data_version", "model_version", "context_version")
|
||||
}
|
||||
|
||||
|
||||
def _build_subset(raw: Path, destination: Path, days: int = 60) -> tuple[Path, dict]:
|
||||
frame = pd.read_csv(raw)
|
||||
frame = frame[frame["type"].astype(str) == "densityData"].copy()
|
||||
frame["time"] = pd.to_datetime(frame["time"], utc=True)
|
||||
end = frame["time"].max()
|
||||
cut = end - pd.Timedelta(days=days)
|
||||
frame = frame[frame["time"] >= cut][["time", "name", "value"]]
|
||||
frame["time"] = frame["time"].dt.strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
frame.to_csv(destination, index=False)
|
||||
return destination, {
|
||||
"rows": len(frame),
|
||||
"sensors": int(frame["name"].nunique()),
|
||||
"from": frame["time"].min(),
|
||||
"to": frame["time"].max(),
|
||||
"days": days,
|
||||
}
|
||||
|
||||
|
||||
def _port_free(port: int) -> bool:
|
||||
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
|
||||
sock.settimeout(0.5)
|
||||
return sock.connect_ex(("127.0.0.1", port)) != 0
|
||||
|
||||
|
||||
def run(args: argparse.Namespace, runner: Runner) -> dict:
|
||||
raw = Path(args.csv)
|
||||
if not raw.exists():
|
||||
raise CheckFailed(f"CSV not found: {raw}")
|
||||
if not _port_free(args.port):
|
||||
raise CheckFailed(f"port {args.port} is already in use")
|
||||
|
||||
workdir = Path(tempfile.mkdtemp(prefix="pulsecast-e2e-"))
|
||||
models_dir = workdir / "models"
|
||||
uploads_dir = workdir / "uploads"
|
||||
subset_path, subset_info = _build_subset(raw, workdir / "small-subset.csv", args.days)
|
||||
garbage_path = workdir / "garbage.csv"
|
||||
garbage_path.write_text("not,a,pulsecast,file\n1,2,3,4\n")
|
||||
|
||||
env = os.environ.copy()
|
||||
env.update(
|
||||
CONTROL_TOKEN=args.token,
|
||||
PULSECAST_MODELS=str(models_dir),
|
||||
PULSECAST_UPLOADS=str(uploads_dir),
|
||||
)
|
||||
log_path = workdir / "server.log"
|
||||
log_handle = log_path.open("wb")
|
||||
command = [
|
||||
sys.executable,
|
||||
"-m",
|
||||
"uvicorn",
|
||||
"pulsecast.server:app",
|
||||
"--host",
|
||||
"127.0.0.1",
|
||||
"--port",
|
||||
str(args.port),
|
||||
"--log-level",
|
||||
"warning",
|
||||
]
|
||||
process = subprocess.Popen(command, env=env, stdout=log_handle, stderr=subprocess.STDOUT)
|
||||
report: dict = {
|
||||
"started_at": _utc_now(),
|
||||
"base_url": f"http://127.0.0.1:{args.port}",
|
||||
"subset": subset_info,
|
||||
"raw_csv_bytes": raw.stat().st_size,
|
||||
}
|
||||
|
||||
try:
|
||||
with httpx.Client(base_url=f"http://127.0.0.1:{args.port}", timeout=120.0) as client:
|
||||
_wait_http(client, "/healthz")
|
||||
headers = _job_headers(args.token)
|
||||
|
||||
print("Boot and snapshot")
|
||||
health = client.get("/healthz").json()
|
||||
runner.check(health == {"ok": True}, "/healthz returns {ok:true}")
|
||||
ready = client.get("/readyz")
|
||||
runner.check(ready.status_code == 200 and ready.json()["model"] is True, "/readyz 200 model=true")
|
||||
meta_before = client.get("/meta").json()
|
||||
runner.check(meta_before["coverage"]["sensors"] == 8, "/meta covers 8 sensors")
|
||||
runner.check(
|
||||
meta_before["metrics"]["mae"] < meta_before["metrics"]["baseline_mae"],
|
||||
"baked model beats seasonal naive on the holdout",
|
||||
)
|
||||
bootstrap = client.get("/bootstrap").json()
|
||||
runner.check(len(bootstrap["sensors"]) == 8, "/bootstrap returns 8 sensors")
|
||||
runner.check(
|
||||
set(bootstrap["series"]) == set(SENSORS),
|
||||
"/bootstrap series covers all 8 sensors",
|
||||
)
|
||||
series_points = sum(len(points) for points in bootstrap["series"].values())
|
||||
runner.check(series_points > 20_000, f"/bootstrap carries {series_points} hourly points")
|
||||
context = bootstrap["context"]
|
||||
runner.check(
|
||||
len(context["weather"]["forecast"]) == 168,
|
||||
"/bootstrap context has 168 forecast weather hours",
|
||||
)
|
||||
runner.check(bool(context["holidays"]), "/bootstrap context has holidays")
|
||||
runner.check(
|
||||
_versions(bootstrap) == _versions(meta_before),
|
||||
"/bootstrap versions match /meta",
|
||||
)
|
||||
report["versions_before"] = _versions(meta_before)
|
||||
report["bootstrap"] = {"sensors": 8, "series_points": series_points}
|
||||
|
||||
print("Robustness: /predict every sensor x horizon x scenario")
|
||||
predict_calls = 0
|
||||
for sensor in SENSORS:
|
||||
for horizon, count in HORIZON_SIZES.items():
|
||||
for scenario in SCENARIOS:
|
||||
response = client.post(
|
||||
"/predict",
|
||||
json={"sensor": sensor, "horizon": horizon, "scenario": scenario},
|
||||
)
|
||||
if response.status_code != 200:
|
||||
raise CheckFailed(
|
||||
f"/predict {sensor}/{horizon}/{scenario} -> "
|
||||
f"{response.status_code}: {response.text}"
|
||||
)
|
||||
payload = response.json()
|
||||
_assert_points(
|
||||
payload["points"],
|
||||
count,
|
||||
f"/predict {sensor}/{horizon}/{scenario}",
|
||||
)
|
||||
predict_calls += 1
|
||||
runner.check(predict_calls == 128, "128 /predict combinations returned valid points")
|
||||
|
||||
print("Robustness: /simulate every scenario, all sensors")
|
||||
simulate_calls = 0
|
||||
for scenario in SCENARIOS:
|
||||
response = client.post(
|
||||
"/simulate", json={"scenario": scenario, "sensors": "all"}
|
||||
)
|
||||
if response.status_code != 200:
|
||||
raise CheckFailed(f"/simulate {scenario} -> {response.status_code}: {response.text}")
|
||||
payload = response.json()
|
||||
if set(payload["forecasts"]) != set(SENSORS):
|
||||
raise CheckFailed(f"/simulate {scenario}: sensor set mismatch")
|
||||
for sensor, by_horizon in payload["forecasts"].items():
|
||||
if set(by_horizon) != set(HORIZON_SIZES):
|
||||
raise CheckFailed(f"/simulate {scenario}/{sensor}: horizon set mismatch")
|
||||
for horizon, count in HORIZON_SIZES.items():
|
||||
_assert_points(
|
||||
by_horizon[horizon],
|
||||
count,
|
||||
f"/simulate {scenario}/{sensor}/{horizon}",
|
||||
)
|
||||
simulate_calls += 1
|
||||
runner.check(simulate_calls == 4, "4 /simulate scenarios x 8 sensors x 4 horizons")
|
||||
|
||||
cold = client.post(
|
||||
"/predict",
|
||||
json={"sensor": SENSORS[0], "horizon": "48h", "weather": {"temp_c": -8.0, "precip_mm": 6.0}},
|
||||
).json()["points"]
|
||||
warm = client.post(
|
||||
"/predict",
|
||||
json={"sensor": SENSORS[0], "horizon": "48h", "weather": {"temp_c": 30.0, "precip_mm": 0.0}},
|
||||
).json()["points"]
|
||||
runner.check(cold != warm, "weather override changes the forecast")
|
||||
|
||||
print("Error handling")
|
||||
no_token = client.post("/predict", json={"sensor": "nope"})
|
||||
runner.check(no_token.status_code == 404, "/predict unknown sensor -> 404")
|
||||
unauthorized = _submit(client, {}, subset_path, "small")
|
||||
runner.check(
|
||||
unauthorized.status_code == 401
|
||||
and unauthorized.json()["error"]["code"] == "unauthorized",
|
||||
"/train without token -> 401 unauthorized",
|
||||
)
|
||||
bad_token = _submit(client, _job_headers("wrong-token"), subset_path, "small")
|
||||
runner.check(bad_token.status_code == 401, "/train with wrong token -> 401")
|
||||
unknown_job = client.get("/train/nope", headers=headers)
|
||||
runner.check(
|
||||
unknown_job.status_code == 404
|
||||
and unknown_job.json()["error"]["code"] == "not_found",
|
||||
"/train/nope -> 404 not_found",
|
||||
)
|
||||
bad_model = _submit(client, headers, subset_path, "giant")
|
||||
runner.check(
|
||||
bad_model.status_code == 400
|
||||
and bad_model.json()["error"]["code"] == "bad_request",
|
||||
"/train invalid model -> 400 bad_request",
|
||||
)
|
||||
missing_path = client.post(
|
||||
"/train", headers=headers, json={"path": "/tmp/no-such-pulsecast.csv", "model": "small"}
|
||||
)
|
||||
runner.check(missing_path.status_code == 400, "/train missing JSON path -> 400")
|
||||
|
||||
print("Small model: real CSV subset through the API")
|
||||
started = time.monotonic()
|
||||
submitted = _submit(client, headers, subset_path, "small")
|
||||
if submitted.status_code != 202:
|
||||
raise CheckFailed(f"/train small -> {submitted.status_code}: {submitted.text}")
|
||||
small_submitted = submitted.json()
|
||||
runner.check(
|
||||
small_submitted["status"] == "queued" and small_submitted["model"] == "small",
|
||||
f"/train small queued as job {small_submitted['job_id']}",
|
||||
)
|
||||
small_job = _wait_job(client, small_submitted["job_id"], headers)
|
||||
wall = round(time.monotonic() - started, 3)
|
||||
if small_job["status"] != "succeeded":
|
||||
raise CheckFailed(f"small job failed: {small_job}")
|
||||
small_metrics = small_job["metrics"]
|
||||
for key in ("accuracy", "precision", "recall", "f1", "roc_auc", "n_train", "n_test", "positive_rate"):
|
||||
if key not in small_metrics:
|
||||
raise CheckFailed(f"small metrics missing {key}: {small_metrics}")
|
||||
train_seconds = _seconds(small_job["started_at"], small_job["finished_at"])
|
||||
runner.check(
|
||||
small_job["model_version"].startswith("logit-"),
|
||||
f"small model_version {small_job['model_version']}",
|
||||
)
|
||||
runner.check(
|
||||
wall < 5.0,
|
||||
f"small job took {wall}s wall time including upload (contract < 5 s; "
|
||||
f"server timestamps have 1 s resolution, train_seconds={train_seconds})",
|
||||
)
|
||||
runner.check(small_metrics["accuracy"] > 0.6, f"small accuracy {small_metrics['accuracy']}")
|
||||
runner.check(
|
||||
(models_dir / "small_latest.joblib").exists(),
|
||||
"small model persisted to PULSECAST_MODELS",
|
||||
)
|
||||
meta_after_small = client.get("/meta").json()
|
||||
runner.check(
|
||||
meta_after_small["small_model"]["model_version"] == small_job["model_version"],
|
||||
"small model hot-swapped into /meta",
|
||||
)
|
||||
runner.check(
|
||||
meta_after_small["small_model"]["metrics"]["accuracy"] == small_metrics["accuracy"],
|
||||
"hot-swapped small metrics match the job",
|
||||
)
|
||||
predict_after_small = client.post(
|
||||
"/predict", json={"sensor": SENSORS[0], "horizon": "7d", "scenario": "base"}
|
||||
)
|
||||
runner.check(
|
||||
predict_after_small.status_code == 200
|
||||
and len(predict_after_small.json()["points"]) == 168,
|
||||
"/predict still works after the small swap",
|
||||
)
|
||||
report["jobs"] = {
|
||||
"small": {
|
||||
"job": small_job,
|
||||
"wall_seconds": wall,
|
||||
"train_seconds": train_seconds,
|
||||
"input": subset_info,
|
||||
}
|
||||
}
|
||||
|
||||
print("Forecast model: full CSV, hot-swap, 409 while busy")
|
||||
forecast_started = time.monotonic()
|
||||
forecast_response = _submit(client, headers, raw, "forecast")
|
||||
if forecast_response.status_code != 202:
|
||||
raise CheckFailed(
|
||||
f"/train forecast -> {forecast_response.status_code}: {forecast_response.text}"
|
||||
)
|
||||
forecast_submitted = forecast_response.json()
|
||||
runner.check(
|
||||
forecast_submitted["model"] == "forecast",
|
||||
f"/train forecast queued as job {forecast_submitted['job_id']}",
|
||||
)
|
||||
_wait_job(client, forecast_submitted["job_id"], headers, state="running", timeout=30)
|
||||
busy = _submit(client, headers, subset_path, "small")
|
||||
runner.check(
|
||||
busy.status_code == 409 and busy.json()["error"]["code"] == "conflict",
|
||||
"second /train while forecast is running -> 409 conflict",
|
||||
)
|
||||
forecast_job = _wait_job(client, forecast_submitted["job_id"], headers, timeout=300)
|
||||
forecast_wall = round(time.monotonic() - forecast_started, 3)
|
||||
if forecast_job["status"] != "succeeded":
|
||||
raise CheckFailed(f"forecast job failed: {forecast_job}")
|
||||
forecast_metrics = forecast_job["metrics"]
|
||||
for key in ("mae", "wape", "r2", "baseline_mae", "n_eval"):
|
||||
if key not in forecast_metrics:
|
||||
raise CheckFailed(f"forecast metrics missing {key}: {forecast_metrics}")
|
||||
runner.check(
|
||||
forecast_job["model_version"].startswith("histgb-"),
|
||||
f"forecast model_version {forecast_job['model_version']}",
|
||||
)
|
||||
runner.check(
|
||||
forecast_metrics["mae"] < forecast_metrics["baseline_mae"],
|
||||
f"retrained forecast MAE {forecast_metrics['mae']} < baseline {forecast_metrics['baseline_mae']}",
|
||||
)
|
||||
runner.check(
|
||||
(models_dir / "forecast_latest.joblib").exists(),
|
||||
"forecast model persisted to PULSECAST_MODELS",
|
||||
)
|
||||
meta_after_forecast = client.get("/meta").json()
|
||||
runner.check(
|
||||
meta_after_forecast["model_version"] == forecast_job["model_version"]
|
||||
and meta_after_forecast["model_source"] == "retrained",
|
||||
"forecast model hot-swapped into /meta as retrained",
|
||||
)
|
||||
predict_after_forecast = client.post(
|
||||
"/predict", json={"sensor": SENSORS[-1], "horizon": "48h", "scenario": "rainy-saturday"}
|
||||
)
|
||||
runner.check(
|
||||
predict_after_forecast.status_code == 200
|
||||
and predict_after_forecast.json()["model_version"] == forecast_job["model_version"],
|
||||
"/predict uses the retrained forecast model",
|
||||
)
|
||||
runner.check(client.get("/readyz").status_code == 200, "/readyz stays 200 after hot-swap")
|
||||
report["jobs"]["forecast"] = {
|
||||
"job": forecast_job,
|
||||
"wall_seconds": forecast_wall,
|
||||
"input": {"rows": "full challenge CSV", "bytes": raw.stat().st_size},
|
||||
}
|
||||
|
||||
print("Malformed CSV fails the job (not the process)")
|
||||
bad_csv = _submit(client, headers, garbage_path, "small")
|
||||
runner.check(bad_csv.status_code == 202, "malformed CSV is accepted as a job")
|
||||
bad_job = _wait_job(client, bad_csv.json()["job_id"], headers)
|
||||
runner.check(
|
||||
bad_job["status"] == "failed" and bool(bad_job["error"]),
|
||||
f"malformed CSV job failed with {bad_job['error']!r}",
|
||||
)
|
||||
runner.check(
|
||||
client.get("/healthz").status_code == 200,
|
||||
"/healthz still 200 after a failed job",
|
||||
)
|
||||
|
||||
report["versions_after_small"] = _versions(meta_after_small)
|
||||
report["versions_after_forecast"] = _versions(meta_after_forecast)
|
||||
report["checks"] = runner.checks
|
||||
report["ok"] = True
|
||||
finally:
|
||||
process.terminate()
|
||||
try:
|
||||
process.wait(timeout=10)
|
||||
except subprocess.TimeoutExpired:
|
||||
process.kill()
|
||||
process.wait(timeout=5)
|
||||
log_handle.close()
|
||||
if process.returncode not in (0, -15):
|
||||
print(f"server exited with {process.returncode}; log: {log_path}", file=sys.stderr)
|
||||
print(log_path.read_text(errors="replace")[-4000:], file=sys.stderr)
|
||||
report["server_log_tail"] = log_path.read_text(errors="replace")[-1000:]
|
||||
if args.keep:
|
||||
print(f"kept workdir {workdir}")
|
||||
else:
|
||||
shutil.rmtree(workdir, ignore_errors=True)
|
||||
|
||||
report["finished_at"] = _utc_now()
|
||||
return report
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
parser.add_argument("--port", type=int, default=18000)
|
||||
parser.add_argument("--token", default=os.environ.get("CONTROL_TOKEN") or "e2e-local-token")
|
||||
parser.add_argument("--csv", default=str(config.raw_csv_path()))
|
||||
parser.add_argument("--days", type=int, default=60, help="days in the small-model CSV subset")
|
||||
parser.add_argument("--report", type=Path, help="write the JSON result here")
|
||||
parser.add_argument("--keep", action="store_true", help="keep the temporary workdir")
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
runner = Runner()
|
||||
print(f"PulseCast end-to-end check on port {args.port}")
|
||||
started = time.monotonic()
|
||||
try:
|
||||
report = run(args, runner)
|
||||
except CheckFailed as exc:
|
||||
print(f"\nFAIL after {runner.checks} checks: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
elapsed = round(time.monotonic() - started, 1)
|
||||
report["elapsed_seconds"] = elapsed
|
||||
print(f"\nPASS: {runner.checks} checks in {elapsed}s")
|
||||
if args.report:
|
||||
args.report.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.report.write_text(json.dumps(report, indent=2, ensure_ascii=False) + "\n")
|
||||
print(f"report: {args.report}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,262 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Render dependency-free SVG figures from ``artifacts/metrics.json``.
|
||||
|
||||
The paper figures are generated from the measured holdout metrics, never
|
||||
hand-written. Run ``make figures`` after ``make train``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
from xml.sax.saxutils import escape
|
||||
|
||||
from pulsecast import config
|
||||
|
||||
MODEL_COLOR = "#2563eb"
|
||||
BASELINE_COLOR = "#94a3b8"
|
||||
PRODUCTION_COLOR = "#0f766e"
|
||||
TEXT_COLOR = "#0f172a"
|
||||
GRID_COLOR = "#e2e8f0"
|
||||
|
||||
|
||||
def _svg_document(
|
||||
title: str,
|
||||
subtitle: str,
|
||||
body: list[str],
|
||||
width: int,
|
||||
height: int,
|
||||
footer: str = "Source: artifacts/metrics.json · PulseCast 28-day time-based holdout",
|
||||
) -> str:
|
||||
return "\n".join(
|
||||
[
|
||||
f'<svg xmlns="http://www.w3.org/2000/svg" width="{width}" height="{height}" '
|
||||
f'viewBox="0 0 {width} {height}" role="img" aria-label="{escape(title)}">',
|
||||
'<rect width="100%" height="100%" fill="#ffffff"/>',
|
||||
'<g font-family="-apple-system, Segoe UI, Helvetica, Arial, sans-serif">',
|
||||
f'<text x="24" y="34" font-size="20" font-weight="600" fill="{TEXT_COLOR}">'
|
||||
f"{escape(title)}</text>",
|
||||
f'<text x="24" y="56" font-size="13" fill="#475569">{escape(subtitle)}</text>',
|
||||
*body,
|
||||
f'<text x="24" y="{height - 12}" font-size="11" fill="#64748b">'
|
||||
f"{escape(footer)}</text>",
|
||||
"</g>",
|
||||
"</svg>",
|
||||
"",
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def _bar_chart(
|
||||
title: str,
|
||||
subtitle: str,
|
||||
rows: list[dict],
|
||||
*,
|
||||
value_key: str = "mae",
|
||||
secondary_key: str | None = None,
|
||||
secondary_label: str | None = None,
|
||||
primary_label: str = "MAE",
|
||||
production: str | None = None,
|
||||
) -> str:
|
||||
width = 960
|
||||
left = 230
|
||||
right = 96
|
||||
top = 86
|
||||
row_height = 38
|
||||
height = top + row_height * len(rows) + 56
|
||||
plot_width = width - left - right
|
||||
values = [row[value_key] for row in rows]
|
||||
if secondary_key:
|
||||
values += [row[secondary_key] for row in rows]
|
||||
maximum = max(values) * 1.08 if values else 1.0
|
||||
|
||||
body: list[str] = []
|
||||
for tick in range(5):
|
||||
x = left + plot_width * tick / 4
|
||||
value = maximum * tick / 4
|
||||
body.append(
|
||||
f'<line x1="{x:.1f}" y1="{top - 18}" x2="{x:.1f}" y2="{top + row_height * len(rows)}" '
|
||||
f'stroke="{GRID_COLOR}" stroke-width="1"/>'
|
||||
)
|
||||
body.append(
|
||||
f'<text x="{x:.1f}" y="{top - 24}" font-size="11" fill="#64748b" '
|
||||
f'text-anchor="middle">{value:.1f}</text>'
|
||||
)
|
||||
|
||||
legend_y = top - 34
|
||||
if secondary_key:
|
||||
body.append(
|
||||
f'<rect x="{left}" y="{legend_y}" width="10" height="10" fill="{BASELINE_COLOR}"/>'
|
||||
f'<text x="{left + 16}" y="{legend_y + 9}" font-size="12" fill="#475569">'
|
||||
f"{escape(secondary_label or 'baseline')}</text>"
|
||||
)
|
||||
body.append(
|
||||
f'<rect x="{left + 140}" y="{legend_y}" width="10" height="10" fill="{MODEL_COLOR}"/>'
|
||||
f'<text x="{left + 156}" y="{legend_y + 9}" font-size="12" fill="#475569">'
|
||||
f"{escape(primary_label)}</text>"
|
||||
)
|
||||
|
||||
for index, row in enumerate(rows):
|
||||
y = top + index * row_height
|
||||
label = str(row.get("name") or row.get("sensor"))
|
||||
body.append(
|
||||
f'<text x="{left - 12}" y="{y + 20}" font-size="12" fill="{TEXT_COLOR}" '
|
||||
f'text-anchor="end">{escape(label)}</text>'
|
||||
)
|
||||
if secondary_key:
|
||||
baseline_width = plot_width * row[secondary_key] / maximum
|
||||
body.append(
|
||||
f'<rect x="{left}" y="{y + 3}" width="{baseline_width:.1f}" height="12" '
|
||||
f'fill="{BASELINE_COLOR}"/>'
|
||||
)
|
||||
body.append(
|
||||
f'<text x="{left + baseline_width + 6:.1f}" y="{y + 13}" font-size="11" '
|
||||
f'fill="#64748b">{row[secondary_key]:.2f}</text>'
|
||||
)
|
||||
model_width = plot_width * row[value_key] / maximum
|
||||
body.append(
|
||||
f'<rect x="{left}" y="{y + 18}" width="{model_width:.1f}" height="12" '
|
||||
f'fill="{MODEL_COLOR}"/>'
|
||||
)
|
||||
body.append(
|
||||
f'<text x="{left + model_width + 6:.1f}" y="{y + 28}" font-size="11" '
|
||||
f'fill="{TEXT_COLOR}">{row[value_key]:.2f}</text>'
|
||||
)
|
||||
else:
|
||||
color = PRODUCTION_COLOR if production and row.get("name") == production else MODEL_COLOR
|
||||
bar_width = plot_width * row[value_key] / maximum
|
||||
body.append(
|
||||
f'<rect x="{left}" y="{y + 6}" width="{bar_width:.1f}" height="22" fill="{color}"/>'
|
||||
)
|
||||
body.append(
|
||||
f'<text x="{left + bar_width + 6:.1f}" y="{y + 21}" font-size="12" '
|
||||
f'fill="{TEXT_COLOR}">{row[value_key]:.2f}</text>'
|
||||
)
|
||||
|
||||
return _svg_document(title, subtitle, body, width, height)
|
||||
|
||||
|
||||
def _confusion_svg(small: dict) -> str:
|
||||
matrix = small["confusion_matrix"]
|
||||
metrics = small["metrics"]
|
||||
width, height = 760, 520
|
||||
left, top, cell = 200, 130, 190
|
||||
values = {
|
||||
(0, 0): ("TN", matrix["tn"]),
|
||||
(0, 1): ("FP", matrix["fp"]),
|
||||
(1, 0): ("FN", matrix["fn"]),
|
||||
(1, 1): ("TP", matrix["tp"]),
|
||||
}
|
||||
maximum = max(value for _, value in values.values())
|
||||
body: list[str] = []
|
||||
body.append(
|
||||
f'<text x="{left + cell / 2}" y="{top - 16}" font-size="13" fill="#475569" '
|
||||
f'text-anchor="middle">Predicted not busy</text>'
|
||||
)
|
||||
body.append(
|
||||
f'<text x="{left + cell + cell / 2}" y="{top - 16}" font-size="13" fill="#475569" '
|
||||
f'text-anchor="middle">Predicted busy</text>'
|
||||
)
|
||||
row_labels = ("Actually\nnot busy", "Actually\nbusy")
|
||||
for row, label in enumerate(row_labels):
|
||||
y = top + row * cell
|
||||
body.append(
|
||||
f'<text x="{left - 18}" y="{y + cell / 2}" font-size="13" fill="#475569" '
|
||||
f'text-anchor="end">{escape(label).replace(chr(10), " ")}</text>'
|
||||
)
|
||||
for (row, column), (code, value) in values.items():
|
||||
x = left + column * cell
|
||||
y = top + row * cell
|
||||
intensity = 0.12 + 0.88 * (value / maximum)
|
||||
red = round(219 + (29 - 219) * intensity)
|
||||
green = round(234 + (78 - 234) * intensity)
|
||||
blue = round(254 + (216 - 254) * intensity)
|
||||
fill = f"#{red:02x}{green:02x}{blue:02x}"
|
||||
text_color = "#ffffff" if intensity > 0.55 else TEXT_COLOR
|
||||
row_total = matrix["tn"] + matrix["fp"] if row == 0 else matrix["fn"] + matrix["tp"]
|
||||
share = 100.0 * value / row_total if row_total else 0.0
|
||||
body.append(
|
||||
f'<rect x="{x}" y="{y}" width="{cell}" height="{cell}" fill="{fill}" '
|
||||
f'stroke="#ffffff" stroke-width="2"/>'
|
||||
)
|
||||
body.append(
|
||||
f'<text x="{x + cell / 2}" y="{y + cell / 2 - 8}" font-size="30" '
|
||||
f'font-weight="600" fill="{text_color}" text-anchor="middle">{value:,}</text>'
|
||||
)
|
||||
body.append(
|
||||
f'<text x="{x + cell / 2}" y="{y + cell / 2 + 20}" font-size="13" '
|
||||
f'fill="{text_color}" text-anchor="middle">{code} · {share:.1f}% of row</text>'
|
||||
)
|
||||
subtitle = (
|
||||
f"n={metrics['n_test']:,} · accuracy {metrics['accuracy']:.3f} · "
|
||||
f"precision {metrics['precision']:.3f} · recall {metrics['recall']:.3f} · "
|
||||
f"F1 {metrics['f1']:.3f} · ROC-AUC {metrics['roc_auc']:.3f}"
|
||||
)
|
||||
return _svg_document(
|
||||
"Small busy-hour classifier: holdout confusion matrix",
|
||||
subtitle,
|
||||
body,
|
||||
width,
|
||||
height,
|
||||
footer="Source: artifacts/small_metrics.json · chronological 80/20 split",
|
||||
)
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument(
|
||||
"--artifacts",
|
||||
type=Path,
|
||||
default=config.artifacts_dir(),
|
||||
help="artifact directory containing metrics.json (default: %(default)s)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--out",
|
||||
type=Path,
|
||||
default=Path(__file__).resolve().parents[1] / "docs" / "figures",
|
||||
help="output directory for SVG files (default: %(default)s)",
|
||||
)
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
metrics = json.loads((args.artifacts / "metrics.json").read_text())
|
||||
holdout = f"Holdout {metrics['test_from']} .. {metrics['test_to']}, n={metrics['metrics']['n_eval']}"
|
||||
args.out.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
ablation = _bar_chart(
|
||||
"Ablation: holdout MAE by feature set",
|
||||
holdout,
|
||||
[
|
||||
{
|
||||
"name": row["name"].replace("_", " "),
|
||||
"mae": row["mae"],
|
||||
}
|
||||
for row in metrics["ablation"]
|
||||
],
|
||||
production="week weather",
|
||||
)
|
||||
ablation_path = args.out / "ablation_mae.svg"
|
||||
ablation_path.write_text(ablation)
|
||||
|
||||
per_sensor = _bar_chart(
|
||||
"Per-sensor holdout MAE: production model vs seasonal naive",
|
||||
holdout,
|
||||
metrics["per_sensor"],
|
||||
secondary_key="baseline_mae",
|
||||
secondary_label="seasonal naive",
|
||||
primary_label="weather-7d",
|
||||
)
|
||||
per_sensor_path = args.out / "per_sensor_mae.svg"
|
||||
per_sensor_path.write_text(per_sensor)
|
||||
|
||||
confusion_path = args.out / "small_model_confusion.svg"
|
||||
confusion_path.write_text(_confusion_svg(metrics["small"]))
|
||||
|
||||
print(f"wrote {ablation_path}")
|
||||
print(f"wrote {per_sensor_path}")
|
||||
print(f"wrote {confusion_path}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,153 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Train and evaluate the small "busy hour" classifier from a CSV.
|
||||
|
||||
This is the repeatable artifact behind ``make small`` and the runtime
|
||||
``model=small`` training job. It prints every number reported in
|
||||
``docs/small-model.md``; ``--save`` bakes the bundle and metrics into
|
||||
``artifacts/`` exactly like ``make train`` does.
|
||||
|
||||
Examples:
|
||||
uv run python scripts/train_small.py
|
||||
uv run python scripts/train_small.py --csv new-data.csv --all
|
||||
uv run python scripts/train_small.py --save --json
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import joblib
|
||||
import pandas as pd
|
||||
|
||||
from pulsecast import config
|
||||
from pulsecast.small import load_hourly_csv, train_small_from_csv
|
||||
|
||||
|
||||
def _parse_args(argv: list[str] | None = None) -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
parser.add_argument(
|
||||
"csv_positional",
|
||||
nargs="?",
|
||||
metavar="CSV",
|
||||
help=f"CSV to train from (default: {config.raw_csv_path()})",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--csv",
|
||||
dest="csv_option",
|
||||
help="CSV to train from; overrides the positional argument",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--start",
|
||||
default=config.MULTI_SENSOR_START,
|
||||
help=(
|
||||
"drop rows before this UTC timestamp (default: %(default)s, the "
|
||||
"8-sensor challenge window); ignored with --all"
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--all",
|
||||
action="store_true",
|
||||
help="use every row in the CSV, including density-01's older history",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--save",
|
||||
action="store_true",
|
||||
help="write artifacts/small_model.joblib and artifacts/small_metrics.json",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--json",
|
||||
action="store_true",
|
||||
help="print the full metrics payload as JSON after the summary",
|
||||
)
|
||||
return parser.parse_args(argv)
|
||||
|
||||
|
||||
def _confusion_lines(matrix: dict) -> list[str]:
|
||||
tn, fp, fn, tp = matrix["tn"], matrix["fp"], matrix["fn"], matrix["tp"]
|
||||
total = tn + fp + fn + tp
|
||||
specificity = tn / (tn + fp) if tn + fp else 0.0
|
||||
negative_predictive = tn / (tn + fn) if tn + fn else 0.0
|
||||
return [
|
||||
"Confusion matrix (rows = truth, columns = prediction)",
|
||||
f" not busy: {tn:5d} {fp:5d}",
|
||||
f" busy : {fn:5d} {tp:5d}",
|
||||
f" n={total} specificity={specificity:.3f} NPV={negative_predictive:.3f}",
|
||||
]
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
args = _parse_args(argv)
|
||||
path = Path(args.csv_option or args.csv_positional or config.raw_csv_path())
|
||||
if not path.exists():
|
||||
print(f"CSV not found: {path}", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
start = None if args.all else pd.Timestamp(args.start)
|
||||
started = time.perf_counter()
|
||||
try:
|
||||
bundle, payload = train_small_from_csv(path, start=start)
|
||||
except (ValueError, FileNotFoundError) as exc:
|
||||
print(f"training failed: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
elapsed = time.perf_counter() - started
|
||||
|
||||
hourly = load_hourly_csv(path)
|
||||
if start is not None:
|
||||
hourly = hourly[hourly["t"] >= start]
|
||||
metrics = payload["metrics"]
|
||||
|
||||
print(f"CSV {path}")
|
||||
print(f"Rows {len(hourly):,} hourly rows, {metrics['sensors']} sensor(s)")
|
||||
print(f"Window {payload['train_from']} .. {payload['test_to']}")
|
||||
print(f"Split (time) train {payload['train_from']} .. {payload['train_to']}"
|
||||
f" ({metrics['n_train']:,} rows)")
|
||||
print(f" test {payload['test_from']} .. {payload['test_to']}"
|
||||
f" ({metrics['n_test']:,} rows)")
|
||||
print("Target busy = hourly density > sensor p75 (training window only)")
|
||||
print(f"Features {', '.join(payload['features'])}")
|
||||
print("Model standardized logistic regression (class_weight=balanced)")
|
||||
print(f"Model version {payload['model_version']}")
|
||||
print()
|
||||
print("Metrics")
|
||||
print(f" accuracy {metrics['accuracy']:.3f}")
|
||||
print(f" precision {metrics['precision']:.3f}")
|
||||
print(f" recall {metrics['recall']:.3f}")
|
||||
print(f" f1 {metrics['f1']:.3f}")
|
||||
print(f" roc_auc {metrics['roc_auc'] if metrics['roc_auc'] is not None else 'n/a'}")
|
||||
print(f" base rate {metrics['positive_rate']:.3f}")
|
||||
print()
|
||||
print("\n".join(_confusion_lines(payload["confusion_matrix"])))
|
||||
print()
|
||||
print("Per sensor (holdout)")
|
||||
print(f" {'sensor':<24} {'n':>6} {'pos':>6} {'acc':>6} {'prec':>6} {'rec':>6} {'f1':>6}")
|
||||
for row in payload.get("per_sensor", []):
|
||||
print(
|
||||
f" {row['sensor']:<24} {row['n']:>6} {row['positive_rate']:>6.3f}"
|
||||
f" {row['accuracy']:>6.3f} {row['precision']:>6.3f}"
|
||||
f" {row['recall']:>6.3f} {row['f1']:>6.3f}"
|
||||
)
|
||||
print()
|
||||
print(f"Fit time {elapsed:.2f} s (contract: < 5 s)")
|
||||
|
||||
if args.save:
|
||||
artifacts = config.artifacts_dir()
|
||||
artifacts.mkdir(parents=True, exist_ok=True)
|
||||
joblib.dump(bundle, artifacts / "small_model.joblib")
|
||||
(artifacts / "small_metrics.json").write_text(
|
||||
json.dumps(payload, indent=2, ensure_ascii=False) + "\n"
|
||||
)
|
||||
print(f"Saved {artifacts / 'small_model.joblib'}")
|
||||
print(f" {artifacts / 'small_metrics.json'}")
|
||||
|
||||
if args.json:
|
||||
print()
|
||||
print(json.dumps(payload, indent=2, ensure_ascii=False))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in new issue
Block a user