feat: repeatable small-model, figures, and training e2e scripts
- make small trains from a CSV and prints metrics + confusion matrix - make figures renders dependency-free SVGs from metrics.json - scripts/e2e_train.py boots a real server and checks snapshot, robustness, auth, one-job-at-a-time, and both training jobs with hot-swap - CI/ruff/compileall now include scripts/
This commit is contained in:
1 parent
000aaed7a5
commit
2bcf66c550
5 files changed
+952
-6
No files matched your search
@@ -32,10 +32,10 @@ jobs:
|
|||||||
run: uv sync --frozen
|
run: uv sync --frozen
|
||||||
|
|
||||||
- name: Static analysis (ruff)
|
- name: Static analysis (ruff)
|
||||||
run: uv run ruff check src tests main.py
|
run: uv run ruff check src tests scripts main.py
|
||||||
|
|
||||||
- name: Byte-compile
|
- name: Byte-compile
|
||||||
run: uv run python -m compileall src
|
run: uv run python -m compileall src scripts
|
||||||
|
|
||||||
- name: Tests
|
- name: Tests
|
||||||
run: uv run pytest -q
|
run: uv run pytest -q
|
||||||
@@ -1,11 +1,13 @@
|
|||||||
UV ?= uv
|
UV ?= uv
|
||||||
|
|
||||||
.PHONY: help data train test lint check serve docker clean
|
.PHONY: help data train small figures test lint check serve docker clean
|
||||||
|
|
||||||
help:
|
help:
|
||||||
@printf '%s\n' \
|
@printf '%s\n' \
|
||||||
'make data parse/clean the raw CSV, join weather + holidays, export JSON' \
|
'make data parse/clean the raw CSV, join weather + holidays, export JSON' \
|
||||||
'make train fit the 28-day holdout model, export metrics/forecasts/model' \
|
'make train fit the 28-day holdout model, export metrics/forecasts/model' \
|
||||||
|
'make small train + print the busy-hour classifier metrics from the raw CSV' \
|
||||||
|
'make figures regenerate docs/figures/*.svg from artifacts/metrics.json' \
|
||||||
'make test run pytest' \
|
'make test run pytest' \
|
||||||
'make lint run ruff static checks' \
|
'make lint run ruff static checks' \
|
||||||
'make check CI gate: sync --frozen, ruff, compileall, pytest' \
|
'make check CI gate: sync --frozen, ruff, compileall, pytest' \
|
||||||
@@ -19,16 +21,22 @@ data:
|
|||||||
train:
|
train:
|
||||||
$(UV) run python -m pulsecast.train
|
$(UV) run python -m pulsecast.train
|
||||||
|
|
||||||
|
small:
|
||||||
|
$(UV) run python scripts/train_small.py
|
||||||
|
|
||||||
|
figures:
|
||||||
|
$(UV) run python scripts/plot_metrics.py
|
||||||
|
|
||||||
test:
|
test:
|
||||||
$(UV) run pytest
|
$(UV) run pytest
|
||||||
|
|
||||||
lint:
|
lint:
|
||||||
$(UV) run ruff check src tests main.py
|
$(UV) run ruff check src tests scripts main.py
|
||||||
|
|
||||||
check:
|
check:
|
||||||
$(UV) sync --frozen
|
$(UV) sync --frozen
|
||||||
$(UV) run ruff check src tests main.py
|
$(UV) run ruff check src tests scripts main.py
|
||||||
$(UV) run python -m compileall src
|
$(UV) run python -m compileall src scripts
|
||||||
$(UV) run pytest -q
|
$(UV) run pytest -q
|
||||||
|
|
||||||
serve:
|
serve:
|
||||||
|
|||||||
@@ -0,0 +1,523 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""End-to-end acceptance run against a real FastAPI process on port 18000.
|
||||||
|
|
||||||
|
Starts uvicorn with ``CONTROL_TOKEN``, then exercises robustness and the full
|
||||||
|
training lifecycle:
|
||||||
|
|
||||||
|
- /healthz, /readyz, /meta, /bootstrap (8 sensors, hourly series, context)
|
||||||
|
- /predict for every sensor x horizon x scenario (128 calls)
|
||||||
|
- /simulate for every scenario with all sensors
|
||||||
|
- auth, unknown job, unknown model, malformed CSV, one-job-at-a-time
|
||||||
|
- POST /train model=small on a real CSV subset; poll to success; hot-swap
|
||||||
|
- POST /train model=forecast on the full CSV; poll to success; hot-swap
|
||||||
|
- /predict still works after each swap
|
||||||
|
|
||||||
|
Exit code is 0 only when every check passes. Figures printed to stdout are the
|
||||||
|
measured numbers; ``--report`` writes the same result as JSON.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import shutil
|
||||||
|
import socket
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import tempfile
|
||||||
|
import time
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import httpx
|
||||||
|
import pandas as pd
|
||||||
|
|
||||||
|
from pulsecast import config
|
||||||
|
|
||||||
|
HORIZON_SIZES = {"15m": 1, "1h": 4, "48h": 48, "7d": 168}
|
||||||
|
SCENARIOS = tuple(config.SCENARIOS)
|
||||||
|
SENSORS = tuple(sorted(config.USABLE_SENSORS))
|
||||||
|
|
||||||
|
|
||||||
|
class CheckFailed(RuntimeError):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
class Runner:
|
||||||
|
def __init__(self) -> None:
|
||||||
|
self.checks = 0
|
||||||
|
self.results: dict = {}
|
||||||
|
|
||||||
|
def check(self, condition: bool, message: str) -> None:
|
||||||
|
self.checks += 1
|
||||||
|
if not condition:
|
||||||
|
raise CheckFailed(message)
|
||||||
|
print(f" [ok] {message}")
|
||||||
|
|
||||||
|
|
||||||
|
def _utc_now() -> str:
|
||||||
|
return datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_ts(value: str) -> pd.Timestamp:
|
||||||
|
return pd.Timestamp(value)
|
||||||
|
|
||||||
|
|
||||||
|
def _seconds(start: str | None, end: str | None) -> float | None:
|
||||||
|
if not start or not end:
|
||||||
|
return None
|
||||||
|
return round((_parse_ts(end) - _parse_ts(start)).total_seconds(), 3)
|
||||||
|
|
||||||
|
|
||||||
|
def _wait_http(client: httpx.Client, url: str, timeout: float = 30.0) -> None:
|
||||||
|
deadline = time.monotonic() + timeout
|
||||||
|
last: Exception | None = None
|
||||||
|
while time.monotonic() < deadline:
|
||||||
|
try:
|
||||||
|
response = client.get(url, timeout=2.0)
|
||||||
|
if response.status_code == 200:
|
||||||
|
return
|
||||||
|
except Exception as exc: # noqa: BLE001 - startup race
|
||||||
|
last = exc
|
||||||
|
time.sleep(0.1)
|
||||||
|
raise CheckFailed(f"server did not answer {url} within {timeout}s: {last}")
|
||||||
|
|
||||||
|
|
||||||
|
def _wait_job(
|
||||||
|
client: httpx.Client,
|
||||||
|
job_id: str,
|
||||||
|
headers: dict[str, str],
|
||||||
|
*,
|
||||||
|
timeout: float = 180.0,
|
||||||
|
state: str | None = None,
|
||||||
|
) -> dict:
|
||||||
|
deadline = time.monotonic() + timeout
|
||||||
|
last_status = None
|
||||||
|
while time.monotonic() < deadline:
|
||||||
|
response = client.get(f"/train/{job_id}", headers=headers)
|
||||||
|
if response.status_code != 200:
|
||||||
|
raise CheckFailed(f"GET /train/{job_id} -> {response.status_code}: {response.text}")
|
||||||
|
job = response.json()
|
||||||
|
last_status = job["status"]
|
||||||
|
if state is not None and last_status == state:
|
||||||
|
return job
|
||||||
|
if state is None and last_status in {"succeeded", "failed"}:
|
||||||
|
return job
|
||||||
|
time.sleep(0.05)
|
||||||
|
raise CheckFailed(f"job {job_id} did not reach {state or 'terminal'} (last={last_status})")
|
||||||
|
|
||||||
|
|
||||||
|
def _job_headers(token: str) -> dict[str, str]:
|
||||||
|
return {"Authorization": f"Bearer {token}"}
|
||||||
|
|
||||||
|
|
||||||
|
def _submit(
|
||||||
|
client: httpx.Client, headers: dict[str, str], path: Path, model: str
|
||||||
|
) -> httpx.Response:
|
||||||
|
return client.post(
|
||||||
|
"/train",
|
||||||
|
headers=headers,
|
||||||
|
files={"file": (path.name, path.read_bytes(), "text/csv")},
|
||||||
|
data={"model": model},
|
||||||
|
timeout=120.0,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _assert_points(points: list[dict], expected: int, label: str) -> None:
|
||||||
|
if len(points) != expected:
|
||||||
|
raise CheckFailed(f"{label}: expected {expected} points, got {len(points)}")
|
||||||
|
previous = None
|
||||||
|
for point in points:
|
||||||
|
if set(point) != {"t", "v", "lo", "hi"}:
|
||||||
|
raise CheckFailed(f"{label}: unexpected point keys {sorted(point)}")
|
||||||
|
if not point["t"].endswith("Z"):
|
||||||
|
raise CheckFailed(f"{label}: timestamp is not UTC RFC 3339: {point['t']}")
|
||||||
|
if point["v"] < 0 or point["lo"] > point["v"] or point["v"] > point["hi"]:
|
||||||
|
raise CheckFailed(f"{label}: invalid band {point}")
|
||||||
|
stamp = _parse_ts(point["t"])
|
||||||
|
if previous is not None and stamp <= previous:
|
||||||
|
raise CheckFailed(f"{label}: timestamps are not increasing at {point['t']}")
|
||||||
|
previous = stamp
|
||||||
|
|
||||||
|
|
||||||
|
def _versions(payload: dict) -> dict:
|
||||||
|
return {
|
||||||
|
key: payload.get(key)
|
||||||
|
for key in ("data_version", "model_version", "context_version")
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _build_subset(raw: Path, destination: Path, days: int = 60) -> tuple[Path, dict]:
|
||||||
|
frame = pd.read_csv(raw)
|
||||||
|
frame = frame[frame["type"].astype(str) == "densityData"].copy()
|
||||||
|
frame["time"] = pd.to_datetime(frame["time"], utc=True)
|
||||||
|
end = frame["time"].max()
|
||||||
|
cut = end - pd.Timedelta(days=days)
|
||||||
|
frame = frame[frame["time"] >= cut][["time", "name", "value"]]
|
||||||
|
frame["time"] = frame["time"].dt.strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||||
|
frame.to_csv(destination, index=False)
|
||||||
|
return destination, {
|
||||||
|
"rows": len(frame),
|
||||||
|
"sensors": int(frame["name"].nunique()),
|
||||||
|
"from": frame["time"].min(),
|
||||||
|
"to": frame["time"].max(),
|
||||||
|
"days": days,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _port_free(port: int) -> bool:
|
||||||
|
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
|
||||||
|
sock.settimeout(0.5)
|
||||||
|
return sock.connect_ex(("127.0.0.1", port)) != 0
|
||||||
|
|
||||||
|
|
||||||
|
def run(args: argparse.Namespace, runner: Runner) -> dict:
|
||||||
|
raw = Path(args.csv)
|
||||||
|
if not raw.exists():
|
||||||
|
raise CheckFailed(f"CSV not found: {raw}")
|
||||||
|
if not _port_free(args.port):
|
||||||
|
raise CheckFailed(f"port {args.port} is already in use")
|
||||||
|
|
||||||
|
workdir = Path(tempfile.mkdtemp(prefix="pulsecast-e2e-"))
|
||||||
|
models_dir = workdir / "models"
|
||||||
|
uploads_dir = workdir / "uploads"
|
||||||
|
subset_path, subset_info = _build_subset(raw, workdir / "small-subset.csv", args.days)
|
||||||
|
garbage_path = workdir / "garbage.csv"
|
||||||
|
garbage_path.write_text("not,a,pulsecast,file\n1,2,3,4\n")
|
||||||
|
|
||||||
|
env = os.environ.copy()
|
||||||
|
env.update(
|
||||||
|
CONTROL_TOKEN=args.token,
|
||||||
|
PULSECAST_MODELS=str(models_dir),
|
||||||
|
PULSECAST_UPLOADS=str(uploads_dir),
|
||||||
|
)
|
||||||
|
log_path = workdir / "server.log"
|
||||||
|
log_handle = log_path.open("wb")
|
||||||
|
command = [
|
||||||
|
sys.executable,
|
||||||
|
"-m",
|
||||||
|
"uvicorn",
|
||||||
|
"pulsecast.server:app",
|
||||||
|
"--host",
|
||||||
|
"127.0.0.1",
|
||||||
|
"--port",
|
||||||
|
str(args.port),
|
||||||
|
"--log-level",
|
||||||
|
"warning",
|
||||||
|
]
|
||||||
|
process = subprocess.Popen(command, env=env, stdout=log_handle, stderr=subprocess.STDOUT)
|
||||||
|
report: dict = {
|
||||||
|
"started_at": _utc_now(),
|
||||||
|
"base_url": f"http://127.0.0.1:{args.port}",
|
||||||
|
"subset": subset_info,
|
||||||
|
"raw_csv_bytes": raw.stat().st_size,
|
||||||
|
}
|
||||||
|
|
||||||
|
try:
|
||||||
|
with httpx.Client(base_url=f"http://127.0.0.1:{args.port}", timeout=120.0) as client:
|
||||||
|
_wait_http(client, "/healthz")
|
||||||
|
headers = _job_headers(args.token)
|
||||||
|
|
||||||
|
print("Boot and snapshot")
|
||||||
|
health = client.get("/healthz").json()
|
||||||
|
runner.check(health == {"ok": True}, "/healthz returns {ok:true}")
|
||||||
|
ready = client.get("/readyz")
|
||||||
|
runner.check(ready.status_code == 200 and ready.json()["model"] is True, "/readyz 200 model=true")
|
||||||
|
meta_before = client.get("/meta").json()
|
||||||
|
runner.check(meta_before["coverage"]["sensors"] == 8, "/meta covers 8 sensors")
|
||||||
|
runner.check(
|
||||||
|
meta_before["metrics"]["mae"] < meta_before["metrics"]["baseline_mae"],
|
||||||
|
"baked model beats seasonal naive on the holdout",
|
||||||
|
)
|
||||||
|
bootstrap = client.get("/bootstrap").json()
|
||||||
|
runner.check(len(bootstrap["sensors"]) == 8, "/bootstrap returns 8 sensors")
|
||||||
|
runner.check(
|
||||||
|
set(bootstrap["series"]) == set(SENSORS),
|
||||||
|
"/bootstrap series covers all 8 sensors",
|
||||||
|
)
|
||||||
|
series_points = sum(len(points) for points in bootstrap["series"].values())
|
||||||
|
runner.check(series_points > 20_000, f"/bootstrap carries {series_points} hourly points")
|
||||||
|
context = bootstrap["context"]
|
||||||
|
runner.check(
|
||||||
|
len(context["weather"]["forecast"]) == 168,
|
||||||
|
"/bootstrap context has 168 forecast weather hours",
|
||||||
|
)
|
||||||
|
runner.check(bool(context["holidays"]), "/bootstrap context has holidays")
|
||||||
|
runner.check(
|
||||||
|
_versions(bootstrap) == _versions(meta_before),
|
||||||
|
"/bootstrap versions match /meta",
|
||||||
|
)
|
||||||
|
report["versions_before"] = _versions(meta_before)
|
||||||
|
report["bootstrap"] = {"sensors": 8, "series_points": series_points}
|
||||||
|
|
||||||
|
print("Robustness: /predict every sensor x horizon x scenario")
|
||||||
|
predict_calls = 0
|
||||||
|
for sensor in SENSORS:
|
||||||
|
for horizon, count in HORIZON_SIZES.items():
|
||||||
|
for scenario in SCENARIOS:
|
||||||
|
response = client.post(
|
||||||
|
"/predict",
|
||||||
|
json={"sensor": sensor, "horizon": horizon, "scenario": scenario},
|
||||||
|
)
|
||||||
|
if response.status_code != 200:
|
||||||
|
raise CheckFailed(
|
||||||
|
f"/predict {sensor}/{horizon}/{scenario} -> "
|
||||||
|
f"{response.status_code}: {response.text}"
|
||||||
|
)
|
||||||
|
payload = response.json()
|
||||||
|
_assert_points(
|
||||||
|
payload["points"],
|
||||||
|
count,
|
||||||
|
f"/predict {sensor}/{horizon}/{scenario}",
|
||||||
|
)
|
||||||
|
predict_calls += 1
|
||||||
|
runner.check(predict_calls == 128, "128 /predict combinations returned valid points")
|
||||||
|
|
||||||
|
print("Robustness: /simulate every scenario, all sensors")
|
||||||
|
simulate_calls = 0
|
||||||
|
for scenario in SCENARIOS:
|
||||||
|
response = client.post(
|
||||||
|
"/simulate", json={"scenario": scenario, "sensors": "all"}
|
||||||
|
)
|
||||||
|
if response.status_code != 200:
|
||||||
|
raise CheckFailed(f"/simulate {scenario} -> {response.status_code}: {response.text}")
|
||||||
|
payload = response.json()
|
||||||
|
if set(payload["forecasts"]) != set(SENSORS):
|
||||||
|
raise CheckFailed(f"/simulate {scenario}: sensor set mismatch")
|
||||||
|
for sensor, by_horizon in payload["forecasts"].items():
|
||||||
|
if set(by_horizon) != set(HORIZON_SIZES):
|
||||||
|
raise CheckFailed(f"/simulate {scenario}/{sensor}: horizon set mismatch")
|
||||||
|
for horizon, count in HORIZON_SIZES.items():
|
||||||
|
_assert_points(
|
||||||
|
by_horizon[horizon],
|
||||||
|
count,
|
||||||
|
f"/simulate {scenario}/{sensor}/{horizon}",
|
||||||
|
)
|
||||||
|
simulate_calls += 1
|
||||||
|
runner.check(simulate_calls == 4, "4 /simulate scenarios x 8 sensors x 4 horizons")
|
||||||
|
|
||||||
|
cold = client.post(
|
||||||
|
"/predict",
|
||||||
|
json={"sensor": SENSORS[0], "horizon": "48h", "weather": {"temp_c": -8.0, "precip_mm": 6.0}},
|
||||||
|
).json()["points"]
|
||||||
|
warm = client.post(
|
||||||
|
"/predict",
|
||||||
|
json={"sensor": SENSORS[0], "horizon": "48h", "weather": {"temp_c": 30.0, "precip_mm": 0.0}},
|
||||||
|
).json()["points"]
|
||||||
|
runner.check(cold != warm, "weather override changes the forecast")
|
||||||
|
|
||||||
|
print("Error handling")
|
||||||
|
no_token = client.post("/predict", json={"sensor": "nope"})
|
||||||
|
runner.check(no_token.status_code == 404, "/predict unknown sensor -> 404")
|
||||||
|
unauthorized = _submit(client, {}, subset_path, "small")
|
||||||
|
runner.check(
|
||||||
|
unauthorized.status_code == 401
|
||||||
|
and unauthorized.json()["error"]["code"] == "unauthorized",
|
||||||
|
"/train without token -> 401 unauthorized",
|
||||||
|
)
|
||||||
|
bad_token = _submit(client, _job_headers("wrong-token"), subset_path, "small")
|
||||||
|
runner.check(bad_token.status_code == 401, "/train with wrong token -> 401")
|
||||||
|
unknown_job = client.get("/train/nope", headers=headers)
|
||||||
|
runner.check(
|
||||||
|
unknown_job.status_code == 404
|
||||||
|
and unknown_job.json()["error"]["code"] == "not_found",
|
||||||
|
"/train/nope -> 404 not_found",
|
||||||
|
)
|
||||||
|
bad_model = _submit(client, headers, subset_path, "giant")
|
||||||
|
runner.check(
|
||||||
|
bad_model.status_code == 400
|
||||||
|
and bad_model.json()["error"]["code"] == "bad_request",
|
||||||
|
"/train invalid model -> 400 bad_request",
|
||||||
|
)
|
||||||
|
missing_path = client.post(
|
||||||
|
"/train", headers=headers, json={"path": "/tmp/no-such-pulsecast.csv", "model": "small"}
|
||||||
|
)
|
||||||
|
runner.check(missing_path.status_code == 400, "/train missing JSON path -> 400")
|
||||||
|
|
||||||
|
print("Small model: real CSV subset through the API")
|
||||||
|
started = time.monotonic()
|
||||||
|
submitted = _submit(client, headers, subset_path, "small")
|
||||||
|
if submitted.status_code != 202:
|
||||||
|
raise CheckFailed(f"/train small -> {submitted.status_code}: {submitted.text}")
|
||||||
|
small_submitted = submitted.json()
|
||||||
|
runner.check(
|
||||||
|
small_submitted["status"] == "queued" and small_submitted["model"] == "small",
|
||||||
|
f"/train small queued as job {small_submitted['job_id']}",
|
||||||
|
)
|
||||||
|
small_job = _wait_job(client, small_submitted["job_id"], headers)
|
||||||
|
wall = round(time.monotonic() - started, 3)
|
||||||
|
if small_job["status"] != "succeeded":
|
||||||
|
raise CheckFailed(f"small job failed: {small_job}")
|
||||||
|
small_metrics = small_job["metrics"]
|
||||||
|
for key in ("accuracy", "precision", "recall", "f1", "roc_auc", "n_train", "n_test", "positive_rate"):
|
||||||
|
if key not in small_metrics:
|
||||||
|
raise CheckFailed(f"small metrics missing {key}: {small_metrics}")
|
||||||
|
train_seconds = _seconds(small_job["started_at"], small_job["finished_at"])
|
||||||
|
runner.check(
|
||||||
|
small_job["model_version"].startswith("logit-"),
|
||||||
|
f"small model_version {small_job['model_version']}",
|
||||||
|
)
|
||||||
|
runner.check(
|
||||||
|
wall < 5.0,
|
||||||
|
f"small job took {wall}s wall time including upload (contract < 5 s; "
|
||||||
|
f"server timestamps have 1 s resolution, train_seconds={train_seconds})",
|
||||||
|
)
|
||||||
|
runner.check(small_metrics["accuracy"] > 0.6, f"small accuracy {small_metrics['accuracy']}")
|
||||||
|
runner.check(
|
||||||
|
(models_dir / "small_latest.joblib").exists(),
|
||||||
|
"small model persisted to PULSECAST_MODELS",
|
||||||
|
)
|
||||||
|
meta_after_small = client.get("/meta").json()
|
||||||
|
runner.check(
|
||||||
|
meta_after_small["small_model"]["model_version"] == small_job["model_version"],
|
||||||
|
"small model hot-swapped into /meta",
|
||||||
|
)
|
||||||
|
runner.check(
|
||||||
|
meta_after_small["small_model"]["metrics"]["accuracy"] == small_metrics["accuracy"],
|
||||||
|
"hot-swapped small metrics match the job",
|
||||||
|
)
|
||||||
|
predict_after_small = client.post(
|
||||||
|
"/predict", json={"sensor": SENSORS[0], "horizon": "7d", "scenario": "base"}
|
||||||
|
)
|
||||||
|
runner.check(
|
||||||
|
predict_after_small.status_code == 200
|
||||||
|
and len(predict_after_small.json()["points"]) == 168,
|
||||||
|
"/predict still works after the small swap",
|
||||||
|
)
|
||||||
|
report["jobs"] = {
|
||||||
|
"small": {
|
||||||
|
"job": small_job,
|
||||||
|
"wall_seconds": wall,
|
||||||
|
"train_seconds": train_seconds,
|
||||||
|
"input": subset_info,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
print("Forecast model: full CSV, hot-swap, 409 while busy")
|
||||||
|
forecast_started = time.monotonic()
|
||||||
|
forecast_response = _submit(client, headers, raw, "forecast")
|
||||||
|
if forecast_response.status_code != 202:
|
||||||
|
raise CheckFailed(
|
||||||
|
f"/train forecast -> {forecast_response.status_code}: {forecast_response.text}"
|
||||||
|
)
|
||||||
|
forecast_submitted = forecast_response.json()
|
||||||
|
runner.check(
|
||||||
|
forecast_submitted["model"] == "forecast",
|
||||||
|
f"/train forecast queued as job {forecast_submitted['job_id']}",
|
||||||
|
)
|
||||||
|
_wait_job(client, forecast_submitted["job_id"], headers, state="running", timeout=30)
|
||||||
|
busy = _submit(client, headers, subset_path, "small")
|
||||||
|
runner.check(
|
||||||
|
busy.status_code == 409 and busy.json()["error"]["code"] == "conflict",
|
||||||
|
"second /train while forecast is running -> 409 conflict",
|
||||||
|
)
|
||||||
|
forecast_job = _wait_job(client, forecast_submitted["job_id"], headers, timeout=300)
|
||||||
|
forecast_wall = round(time.monotonic() - forecast_started, 3)
|
||||||
|
if forecast_job["status"] != "succeeded":
|
||||||
|
raise CheckFailed(f"forecast job failed: {forecast_job}")
|
||||||
|
forecast_metrics = forecast_job["metrics"]
|
||||||
|
for key in ("mae", "wape", "r2", "baseline_mae", "n_eval"):
|
||||||
|
if key not in forecast_metrics:
|
||||||
|
raise CheckFailed(f"forecast metrics missing {key}: {forecast_metrics}")
|
||||||
|
runner.check(
|
||||||
|
forecast_job["model_version"].startswith("histgb-"),
|
||||||
|
f"forecast model_version {forecast_job['model_version']}",
|
||||||
|
)
|
||||||
|
runner.check(
|
||||||
|
forecast_metrics["mae"] < forecast_metrics["baseline_mae"],
|
||||||
|
f"retrained forecast MAE {forecast_metrics['mae']} < baseline {forecast_metrics['baseline_mae']}",
|
||||||
|
)
|
||||||
|
runner.check(
|
||||||
|
(models_dir / "forecast_latest.joblib").exists(),
|
||||||
|
"forecast model persisted to PULSECAST_MODELS",
|
||||||
|
)
|
||||||
|
meta_after_forecast = client.get("/meta").json()
|
||||||
|
runner.check(
|
||||||
|
meta_after_forecast["model_version"] == forecast_job["model_version"]
|
||||||
|
and meta_after_forecast["model_source"] == "retrained",
|
||||||
|
"forecast model hot-swapped into /meta as retrained",
|
||||||
|
)
|
||||||
|
predict_after_forecast = client.post(
|
||||||
|
"/predict", json={"sensor": SENSORS[-1], "horizon": "48h", "scenario": "rainy-saturday"}
|
||||||
|
)
|
||||||
|
runner.check(
|
||||||
|
predict_after_forecast.status_code == 200
|
||||||
|
and predict_after_forecast.json()["model_version"] == forecast_job["model_version"],
|
||||||
|
"/predict uses the retrained forecast model",
|
||||||
|
)
|
||||||
|
runner.check(client.get("/readyz").status_code == 200, "/readyz stays 200 after hot-swap")
|
||||||
|
report["jobs"]["forecast"] = {
|
||||||
|
"job": forecast_job,
|
||||||
|
"wall_seconds": forecast_wall,
|
||||||
|
"input": {"rows": "full challenge CSV", "bytes": raw.stat().st_size},
|
||||||
|
}
|
||||||
|
|
||||||
|
print("Malformed CSV fails the job (not the process)")
|
||||||
|
bad_csv = _submit(client, headers, garbage_path, "small")
|
||||||
|
runner.check(bad_csv.status_code == 202, "malformed CSV is accepted as a job")
|
||||||
|
bad_job = _wait_job(client, bad_csv.json()["job_id"], headers)
|
||||||
|
runner.check(
|
||||||
|
bad_job["status"] == "failed" and bool(bad_job["error"]),
|
||||||
|
f"malformed CSV job failed with {bad_job['error']!r}",
|
||||||
|
)
|
||||||
|
runner.check(
|
||||||
|
client.get("/healthz").status_code == 200,
|
||||||
|
"/healthz still 200 after a failed job",
|
||||||
|
)
|
||||||
|
|
||||||
|
report["versions_after_small"] = _versions(meta_after_small)
|
||||||
|
report["versions_after_forecast"] = _versions(meta_after_forecast)
|
||||||
|
report["checks"] = runner.checks
|
||||||
|
report["ok"] = True
|
||||||
|
finally:
|
||||||
|
process.terminate()
|
||||||
|
try:
|
||||||
|
process.wait(timeout=10)
|
||||||
|
except subprocess.TimeoutExpired:
|
||||||
|
process.kill()
|
||||||
|
process.wait(timeout=5)
|
||||||
|
log_handle.close()
|
||||||
|
if process.returncode not in (0, -15):
|
||||||
|
print(f"server exited with {process.returncode}; log: {log_path}", file=sys.stderr)
|
||||||
|
print(log_path.read_text(errors="replace")[-4000:], file=sys.stderr)
|
||||||
|
report["server_log_tail"] = log_path.read_text(errors="replace")[-1000:]
|
||||||
|
if args.keep:
|
||||||
|
print(f"kept workdir {workdir}")
|
||||||
|
else:
|
||||||
|
shutil.rmtree(workdir, ignore_errors=True)
|
||||||
|
|
||||||
|
report["finished_at"] = _utc_now()
|
||||||
|
return report
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv: list[str] | None = None) -> int:
|
||||||
|
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||||
|
parser.add_argument("--port", type=int, default=18000)
|
||||||
|
parser.add_argument("--token", default=os.environ.get("CONTROL_TOKEN") or "e2e-local-token")
|
||||||
|
parser.add_argument("--csv", default=str(config.raw_csv_path()))
|
||||||
|
parser.add_argument("--days", type=int, default=60, help="days in the small-model CSV subset")
|
||||||
|
parser.add_argument("--report", type=Path, help="write the JSON result here")
|
||||||
|
parser.add_argument("--keep", action="store_true", help="keep the temporary workdir")
|
||||||
|
args = parser.parse_args(argv)
|
||||||
|
|
||||||
|
runner = Runner()
|
||||||
|
print(f"PulseCast end-to-end check on port {args.port}")
|
||||||
|
started = time.monotonic()
|
||||||
|
try:
|
||||||
|
report = run(args, runner)
|
||||||
|
except CheckFailed as exc:
|
||||||
|
print(f"\nFAIL after {runner.checks} checks: {exc}", file=sys.stderr)
|
||||||
|
return 1
|
||||||
|
elapsed = round(time.monotonic() - started, 1)
|
||||||
|
report["elapsed_seconds"] = elapsed
|
||||||
|
print(f"\nPASS: {runner.checks} checks in {elapsed}s")
|
||||||
|
if args.report:
|
||||||
|
args.report.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
args.report.write_text(json.dumps(report, indent=2, ensure_ascii=False) + "\n")
|
||||||
|
print(f"report: {args.report}")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
@@ -0,0 +1,262 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Render dependency-free SVG figures from ``artifacts/metrics.json``.
|
||||||
|
|
||||||
|
The paper figures are generated from the measured holdout metrics, never
|
||||||
|
hand-written. Run ``make figures`` after ``make train``.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
from xml.sax.saxutils import escape
|
||||||
|
|
||||||
|
from pulsecast import config
|
||||||
|
|
||||||
|
MODEL_COLOR = "#2563eb"
|
||||||
|
BASELINE_COLOR = "#94a3b8"
|
||||||
|
PRODUCTION_COLOR = "#0f766e"
|
||||||
|
TEXT_COLOR = "#0f172a"
|
||||||
|
GRID_COLOR = "#e2e8f0"
|
||||||
|
|
||||||
|
|
||||||
|
def _svg_document(
|
||||||
|
title: str,
|
||||||
|
subtitle: str,
|
||||||
|
body: list[str],
|
||||||
|
width: int,
|
||||||
|
height: int,
|
||||||
|
footer: str = "Source: artifacts/metrics.json · PulseCast 28-day time-based holdout",
|
||||||
|
) -> str:
|
||||||
|
return "\n".join(
|
||||||
|
[
|
||||||
|
f'<svg xmlns="http://www.w3.org/2000/svg" width="{width}" height="{height}" '
|
||||||
|
f'viewBox="0 0 {width} {height}" role="img" aria-label="{escape(title)}">',
|
||||||
|
'<rect width="100%" height="100%" fill="#ffffff"/>',
|
||||||
|
'<g font-family="-apple-system, Segoe UI, Helvetica, Arial, sans-serif">',
|
||||||
|
f'<text x="24" y="34" font-size="20" font-weight="600" fill="{TEXT_COLOR}">'
|
||||||
|
f"{escape(title)}</text>",
|
||||||
|
f'<text x="24" y="56" font-size="13" fill="#475569">{escape(subtitle)}</text>',
|
||||||
|
*body,
|
||||||
|
f'<text x="24" y="{height - 12}" font-size="11" fill="#64748b">'
|
||||||
|
f"{escape(footer)}</text>",
|
||||||
|
"</g>",
|
||||||
|
"</svg>",
|
||||||
|
"",
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _bar_chart(
|
||||||
|
title: str,
|
||||||
|
subtitle: str,
|
||||||
|
rows: list[dict],
|
||||||
|
*,
|
||||||
|
value_key: str = "mae",
|
||||||
|
secondary_key: str | None = None,
|
||||||
|
secondary_label: str | None = None,
|
||||||
|
primary_label: str = "MAE",
|
||||||
|
production: str | None = None,
|
||||||
|
) -> str:
|
||||||
|
width = 960
|
||||||
|
left = 230
|
||||||
|
right = 96
|
||||||
|
top = 86
|
||||||
|
row_height = 38
|
||||||
|
height = top + row_height * len(rows) + 56
|
||||||
|
plot_width = width - left - right
|
||||||
|
values = [row[value_key] for row in rows]
|
||||||
|
if secondary_key:
|
||||||
|
values += [row[secondary_key] for row in rows]
|
||||||
|
maximum = max(values) * 1.08 if values else 1.0
|
||||||
|
|
||||||
|
body: list[str] = []
|
||||||
|
for tick in range(5):
|
||||||
|
x = left + plot_width * tick / 4
|
||||||
|
value = maximum * tick / 4
|
||||||
|
body.append(
|
||||||
|
f'<line x1="{x:.1f}" y1="{top - 18}" x2="{x:.1f}" y2="{top + row_height * len(rows)}" '
|
||||||
|
f'stroke="{GRID_COLOR}" stroke-width="1"/>'
|
||||||
|
)
|
||||||
|
body.append(
|
||||||
|
f'<text x="{x:.1f}" y="{top - 24}" font-size="11" fill="#64748b" '
|
||||||
|
f'text-anchor="middle">{value:.1f}</text>'
|
||||||
|
)
|
||||||
|
|
||||||
|
legend_y = top - 34
|
||||||
|
if secondary_key:
|
||||||
|
body.append(
|
||||||
|
f'<rect x="{left}" y="{legend_y}" width="10" height="10" fill="{BASELINE_COLOR}"/>'
|
||||||
|
f'<text x="{left + 16}" y="{legend_y + 9}" font-size="12" fill="#475569">'
|
||||||
|
f"{escape(secondary_label or 'baseline')}</text>"
|
||||||
|
)
|
||||||
|
body.append(
|
||||||
|
f'<rect x="{left + 140}" y="{legend_y}" width="10" height="10" fill="{MODEL_COLOR}"/>'
|
||||||
|
f'<text x="{left + 156}" y="{legend_y + 9}" font-size="12" fill="#475569">'
|
||||||
|
f"{escape(primary_label)}</text>"
|
||||||
|
)
|
||||||
|
|
||||||
|
for index, row in enumerate(rows):
|
||||||
|
y = top + index * row_height
|
||||||
|
label = str(row.get("name") or row.get("sensor"))
|
||||||
|
body.append(
|
||||||
|
f'<text x="{left - 12}" y="{y + 20}" font-size="12" fill="{TEXT_COLOR}" '
|
||||||
|
f'text-anchor="end">{escape(label)}</text>'
|
||||||
|
)
|
||||||
|
if secondary_key:
|
||||||
|
baseline_width = plot_width * row[secondary_key] / maximum
|
||||||
|
body.append(
|
||||||
|
f'<rect x="{left}" y="{y + 3}" width="{baseline_width:.1f}" height="12" '
|
||||||
|
f'fill="{BASELINE_COLOR}"/>'
|
||||||
|
)
|
||||||
|
body.append(
|
||||||
|
f'<text x="{left + baseline_width + 6:.1f}" y="{y + 13}" font-size="11" '
|
||||||
|
f'fill="#64748b">{row[secondary_key]:.2f}</text>'
|
||||||
|
)
|
||||||
|
model_width = plot_width * row[value_key] / maximum
|
||||||
|
body.append(
|
||||||
|
f'<rect x="{left}" y="{y + 18}" width="{model_width:.1f}" height="12" '
|
||||||
|
f'fill="{MODEL_COLOR}"/>'
|
||||||
|
)
|
||||||
|
body.append(
|
||||||
|
f'<text x="{left + model_width + 6:.1f}" y="{y + 28}" font-size="11" '
|
||||||
|
f'fill="{TEXT_COLOR}">{row[value_key]:.2f}</text>'
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
color = PRODUCTION_COLOR if production and row.get("name") == production else MODEL_COLOR
|
||||||
|
bar_width = plot_width * row[value_key] / maximum
|
||||||
|
body.append(
|
||||||
|
f'<rect x="{left}" y="{y + 6}" width="{bar_width:.1f}" height="22" fill="{color}"/>'
|
||||||
|
)
|
||||||
|
body.append(
|
||||||
|
f'<text x="{left + bar_width + 6:.1f}" y="{y + 21}" font-size="12" '
|
||||||
|
f'fill="{TEXT_COLOR}">{row[value_key]:.2f}</text>'
|
||||||
|
)
|
||||||
|
|
||||||
|
return _svg_document(title, subtitle, body, width, height)
|
||||||
|
|
||||||
|
|
||||||
|
def _confusion_svg(small: dict) -> str:
|
||||||
|
matrix = small["confusion_matrix"]
|
||||||
|
metrics = small["metrics"]
|
||||||
|
width, height = 760, 520
|
||||||
|
left, top, cell = 200, 130, 190
|
||||||
|
values = {
|
||||||
|
(0, 0): ("TN", matrix["tn"]),
|
||||||
|
(0, 1): ("FP", matrix["fp"]),
|
||||||
|
(1, 0): ("FN", matrix["fn"]),
|
||||||
|
(1, 1): ("TP", matrix["tp"]),
|
||||||
|
}
|
||||||
|
maximum = max(value for _, value in values.values())
|
||||||
|
body: list[str] = []
|
||||||
|
body.append(
|
||||||
|
f'<text x="{left + cell / 2}" y="{top - 16}" font-size="13" fill="#475569" '
|
||||||
|
f'text-anchor="middle">Predicted not busy</text>'
|
||||||
|
)
|
||||||
|
body.append(
|
||||||
|
f'<text x="{left + cell + cell / 2}" y="{top - 16}" font-size="13" fill="#475569" '
|
||||||
|
f'text-anchor="middle">Predicted busy</text>'
|
||||||
|
)
|
||||||
|
row_labels = ("Actually\nnot busy", "Actually\nbusy")
|
||||||
|
for row, label in enumerate(row_labels):
|
||||||
|
y = top + row * cell
|
||||||
|
body.append(
|
||||||
|
f'<text x="{left - 18}" y="{y + cell / 2}" font-size="13" fill="#475569" '
|
||||||
|
f'text-anchor="end">{escape(label).replace(chr(10), " ")}</text>'
|
||||||
|
)
|
||||||
|
for (row, column), (code, value) in values.items():
|
||||||
|
x = left + column * cell
|
||||||
|
y = top + row * cell
|
||||||
|
intensity = 0.12 + 0.88 * (value / maximum)
|
||||||
|
red = round(219 + (29 - 219) * intensity)
|
||||||
|
green = round(234 + (78 - 234) * intensity)
|
||||||
|
blue = round(254 + (216 - 254) * intensity)
|
||||||
|
fill = f"#{red:02x}{green:02x}{blue:02x}"
|
||||||
|
text_color = "#ffffff" if intensity > 0.55 else TEXT_COLOR
|
||||||
|
row_total = matrix["tn"] + matrix["fp"] if row == 0 else matrix["fn"] + matrix["tp"]
|
||||||
|
share = 100.0 * value / row_total if row_total else 0.0
|
||||||
|
body.append(
|
||||||
|
f'<rect x="{x}" y="{y}" width="{cell}" height="{cell}" fill="{fill}" '
|
||||||
|
f'stroke="#ffffff" stroke-width="2"/>'
|
||||||
|
)
|
||||||
|
body.append(
|
||||||
|
f'<text x="{x + cell / 2}" y="{y + cell / 2 - 8}" font-size="30" '
|
||||||
|
f'font-weight="600" fill="{text_color}" text-anchor="middle">{value:,}</text>'
|
||||||
|
)
|
||||||
|
body.append(
|
||||||
|
f'<text x="{x + cell / 2}" y="{y + cell / 2 + 20}" font-size="13" '
|
||||||
|
f'fill="{text_color}" text-anchor="middle">{code} · {share:.1f}% of row</text>'
|
||||||
|
)
|
||||||
|
subtitle = (
|
||||||
|
f"n={metrics['n_test']:,} · accuracy {metrics['accuracy']:.3f} · "
|
||||||
|
f"precision {metrics['precision']:.3f} · recall {metrics['recall']:.3f} · "
|
||||||
|
f"F1 {metrics['f1']:.3f} · ROC-AUC {metrics['roc_auc']:.3f}"
|
||||||
|
)
|
||||||
|
return _svg_document(
|
||||||
|
"Small busy-hour classifier: holdout confusion matrix",
|
||||||
|
subtitle,
|
||||||
|
body,
|
||||||
|
width,
|
||||||
|
height,
|
||||||
|
footer="Source: artifacts/small_metrics.json · chronological 80/20 split",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv: list[str] | None = None) -> int:
|
||||||
|
parser = argparse.ArgumentParser(description=__doc__)
|
||||||
|
parser.add_argument(
|
||||||
|
"--artifacts",
|
||||||
|
type=Path,
|
||||||
|
default=config.artifacts_dir(),
|
||||||
|
help="artifact directory containing metrics.json (default: %(default)s)",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--out",
|
||||||
|
type=Path,
|
||||||
|
default=Path(__file__).resolve().parents[1] / "docs" / "figures",
|
||||||
|
help="output directory for SVG files (default: %(default)s)",
|
||||||
|
)
|
||||||
|
args = parser.parse_args(argv)
|
||||||
|
|
||||||
|
metrics = json.loads((args.artifacts / "metrics.json").read_text())
|
||||||
|
holdout = f"Holdout {metrics['test_from']} .. {metrics['test_to']}, n={metrics['metrics']['n_eval']}"
|
||||||
|
args.out.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
|
ablation = _bar_chart(
|
||||||
|
"Ablation: holdout MAE by feature set",
|
||||||
|
holdout,
|
||||||
|
[
|
||||||
|
{
|
||||||
|
"name": row["name"].replace("_", " "),
|
||||||
|
"mae": row["mae"],
|
||||||
|
}
|
||||||
|
for row in metrics["ablation"]
|
||||||
|
],
|
||||||
|
production="week weather",
|
||||||
|
)
|
||||||
|
ablation_path = args.out / "ablation_mae.svg"
|
||||||
|
ablation_path.write_text(ablation)
|
||||||
|
|
||||||
|
per_sensor = _bar_chart(
|
||||||
|
"Per-sensor holdout MAE: production model vs seasonal naive",
|
||||||
|
holdout,
|
||||||
|
metrics["per_sensor"],
|
||||||
|
secondary_key="baseline_mae",
|
||||||
|
secondary_label="seasonal naive",
|
||||||
|
primary_label="weather-7d",
|
||||||
|
)
|
||||||
|
per_sensor_path = args.out / "per_sensor_mae.svg"
|
||||||
|
per_sensor_path.write_text(per_sensor)
|
||||||
|
|
||||||
|
confusion_path = args.out / "small_model_confusion.svg"
|
||||||
|
confusion_path.write_text(_confusion_svg(metrics["small"]))
|
||||||
|
|
||||||
|
print(f"wrote {ablation_path}")
|
||||||
|
print(f"wrote {per_sensor_path}")
|
||||||
|
print(f"wrote {confusion_path}")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
@@ -0,0 +1,153 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Train and evaluate the small "busy hour" classifier from a CSV.
|
||||||
|
|
||||||
|
This is the repeatable artifact behind ``make small`` and the runtime
|
||||||
|
``model=small`` training job. It prints every number reported in
|
||||||
|
``docs/small-model.md``; ``--save`` bakes the bundle and metrics into
|
||||||
|
``artifacts/`` exactly like ``make train`` does.
|
||||||
|
|
||||||
|
Examples:
|
||||||
|
uv run python scripts/train_small.py
|
||||||
|
uv run python scripts/train_small.py --csv new-data.csv --all
|
||||||
|
uv run python scripts/train_small.py --save --json
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import json
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import joblib
|
||||||
|
import pandas as pd
|
||||||
|
|
||||||
|
from pulsecast import config
|
||||||
|
from pulsecast.small import load_hourly_csv, train_small_from_csv
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_args(argv: list[str] | None = None) -> argparse.Namespace:
|
||||||
|
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||||
|
parser.add_argument(
|
||||||
|
"csv_positional",
|
||||||
|
nargs="?",
|
||||||
|
metavar="CSV",
|
||||||
|
help=f"CSV to train from (default: {config.raw_csv_path()})",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--csv",
|
||||||
|
dest="csv_option",
|
||||||
|
help="CSV to train from; overrides the positional argument",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--start",
|
||||||
|
default=config.MULTI_SENSOR_START,
|
||||||
|
help=(
|
||||||
|
"drop rows before this UTC timestamp (default: %(default)s, the "
|
||||||
|
"8-sensor challenge window); ignored with --all"
|
||||||
|
),
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--all",
|
||||||
|
action="store_true",
|
||||||
|
help="use every row in the CSV, including density-01's older history",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--save",
|
||||||
|
action="store_true",
|
||||||
|
help="write artifacts/small_model.joblib and artifacts/small_metrics.json",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--json",
|
||||||
|
action="store_true",
|
||||||
|
help="print the full metrics payload as JSON after the summary",
|
||||||
|
)
|
||||||
|
return parser.parse_args(argv)
|
||||||
|
|
||||||
|
|
||||||
|
def _confusion_lines(matrix: dict) -> list[str]:
|
||||||
|
tn, fp, fn, tp = matrix["tn"], matrix["fp"], matrix["fn"], matrix["tp"]
|
||||||
|
total = tn + fp + fn + tp
|
||||||
|
specificity = tn / (tn + fp) if tn + fp else 0.0
|
||||||
|
negative_predictive = tn / (tn + fn) if tn + fn else 0.0
|
||||||
|
return [
|
||||||
|
"Confusion matrix (rows = truth, columns = prediction)",
|
||||||
|
f" not busy: {tn:5d} {fp:5d}",
|
||||||
|
f" busy : {fn:5d} {tp:5d}",
|
||||||
|
f" n={total} specificity={specificity:.3f} NPV={negative_predictive:.3f}",
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv: list[str] | None = None) -> int:
|
||||||
|
args = _parse_args(argv)
|
||||||
|
path = Path(args.csv_option or args.csv_positional or config.raw_csv_path())
|
||||||
|
if not path.exists():
|
||||||
|
print(f"CSV not found: {path}", file=sys.stderr)
|
||||||
|
return 2
|
||||||
|
|
||||||
|
start = None if args.all else pd.Timestamp(args.start)
|
||||||
|
started = time.perf_counter()
|
||||||
|
try:
|
||||||
|
bundle, payload = train_small_from_csv(path, start=start)
|
||||||
|
except (ValueError, FileNotFoundError) as exc:
|
||||||
|
print(f"training failed: {exc}", file=sys.stderr)
|
||||||
|
return 1
|
||||||
|
elapsed = time.perf_counter() - started
|
||||||
|
|
||||||
|
hourly = load_hourly_csv(path)
|
||||||
|
if start is not None:
|
||||||
|
hourly = hourly[hourly["t"] >= start]
|
||||||
|
metrics = payload["metrics"]
|
||||||
|
|
||||||
|
print(f"CSV {path}")
|
||||||
|
print(f"Rows {len(hourly):,} hourly rows, {metrics['sensors']} sensor(s)")
|
||||||
|
print(f"Window {payload['train_from']} .. {payload['test_to']}")
|
||||||
|
print(f"Split (time) train {payload['train_from']} .. {payload['train_to']}"
|
||||||
|
f" ({metrics['n_train']:,} rows)")
|
||||||
|
print(f" test {payload['test_from']} .. {payload['test_to']}"
|
||||||
|
f" ({metrics['n_test']:,} rows)")
|
||||||
|
print("Target busy = hourly density > sensor p75 (training window only)")
|
||||||
|
print(f"Features {', '.join(payload['features'])}")
|
||||||
|
print("Model standardized logistic regression (class_weight=balanced)")
|
||||||
|
print(f"Model version {payload['model_version']}")
|
||||||
|
print()
|
||||||
|
print("Metrics")
|
||||||
|
print(f" accuracy {metrics['accuracy']:.3f}")
|
||||||
|
print(f" precision {metrics['precision']:.3f}")
|
||||||
|
print(f" recall {metrics['recall']:.3f}")
|
||||||
|
print(f" f1 {metrics['f1']:.3f}")
|
||||||
|
print(f" roc_auc {metrics['roc_auc'] if metrics['roc_auc'] is not None else 'n/a'}")
|
||||||
|
print(f" base rate {metrics['positive_rate']:.3f}")
|
||||||
|
print()
|
||||||
|
print("\n".join(_confusion_lines(payload["confusion_matrix"])))
|
||||||
|
print()
|
||||||
|
print("Per sensor (holdout)")
|
||||||
|
print(f" {'sensor':<24} {'n':>6} {'pos':>6} {'acc':>6} {'prec':>6} {'rec':>6} {'f1':>6}")
|
||||||
|
for row in payload.get("per_sensor", []):
|
||||||
|
print(
|
||||||
|
f" {row['sensor']:<24} {row['n']:>6} {row['positive_rate']:>6.3f}"
|
||||||
|
f" {row['accuracy']:>6.3f} {row['precision']:>6.3f}"
|
||||||
|
f" {row['recall']:>6.3f} {row['f1']:>6.3f}"
|
||||||
|
)
|
||||||
|
print()
|
||||||
|
print(f"Fit time {elapsed:.2f} s (contract: < 5 s)")
|
||||||
|
|
||||||
|
if args.save:
|
||||||
|
artifacts = config.artifacts_dir()
|
||||||
|
artifacts.mkdir(parents=True, exist_ok=True)
|
||||||
|
joblib.dump(bundle, artifacts / "small_model.joblib")
|
||||||
|
(artifacts / "small_metrics.json").write_text(
|
||||||
|
json.dumps(payload, indent=2, ensure_ascii=False) + "\n"
|
||||||
|
)
|
||||||
|
print(f"Saved {artifacts / 'small_model.joblib'}")
|
||||||
|
print(f" {artifacts / 'small_metrics.json'}")
|
||||||
|
|
||||||
|
if args.json:
|
||||||
|
print()
|
||||||
|
print(json.dumps(payload, indent=2, ensure_ascii=False))
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
Reference in new issue
Block a user