Files
ML-py/scripts/e2e_train.py
Noel Sabia 2bcf66c550 feat: repeatable small-model, figures, and training e2e scripts
- make small trains from a CSV and prints metrics + confusion matrix
- make figures renders dependency-free SVGs from metrics.json
- scripts/e2e_train.py boots a real server and checks snapshot, robustness,
  auth, one-job-at-a-time, and both training jobs with hot-swap
- CI/ruff/compileall now include scripts/
2026-11-14 06:35:19 +01:00

524 lines
22 KiB
Python

#!/usr/bin/env python3
"""End-to-end acceptance run against a real FastAPI process on port 18000.
Starts uvicorn with ``CONTROL_TOKEN``, then exercises robustness and the full
training lifecycle:
- /healthz, /readyz, /meta, /bootstrap (8 sensors, hourly series, context)
- /predict for every sensor x horizon x scenario (128 calls)
- /simulate for every scenario with all sensors
- auth, unknown job, unknown model, malformed CSV, one-job-at-a-time
- POST /train model=small on a real CSV subset; poll to success; hot-swap
- POST /train model=forecast on the full CSV; poll to success; hot-swap
- /predict still works after each swap
Exit code is 0 only when every check passes. Figures printed to stdout are the
measured numbers; ``--report`` writes the same result as JSON.
"""
from __future__ import annotations
import argparse
import json
import os
import shutil
import socket
import subprocess
import sys
import tempfile
import time
from datetime import UTC, datetime
from pathlib import Path
import httpx
import pandas as pd
from pulsecast import config
HORIZON_SIZES = {"15m": 1, "1h": 4, "48h": 48, "7d": 168}
SCENARIOS = tuple(config.SCENARIOS)
SENSORS = tuple(sorted(config.USABLE_SENSORS))
class CheckFailed(RuntimeError):
pass
class Runner:
def __init__(self) -> None:
self.checks = 0
self.results: dict = {}
def check(self, condition: bool, message: str) -> None:
self.checks += 1
if not condition:
raise CheckFailed(message)
print(f" [ok] {message}")
def _utc_now() -> str:
return datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ")
def _parse_ts(value: str) -> pd.Timestamp:
return pd.Timestamp(value)
def _seconds(start: str | None, end: str | None) -> float | None:
if not start or not end:
return None
return round((_parse_ts(end) - _parse_ts(start)).total_seconds(), 3)
def _wait_http(client: httpx.Client, url: str, timeout: float = 30.0) -> None:
deadline = time.monotonic() + timeout
last: Exception | None = None
while time.monotonic() < deadline:
try:
response = client.get(url, timeout=2.0)
if response.status_code == 200:
return
except Exception as exc: # noqa: BLE001 - startup race
last = exc
time.sleep(0.1)
raise CheckFailed(f"server did not answer {url} within {timeout}s: {last}")
def _wait_job(
client: httpx.Client,
job_id: str,
headers: dict[str, str],
*,
timeout: float = 180.0,
state: str | None = None,
) -> dict:
deadline = time.monotonic() + timeout
last_status = None
while time.monotonic() < deadline:
response = client.get(f"/train/{job_id}", headers=headers)
if response.status_code != 200:
raise CheckFailed(f"GET /train/{job_id} -> {response.status_code}: {response.text}")
job = response.json()
last_status = job["status"]
if state is not None and last_status == state:
return job
if state is None and last_status in {"succeeded", "failed"}:
return job
time.sleep(0.05)
raise CheckFailed(f"job {job_id} did not reach {state or 'terminal'} (last={last_status})")
def _job_headers(token: str) -> dict[str, str]:
return {"Authorization": f"Bearer {token}"}
def _submit(
client: httpx.Client, headers: dict[str, str], path: Path, model: str
) -> httpx.Response:
return client.post(
"/train",
headers=headers,
files={"file": (path.name, path.read_bytes(), "text/csv")},
data={"model": model},
timeout=120.0,
)
def _assert_points(points: list[dict], expected: int, label: str) -> None:
if len(points) != expected:
raise CheckFailed(f"{label}: expected {expected} points, got {len(points)}")
previous = None
for point in points:
if set(point) != {"t", "v", "lo", "hi"}:
raise CheckFailed(f"{label}: unexpected point keys {sorted(point)}")
if not point["t"].endswith("Z"):
raise CheckFailed(f"{label}: timestamp is not UTC RFC 3339: {point['t']}")
if point["v"] < 0 or point["lo"] > point["v"] or point["v"] > point["hi"]:
raise CheckFailed(f"{label}: invalid band {point}")
stamp = _parse_ts(point["t"])
if previous is not None and stamp <= previous:
raise CheckFailed(f"{label}: timestamps are not increasing at {point['t']}")
previous = stamp
def _versions(payload: dict) -> dict:
return {
key: payload.get(key)
for key in ("data_version", "model_version", "context_version")
}
def _build_subset(raw: Path, destination: Path, days: int = 60) -> tuple[Path, dict]:
frame = pd.read_csv(raw)
frame = frame[frame["type"].astype(str) == "densityData"].copy()
frame["time"] = pd.to_datetime(frame["time"], utc=True)
end = frame["time"].max()
cut = end - pd.Timedelta(days=days)
frame = frame[frame["time"] >= cut][["time", "name", "value"]]
frame["time"] = frame["time"].dt.strftime("%Y-%m-%dT%H:%M:%SZ")
frame.to_csv(destination, index=False)
return destination, {
"rows": len(frame),
"sensors": int(frame["name"].nunique()),
"from": frame["time"].min(),
"to": frame["time"].max(),
"days": days,
}
def _port_free(port: int) -> bool:
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
sock.settimeout(0.5)
return sock.connect_ex(("127.0.0.1", port)) != 0
def run(args: argparse.Namespace, runner: Runner) -> dict:
raw = Path(args.csv)
if not raw.exists():
raise CheckFailed(f"CSV not found: {raw}")
if not _port_free(args.port):
raise CheckFailed(f"port {args.port} is already in use")
workdir = Path(tempfile.mkdtemp(prefix="pulsecast-e2e-"))
models_dir = workdir / "models"
uploads_dir = workdir / "uploads"
subset_path, subset_info = _build_subset(raw, workdir / "small-subset.csv", args.days)
garbage_path = workdir / "garbage.csv"
garbage_path.write_text("not,a,pulsecast,file\n1,2,3,4\n")
env = os.environ.copy()
env.update(
CONTROL_TOKEN=args.token,
PULSECAST_MODELS=str(models_dir),
PULSECAST_UPLOADS=str(uploads_dir),
)
log_path = workdir / "server.log"
log_handle = log_path.open("wb")
command = [
sys.executable,
"-m",
"uvicorn",
"pulsecast.server:app",
"--host",
"127.0.0.1",
"--port",
str(args.port),
"--log-level",
"warning",
]
process = subprocess.Popen(command, env=env, stdout=log_handle, stderr=subprocess.STDOUT)
report: dict = {
"started_at": _utc_now(),
"base_url": f"http://127.0.0.1:{args.port}",
"subset": subset_info,
"raw_csv_bytes": raw.stat().st_size,
}
try:
with httpx.Client(base_url=f"http://127.0.0.1:{args.port}", timeout=120.0) as client:
_wait_http(client, "/healthz")
headers = _job_headers(args.token)
print("Boot and snapshot")
health = client.get("/healthz").json()
runner.check(health == {"ok": True}, "/healthz returns {ok:true}")
ready = client.get("/readyz")
runner.check(ready.status_code == 200 and ready.json()["model"] is True, "/readyz 200 model=true")
meta_before = client.get("/meta").json()
runner.check(meta_before["coverage"]["sensors"] == 8, "/meta covers 8 sensors")
runner.check(
meta_before["metrics"]["mae"] < meta_before["metrics"]["baseline_mae"],
"baked model beats seasonal naive on the holdout",
)
bootstrap = client.get("/bootstrap").json()
runner.check(len(bootstrap["sensors"]) == 8, "/bootstrap returns 8 sensors")
runner.check(
set(bootstrap["series"]) == set(SENSORS),
"/bootstrap series covers all 8 sensors",
)
series_points = sum(len(points) for points in bootstrap["series"].values())
runner.check(series_points > 20_000, f"/bootstrap carries {series_points} hourly points")
context = bootstrap["context"]
runner.check(
len(context["weather"]["forecast"]) == 168,
"/bootstrap context has 168 forecast weather hours",
)
runner.check(bool(context["holidays"]), "/bootstrap context has holidays")
runner.check(
_versions(bootstrap) == _versions(meta_before),
"/bootstrap versions match /meta",
)
report["versions_before"] = _versions(meta_before)
report["bootstrap"] = {"sensors": 8, "series_points": series_points}
print("Robustness: /predict every sensor x horizon x scenario")
predict_calls = 0
for sensor in SENSORS:
for horizon, count in HORIZON_SIZES.items():
for scenario in SCENARIOS:
response = client.post(
"/predict",
json={"sensor": sensor, "horizon": horizon, "scenario": scenario},
)
if response.status_code != 200:
raise CheckFailed(
f"/predict {sensor}/{horizon}/{scenario} -> "
f"{response.status_code}: {response.text}"
)
payload = response.json()
_assert_points(
payload["points"],
count,
f"/predict {sensor}/{horizon}/{scenario}",
)
predict_calls += 1
runner.check(predict_calls == 128, "128 /predict combinations returned valid points")
print("Robustness: /simulate every scenario, all sensors")
simulate_calls = 0
for scenario in SCENARIOS:
response = client.post(
"/simulate", json={"scenario": scenario, "sensors": "all"}
)
if response.status_code != 200:
raise CheckFailed(f"/simulate {scenario} -> {response.status_code}: {response.text}")
payload = response.json()
if set(payload["forecasts"]) != set(SENSORS):
raise CheckFailed(f"/simulate {scenario}: sensor set mismatch")
for sensor, by_horizon in payload["forecasts"].items():
if set(by_horizon) != set(HORIZON_SIZES):
raise CheckFailed(f"/simulate {scenario}/{sensor}: horizon set mismatch")
for horizon, count in HORIZON_SIZES.items():
_assert_points(
by_horizon[horizon],
count,
f"/simulate {scenario}/{sensor}/{horizon}",
)
simulate_calls += 1
runner.check(simulate_calls == 4, "4 /simulate scenarios x 8 sensors x 4 horizons")
cold = client.post(
"/predict",
json={"sensor": SENSORS[0], "horizon": "48h", "weather": {"temp_c": -8.0, "precip_mm": 6.0}},
).json()["points"]
warm = client.post(
"/predict",
json={"sensor": SENSORS[0], "horizon": "48h", "weather": {"temp_c": 30.0, "precip_mm": 0.0}},
).json()["points"]
runner.check(cold != warm, "weather override changes the forecast")
print("Error handling")
no_token = client.post("/predict", json={"sensor": "nope"})
runner.check(no_token.status_code == 404, "/predict unknown sensor -> 404")
unauthorized = _submit(client, {}, subset_path, "small")
runner.check(
unauthorized.status_code == 401
and unauthorized.json()["error"]["code"] == "unauthorized",
"/train without token -> 401 unauthorized",
)
bad_token = _submit(client, _job_headers("wrong-token"), subset_path, "small")
runner.check(bad_token.status_code == 401, "/train with wrong token -> 401")
unknown_job = client.get("/train/nope", headers=headers)
runner.check(
unknown_job.status_code == 404
and unknown_job.json()["error"]["code"] == "not_found",
"/train/nope -> 404 not_found",
)
bad_model = _submit(client, headers, subset_path, "giant")
runner.check(
bad_model.status_code == 400
and bad_model.json()["error"]["code"] == "bad_request",
"/train invalid model -> 400 bad_request",
)
missing_path = client.post(
"/train", headers=headers, json={"path": "/tmp/no-such-pulsecast.csv", "model": "small"}
)
runner.check(missing_path.status_code == 400, "/train missing JSON path -> 400")
print("Small model: real CSV subset through the API")
started = time.monotonic()
submitted = _submit(client, headers, subset_path, "small")
if submitted.status_code != 202:
raise CheckFailed(f"/train small -> {submitted.status_code}: {submitted.text}")
small_submitted = submitted.json()
runner.check(
small_submitted["status"] == "queued" and small_submitted["model"] == "small",
f"/train small queued as job {small_submitted['job_id']}",
)
small_job = _wait_job(client, small_submitted["job_id"], headers)
wall = round(time.monotonic() - started, 3)
if small_job["status"] != "succeeded":
raise CheckFailed(f"small job failed: {small_job}")
small_metrics = small_job["metrics"]
for key in ("accuracy", "precision", "recall", "f1", "roc_auc", "n_train", "n_test", "positive_rate"):
if key not in small_metrics:
raise CheckFailed(f"small metrics missing {key}: {small_metrics}")
train_seconds = _seconds(small_job["started_at"], small_job["finished_at"])
runner.check(
small_job["model_version"].startswith("logit-"),
f"small model_version {small_job['model_version']}",
)
runner.check(
wall < 5.0,
f"small job took {wall}s wall time including upload (contract < 5 s; "
f"server timestamps have 1 s resolution, train_seconds={train_seconds})",
)
runner.check(small_metrics["accuracy"] > 0.6, f"small accuracy {small_metrics['accuracy']}")
runner.check(
(models_dir / "small_latest.joblib").exists(),
"small model persisted to PULSECAST_MODELS",
)
meta_after_small = client.get("/meta").json()
runner.check(
meta_after_small["small_model"]["model_version"] == small_job["model_version"],
"small model hot-swapped into /meta",
)
runner.check(
meta_after_small["small_model"]["metrics"]["accuracy"] == small_metrics["accuracy"],
"hot-swapped small metrics match the job",
)
predict_after_small = client.post(
"/predict", json={"sensor": SENSORS[0], "horizon": "7d", "scenario": "base"}
)
runner.check(
predict_after_small.status_code == 200
and len(predict_after_small.json()["points"]) == 168,
"/predict still works after the small swap",
)
report["jobs"] = {
"small": {
"job": small_job,
"wall_seconds": wall,
"train_seconds": train_seconds,
"input": subset_info,
}
}
print("Forecast model: full CSV, hot-swap, 409 while busy")
forecast_started = time.monotonic()
forecast_response = _submit(client, headers, raw, "forecast")
if forecast_response.status_code != 202:
raise CheckFailed(
f"/train forecast -> {forecast_response.status_code}: {forecast_response.text}"
)
forecast_submitted = forecast_response.json()
runner.check(
forecast_submitted["model"] == "forecast",
f"/train forecast queued as job {forecast_submitted['job_id']}",
)
_wait_job(client, forecast_submitted["job_id"], headers, state="running", timeout=30)
busy = _submit(client, headers, subset_path, "small")
runner.check(
busy.status_code == 409 and busy.json()["error"]["code"] == "conflict",
"second /train while forecast is running -> 409 conflict",
)
forecast_job = _wait_job(client, forecast_submitted["job_id"], headers, timeout=300)
forecast_wall = round(time.monotonic() - forecast_started, 3)
if forecast_job["status"] != "succeeded":
raise CheckFailed(f"forecast job failed: {forecast_job}")
forecast_metrics = forecast_job["metrics"]
for key in ("mae", "wape", "r2", "baseline_mae", "n_eval"):
if key not in forecast_metrics:
raise CheckFailed(f"forecast metrics missing {key}: {forecast_metrics}")
runner.check(
forecast_job["model_version"].startswith("histgb-"),
f"forecast model_version {forecast_job['model_version']}",
)
runner.check(
forecast_metrics["mae"] < forecast_metrics["baseline_mae"],
f"retrained forecast MAE {forecast_metrics['mae']} < baseline {forecast_metrics['baseline_mae']}",
)
runner.check(
(models_dir / "forecast_latest.joblib").exists(),
"forecast model persisted to PULSECAST_MODELS",
)
meta_after_forecast = client.get("/meta").json()
runner.check(
meta_after_forecast["model_version"] == forecast_job["model_version"]
and meta_after_forecast["model_source"] == "retrained",
"forecast model hot-swapped into /meta as retrained",
)
predict_after_forecast = client.post(
"/predict", json={"sensor": SENSORS[-1], "horizon": "48h", "scenario": "rainy-saturday"}
)
runner.check(
predict_after_forecast.status_code == 200
and predict_after_forecast.json()["model_version"] == forecast_job["model_version"],
"/predict uses the retrained forecast model",
)
runner.check(client.get("/readyz").status_code == 200, "/readyz stays 200 after hot-swap")
report["jobs"]["forecast"] = {
"job": forecast_job,
"wall_seconds": forecast_wall,
"input": {"rows": "full challenge CSV", "bytes": raw.stat().st_size},
}
print("Malformed CSV fails the job (not the process)")
bad_csv = _submit(client, headers, garbage_path, "small")
runner.check(bad_csv.status_code == 202, "malformed CSV is accepted as a job")
bad_job = _wait_job(client, bad_csv.json()["job_id"], headers)
runner.check(
bad_job["status"] == "failed" and bool(bad_job["error"]),
f"malformed CSV job failed with {bad_job['error']!r}",
)
runner.check(
client.get("/healthz").status_code == 200,
"/healthz still 200 after a failed job",
)
report["versions_after_small"] = _versions(meta_after_small)
report["versions_after_forecast"] = _versions(meta_after_forecast)
report["checks"] = runner.checks
report["ok"] = True
finally:
process.terminate()
try:
process.wait(timeout=10)
except subprocess.TimeoutExpired:
process.kill()
process.wait(timeout=5)
log_handle.close()
if process.returncode not in (0, -15):
print(f"server exited with {process.returncode}; log: {log_path}", file=sys.stderr)
print(log_path.read_text(errors="replace")[-4000:], file=sys.stderr)
report["server_log_tail"] = log_path.read_text(errors="replace")[-1000:]
if args.keep:
print(f"kept workdir {workdir}")
else:
shutil.rmtree(workdir, ignore_errors=True)
report["finished_at"] = _utc_now()
return report
def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument("--port", type=int, default=18000)
parser.add_argument("--token", default=os.environ.get("CONTROL_TOKEN") or "e2e-local-token")
parser.add_argument("--csv", default=str(config.raw_csv_path()))
parser.add_argument("--days", type=int, default=60, help="days in the small-model CSV subset")
parser.add_argument("--report", type=Path, help="write the JSON result here")
parser.add_argument("--keep", action="store_true", help="keep the temporary workdir")
args = parser.parse_args(argv)
runner = Runner()
print(f"PulseCast end-to-end check on port {args.port}")
started = time.monotonic()
try:
report = run(args, runner)
except CheckFailed as exc:
print(f"\nFAIL after {runner.checks} checks: {exc}", file=sys.stderr)
return 1
elapsed = round(time.monotonic() - started, 1)
report["elapsed_seconds"] = elapsed
print(f"\nPASS: {runner.checks} checks in {elapsed}s")
if args.report:
args.report.parent.mkdir(parents=True, exist_ok=True)
args.report.write_text(json.dumps(report, indent=2, ensure_ascii=False) + "\n")
print(f"report: {args.report}")
return 0
if __name__ == "__main__":
raise SystemExit(main())