#!/usr/bin/env python3 """End-to-end acceptance run against a real FastAPI process on port 18000. Starts uvicorn with ``CONTROL_TOKEN``, then exercises robustness and the full training lifecycle: - /healthz, /readyz, /meta, /bootstrap (8 sensors, hourly series, context) - /predict for every sensor x horizon x scenario (128 calls) - /simulate for every scenario with all sensors - auth, unknown job, unknown model, malformed CSV, one-job-at-a-time - POST /train model=small on a real CSV subset; poll to success; hot-swap - POST /train model=forecast on the full CSV; poll to success; hot-swap - /predict still works after each swap Exit code is 0 only when every check passes. Figures printed to stdout are the measured numbers; ``--report`` writes the same result as JSON. """ from __future__ import annotations import argparse import json import os import shutil import socket import subprocess import sys import tempfile import time from datetime import UTC, datetime from pathlib import Path import httpx import pandas as pd from pulsecast import config HORIZON_SIZES = {"15m": 1, "1h": 4, "48h": 48, "7d": 168} SCENARIOS = tuple(config.SCENARIOS) SENSORS = tuple(sorted(config.USABLE_SENSORS)) class CheckFailed(RuntimeError): pass class Runner: def __init__(self) -> None: self.checks = 0 self.results: dict = {} def check(self, condition: bool, message: str) -> None: self.checks += 1 if not condition: raise CheckFailed(message) print(f" [ok] {message}") def _utc_now() -> str: return datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ") def _parse_ts(value: str) -> pd.Timestamp: return pd.Timestamp(value) def _seconds(start: str | None, end: str | None) -> float | None: if not start or not end: return None return round((_parse_ts(end) - _parse_ts(start)).total_seconds(), 3) def _wait_http(client: httpx.Client, url: str, timeout: float = 30.0) -> None: deadline = time.monotonic() + timeout last: Exception | None = None while time.monotonic() < deadline: try: response = client.get(url, timeout=2.0) if response.status_code == 200: return except Exception as exc: # noqa: BLE001 - startup race last = exc time.sleep(0.1) raise CheckFailed(f"server did not answer {url} within {timeout}s: {last}") def _wait_job( client: httpx.Client, job_id: str, headers: dict[str, str], *, timeout: float = 180.0, state: str | None = None, ) -> dict: deadline = time.monotonic() + timeout last_status = None while time.monotonic() < deadline: response = client.get(f"/train/{job_id}", headers=headers) if response.status_code != 200: raise CheckFailed(f"GET /train/{job_id} -> {response.status_code}: {response.text}") job = response.json() last_status = job["status"] if state is not None and last_status == state: return job if state is None and last_status in {"succeeded", "failed"}: return job time.sleep(0.05) raise CheckFailed(f"job {job_id} did not reach {state or 'terminal'} (last={last_status})") def _job_headers(token: str) -> dict[str, str]: return {"Authorization": f"Bearer {token}"} def _submit( client: httpx.Client, headers: dict[str, str], path: Path, model: str ) -> httpx.Response: return client.post( "/train", headers=headers, files={"file": (path.name, path.read_bytes(), "text/csv")}, data={"model": model}, timeout=120.0, ) def _assert_points(points: list[dict], expected: int, label: str) -> None: if len(points) != expected: raise CheckFailed(f"{label}: expected {expected} points, got {len(points)}") previous = None for point in points: if set(point) != {"t", "v", "lo", "hi"}: raise CheckFailed(f"{label}: unexpected point keys {sorted(point)}") if not point["t"].endswith("Z"): raise CheckFailed(f"{label}: timestamp is not UTC RFC 3339: {point['t']}") if point["v"] < 0 or point["lo"] > point["v"] or point["v"] > point["hi"]: raise CheckFailed(f"{label}: invalid band {point}") stamp = _parse_ts(point["t"]) if previous is not None and stamp <= previous: raise CheckFailed(f"{label}: timestamps are not increasing at {point['t']}") previous = stamp def _versions(payload: dict) -> dict: return { key: payload.get(key) for key in ("data_version", "model_version", "context_version") } def _build_subset(raw: Path, destination: Path, days: int = 60) -> tuple[Path, dict]: frame = pd.read_csv(raw) frame = frame[frame["type"].astype(str) == "densityData"].copy() frame["time"] = pd.to_datetime(frame["time"], utc=True) end = frame["time"].max() cut = end - pd.Timedelta(days=days) frame = frame[frame["time"] >= cut][["time", "name", "value"]] frame["time"] = frame["time"].dt.strftime("%Y-%m-%dT%H:%M:%SZ") frame.to_csv(destination, index=False) return destination, { "rows": len(frame), "sensors": int(frame["name"].nunique()), "from": frame["time"].min(), "to": frame["time"].max(), "days": days, } def _port_free(port: int) -> bool: with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: sock.settimeout(0.5) return sock.connect_ex(("127.0.0.1", port)) != 0 def run(args: argparse.Namespace, runner: Runner) -> dict: raw = Path(args.csv) if not raw.exists(): raise CheckFailed(f"CSV not found: {raw}") if not _port_free(args.port): raise CheckFailed(f"port {args.port} is already in use") workdir = Path(tempfile.mkdtemp(prefix="pulsecast-e2e-")) models_dir = workdir / "models" uploads_dir = workdir / "uploads" subset_path, subset_info = _build_subset(raw, workdir / "small-subset.csv", args.days) garbage_path = workdir / "garbage.csv" garbage_path.write_text("not,a,pulsecast,file\n1,2,3,4\n") env = os.environ.copy() env.update( CONTROL_TOKEN=args.token, PULSECAST_MODELS=str(models_dir), PULSECAST_UPLOADS=str(uploads_dir), ) log_path = workdir / "server.log" log_handle = log_path.open("wb") command = [ sys.executable, "-m", "uvicorn", "pulsecast.server:app", "--host", "127.0.0.1", "--port", str(args.port), "--log-level", "warning", ] process = subprocess.Popen(command, env=env, stdout=log_handle, stderr=subprocess.STDOUT) report: dict = { "started_at": _utc_now(), "base_url": f"http://127.0.0.1:{args.port}", "subset": subset_info, "raw_csv_bytes": raw.stat().st_size, } try: with httpx.Client(base_url=f"http://127.0.0.1:{args.port}", timeout=120.0) as client: _wait_http(client, "/healthz") headers = _job_headers(args.token) print("Boot and snapshot") health = client.get("/healthz").json() runner.check(health == {"ok": True}, "/healthz returns {ok:true}") ready = client.get("/readyz") runner.check(ready.status_code == 200 and ready.json()["model"] is True, "/readyz 200 model=true") meta_before = client.get("/meta").json() runner.check(meta_before["coverage"]["sensors"] == 8, "/meta covers 8 sensors") runner.check( meta_before["metrics"]["mae"] < meta_before["metrics"]["baseline_mae"], "baked model beats seasonal naive on the holdout", ) bootstrap = client.get("/bootstrap").json() runner.check(len(bootstrap["sensors"]) == 8, "/bootstrap returns 8 sensors") runner.check( set(bootstrap["series"]) == set(SENSORS), "/bootstrap series covers all 8 sensors", ) series_points = sum(len(points) for points in bootstrap["series"].values()) runner.check(series_points > 20_000, f"/bootstrap carries {series_points} hourly points") context = bootstrap["context"] runner.check( len(context["weather"]["forecast"]) == 168, "/bootstrap context has 168 forecast weather hours", ) runner.check(bool(context["holidays"]), "/bootstrap context has holidays") runner.check( _versions(bootstrap) == _versions(meta_before), "/bootstrap versions match /meta", ) report["versions_before"] = _versions(meta_before) report["bootstrap"] = {"sensors": 8, "series_points": series_points} print("Robustness: /predict every sensor x horizon x scenario") predict_calls = 0 for sensor in SENSORS: for horizon, count in HORIZON_SIZES.items(): for scenario in SCENARIOS: response = client.post( "/predict", json={"sensor": sensor, "horizon": horizon, "scenario": scenario}, ) if response.status_code != 200: raise CheckFailed( f"/predict {sensor}/{horizon}/{scenario} -> " f"{response.status_code}: {response.text}" ) payload = response.json() _assert_points( payload["points"], count, f"/predict {sensor}/{horizon}/{scenario}", ) predict_calls += 1 runner.check(predict_calls == 128, "128 /predict combinations returned valid points") print("Robustness: /simulate every scenario, all sensors") simulate_calls = 0 for scenario in SCENARIOS: response = client.post( "/simulate", json={"scenario": scenario, "sensors": "all"} ) if response.status_code != 200: raise CheckFailed(f"/simulate {scenario} -> {response.status_code}: {response.text}") payload = response.json() if set(payload["forecasts"]) != set(SENSORS): raise CheckFailed(f"/simulate {scenario}: sensor set mismatch") for sensor, by_horizon in payload["forecasts"].items(): if set(by_horizon) != set(HORIZON_SIZES): raise CheckFailed(f"/simulate {scenario}/{sensor}: horizon set mismatch") for horizon, count in HORIZON_SIZES.items(): _assert_points( by_horizon[horizon], count, f"/simulate {scenario}/{sensor}/{horizon}", ) simulate_calls += 1 runner.check(simulate_calls == 4, "4 /simulate scenarios x 8 sensors x 4 horizons") cold = client.post( "/predict", json={"sensor": SENSORS[0], "horizon": "48h", "weather": {"temp_c": -8.0, "precip_mm": 6.0}}, ).json()["points"] warm = client.post( "/predict", json={"sensor": SENSORS[0], "horizon": "48h", "weather": {"temp_c": 30.0, "precip_mm": 0.0}}, ).json()["points"] runner.check(cold != warm, "weather override changes the forecast") print("Error handling") no_token = client.post("/predict", json={"sensor": "nope"}) runner.check(no_token.status_code == 404, "/predict unknown sensor -> 404") unauthorized = _submit(client, {}, subset_path, "small") runner.check( unauthorized.status_code == 401 and unauthorized.json()["error"]["code"] == "unauthorized", "/train without token -> 401 unauthorized", ) bad_token = _submit(client, _job_headers("wrong-token"), subset_path, "small") runner.check(bad_token.status_code == 401, "/train with wrong token -> 401") unknown_job = client.get("/train/nope", headers=headers) runner.check( unknown_job.status_code == 404 and unknown_job.json()["error"]["code"] == "not_found", "/train/nope -> 404 not_found", ) bad_model = _submit(client, headers, subset_path, "giant") runner.check( bad_model.status_code == 400 and bad_model.json()["error"]["code"] == "bad_request", "/train invalid model -> 400 bad_request", ) missing_path = client.post( "/train", headers=headers, json={"path": "/tmp/no-such-pulsecast.csv", "model": "small"} ) runner.check(missing_path.status_code == 400, "/train missing JSON path -> 400") print("Small model: real CSV subset through the API") started = time.monotonic() submitted = _submit(client, headers, subset_path, "small") if submitted.status_code != 202: raise CheckFailed(f"/train small -> {submitted.status_code}: {submitted.text}") small_submitted = submitted.json() runner.check( small_submitted["status"] == "queued" and small_submitted["model"] == "small", f"/train small queued as job {small_submitted['job_id']}", ) small_job = _wait_job(client, small_submitted["job_id"], headers) wall = round(time.monotonic() - started, 3) if small_job["status"] != "succeeded": raise CheckFailed(f"small job failed: {small_job}") small_metrics = small_job["metrics"] for key in ("accuracy", "precision", "recall", "f1", "roc_auc", "n_train", "n_test", "positive_rate"): if key not in small_metrics: raise CheckFailed(f"small metrics missing {key}: {small_metrics}") train_seconds = _seconds(small_job["started_at"], small_job["finished_at"]) runner.check( small_job["model_version"].startswith("logit-"), f"small model_version {small_job['model_version']}", ) runner.check( wall < 5.0, f"small job took {wall}s wall time including upload (contract < 5 s; " f"server timestamps have 1 s resolution, train_seconds={train_seconds})", ) runner.check(small_metrics["accuracy"] > 0.6, f"small accuracy {small_metrics['accuracy']}") runner.check( (models_dir / "small_latest.joblib").exists(), "small model persisted to PULSECAST_MODELS", ) meta_after_small = client.get("/meta").json() runner.check( meta_after_small["small_model"]["model_version"] == small_job["model_version"], "small model hot-swapped into /meta", ) runner.check( meta_after_small["small_model"]["metrics"]["accuracy"] == small_metrics["accuracy"], "hot-swapped small metrics match the job", ) predict_after_small = client.post( "/predict", json={"sensor": SENSORS[0], "horizon": "7d", "scenario": "base"} ) runner.check( predict_after_small.status_code == 200 and len(predict_after_small.json()["points"]) == 168, "/predict still works after the small swap", ) report["jobs"] = { "small": { "job": small_job, "wall_seconds": wall, "train_seconds": train_seconds, "input": subset_info, } } print("Forecast model: full CSV, hot-swap, 409 while busy") forecast_started = time.monotonic() forecast_response = _submit(client, headers, raw, "forecast") if forecast_response.status_code != 202: raise CheckFailed( f"/train forecast -> {forecast_response.status_code}: {forecast_response.text}" ) forecast_submitted = forecast_response.json() runner.check( forecast_submitted["model"] == "forecast", f"/train forecast queued as job {forecast_submitted['job_id']}", ) _wait_job(client, forecast_submitted["job_id"], headers, state="running", timeout=30) busy = _submit(client, headers, subset_path, "small") runner.check( busy.status_code == 409 and busy.json()["error"]["code"] == "conflict", "second /train while forecast is running -> 409 conflict", ) forecast_job = _wait_job(client, forecast_submitted["job_id"], headers, timeout=300) forecast_wall = round(time.monotonic() - forecast_started, 3) if forecast_job["status"] != "succeeded": raise CheckFailed(f"forecast job failed: {forecast_job}") forecast_metrics = forecast_job["metrics"] for key in ("mae", "wape", "r2", "baseline_mae", "n_eval"): if key not in forecast_metrics: raise CheckFailed(f"forecast metrics missing {key}: {forecast_metrics}") runner.check( forecast_job["model_version"].startswith("histgb-"), f"forecast model_version {forecast_job['model_version']}", ) runner.check( forecast_metrics["mae"] < forecast_metrics["baseline_mae"], f"retrained forecast MAE {forecast_metrics['mae']} < baseline {forecast_metrics['baseline_mae']}", ) runner.check( (models_dir / "forecast_latest.joblib").exists(), "forecast model persisted to PULSECAST_MODELS", ) meta_after_forecast = client.get("/meta").json() runner.check( meta_after_forecast["model_version"] == forecast_job["model_version"] and meta_after_forecast["model_source"] == "retrained", "forecast model hot-swapped into /meta as retrained", ) predict_after_forecast = client.post( "/predict", json={"sensor": SENSORS[-1], "horizon": "48h", "scenario": "rainy-saturday"} ) runner.check( predict_after_forecast.status_code == 200 and predict_after_forecast.json()["model_version"] == forecast_job["model_version"], "/predict uses the retrained forecast model", ) runner.check(client.get("/readyz").status_code == 200, "/readyz stays 200 after hot-swap") report["jobs"]["forecast"] = { "job": forecast_job, "wall_seconds": forecast_wall, "input": {"rows": "full challenge CSV", "bytes": raw.stat().st_size}, } print("Malformed CSV fails the job (not the process)") bad_csv = _submit(client, headers, garbage_path, "small") runner.check(bad_csv.status_code == 202, "malformed CSV is accepted as a job") bad_job = _wait_job(client, bad_csv.json()["job_id"], headers) runner.check( bad_job["status"] == "failed" and bool(bad_job["error"]), f"malformed CSV job failed with {bad_job['error']!r}", ) runner.check( client.get("/healthz").status_code == 200, "/healthz still 200 after a failed job", ) report["versions_after_small"] = _versions(meta_after_small) report["versions_after_forecast"] = _versions(meta_after_forecast) report["checks"] = runner.checks report["ok"] = True finally: process.terminate() try: process.wait(timeout=10) except subprocess.TimeoutExpired: process.kill() process.wait(timeout=5) log_handle.close() if process.returncode not in (0, -15): print(f"server exited with {process.returncode}; log: {log_path}", file=sys.stderr) print(log_path.read_text(errors="replace")[-4000:], file=sys.stderr) report["server_log_tail"] = log_path.read_text(errors="replace")[-1000:] if args.keep: print(f"kept workdir {workdir}") else: shutil.rmtree(workdir, ignore_errors=True) report["finished_at"] = _utc_now() return report def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) parser.add_argument("--port", type=int, default=18000) parser.add_argument("--token", default=os.environ.get("CONTROL_TOKEN") or "e2e-local-token") parser.add_argument("--csv", default=str(config.raw_csv_path())) parser.add_argument("--days", type=int, default=60, help="days in the small-model CSV subset") parser.add_argument("--report", type=Path, help="write the JSON result here") parser.add_argument("--keep", action="store_true", help="keep the temporary workdir") args = parser.parse_args(argv) runner = Runner() print(f"PulseCast end-to-end check on port {args.port}") started = time.monotonic() try: report = run(args, runner) except CheckFailed as exc: print(f"\nFAIL after {runner.checks} checks: {exc}", file=sys.stderr) return 1 elapsed = round(time.monotonic() - started, 1) report["elapsed_seconds"] = elapsed print(f"\nPASS: {runner.checks} checks in {elapsed}s") if args.report: args.report.parent.mkdir(parents=True, exist_ok=True) args.report.write_text(json.dumps(report, indent=2, ensure_ascii=False) + "\n") print(f"report: {args.report}") return 0 if __name__ == "__main__": raise SystemExit(main())