- finish the global HistGB bundle, scenario forecasts and 28-day ablation - export sensors/series/forecasts/metrics/context plus baked weather frames - add FastAPI health/meta/predict/simulate and background /train jobs - add the p75 busy classifier, retraining from CSV, and hot-swap persistence - add pytest coverage, multi-stage non-root Dockerfile, README and model card Holdout: weather 7d MAE 3.32 / WAPE 27.5% / R2 0.777 vs seasonal naive MAE 4.23 / WAPE 35.0% / R2 0.656 (5,350 eval rows).
126 lines
4.5 KiB
Python
126 lines
4.5 KiB
Python
from __future__ import annotations
|
|
|
|
import os
|
|
import time
|
|
from pathlib import Path
|
|
|
|
|
|
def _wait_for_job(client, job_id: str, headers: dict[str, str], timeout: float = 20.0):
|
|
deadline = time.monotonic() + timeout
|
|
while time.monotonic() < deadline:
|
|
response = client.get(f"/train/{job_id}", headers=headers)
|
|
assert response.status_code == 200, response.text
|
|
job = response.json()
|
|
if job["status"] in {"succeeded", "failed"}:
|
|
return job
|
|
time.sleep(0.1)
|
|
raise AssertionError(f"training job {job_id} did not finish in {timeout}s")
|
|
|
|
|
|
def _submit(client, headers, path: Path, model: str = "small"):
|
|
return client.post(
|
|
"/train",
|
|
headers=headers,
|
|
files={"file": ("upload.csv", path.read_bytes(), "text/csv")},
|
|
data={"model": model},
|
|
)
|
|
|
|
|
|
def test_training_requires_token(client, synthetic_csv):
|
|
response = _submit(client, {}, synthetic_csv)
|
|
assert response.status_code == 401
|
|
assert response.json()["error"]["code"] == "unauthorized"
|
|
|
|
response = _submit(client, {"Authorization": "Bearer wrong"}, synthetic_csv)
|
|
assert response.status_code == 401
|
|
|
|
|
|
def test_unknown_job_and_bad_model(client, auth_headers, synthetic_csv):
|
|
response = client.get("/train/does-not-exist", headers=auth_headers)
|
|
assert response.status_code == 404
|
|
assert response.json()["error"]["code"] == "not_found"
|
|
|
|
response = _submit(client, auth_headers, synthetic_csv, model="giant")
|
|
assert response.status_code == 400
|
|
assert response.json()["error"]["code"] == "bad_request"
|
|
|
|
|
|
def test_small_training_job(client, auth_headers, synthetic_csv):
|
|
started = time.monotonic()
|
|
response = _submit(client, auth_headers, synthetic_csv)
|
|
assert response.status_code == 202, response.text
|
|
submitted = response.json()
|
|
assert submitted["status"] == "queued"
|
|
assert submitted["model"] == "small"
|
|
assert submitted["submitted_at"].endswith("Z")
|
|
|
|
job = _wait_for_job(client, submitted["job_id"], auth_headers)
|
|
elapsed = time.monotonic() - started
|
|
assert job["status"] == "succeeded", job
|
|
assert job["error"] is None
|
|
assert job["model"] == "small"
|
|
assert job["started_at"] and job["finished_at"]
|
|
assert job["model_version"].startswith("logit-")
|
|
|
|
metrics = job["metrics"]
|
|
for key in ("accuracy", "precision", "recall", "f1", "n_train", "n_test", "positive_rate"):
|
|
assert key in metrics, key
|
|
assert metrics["accuracy"] > 0.6
|
|
assert 0 < metrics["accuracy"] <= 1
|
|
assert metrics["n_train"] > 0
|
|
assert metrics["n_test"] > 0
|
|
assert metrics["roc_auc"] is None or 0.5 < metrics["roc_auc"] <= 1
|
|
|
|
# Contract requirement: the small model trains in under 5 s.
|
|
assert elapsed < 5.0, elapsed
|
|
|
|
model_dir = Path(os.environ["PULSECAST_MODELS"])
|
|
assert (model_dir / "small_latest.joblib").exists()
|
|
assert (model_dir / f"{submitted['job_id']}-small.joblib").exists()
|
|
|
|
meta = client.get("/meta").json()
|
|
assert meta["small_model"]["model_version"] == job["model_version"]
|
|
assert meta["small_model"]["metrics"]["accuracy"] == metrics["accuracy"]
|
|
|
|
|
|
def test_training_json_path(client, auth_headers, synthetic_csv):
|
|
response = client.post(
|
|
"/train",
|
|
headers=auth_headers,
|
|
json={"path": str(synthetic_csv), "model": "small"},
|
|
)
|
|
assert response.status_code == 202, response.text
|
|
job = _wait_for_job(client, response.json()["job_id"], auth_headers)
|
|
assert job["status"] == "succeeded", job
|
|
assert job["metrics"]["accuracy"] > 0.6
|
|
|
|
|
|
def test_training_missing_file(client, auth_headers):
|
|
response = client.post(
|
|
"/train",
|
|
headers=auth_headers,
|
|
json={"path": "/tmp/definitely-not-a-pulsecast-file.csv", "model": "small"},
|
|
)
|
|
assert response.status_code == 400
|
|
assert response.json()["error"]["code"] == "bad_request"
|
|
|
|
|
|
def test_one_job_at_a_time(client, auth_headers, synthetic_csv, monkeypatch):
|
|
from pulsecast import server
|
|
|
|
original = server.train_small_from_csv
|
|
|
|
def slow_train(*args, **kwargs):
|
|
time.sleep(0.6)
|
|
return original(*args, **kwargs)
|
|
|
|
monkeypatch.setattr(server, "train_small_from_csv", slow_train)
|
|
first = _submit(client, auth_headers, synthetic_csv)
|
|
assert first.status_code == 202
|
|
second = _submit(client, auth_headers, synthetic_csv)
|
|
assert second.status_code == 409
|
|
assert second.json()["error"]["code"] == "conflict"
|
|
|
|
job = _wait_for_job(client, first.json()["job_id"], auth_headers)
|
|
assert job["status"] == "succeeded", job
|