feat: complete ML pipeline, FastAPI service, small model, tests, image
- finish the global HistGB bundle, scenario forecasts and 28-day ablation - export sensors/series/forecasts/metrics/context plus baked weather frames - add FastAPI health/meta/predict/simulate and background /train jobs - add the p75 busy classifier, retraining from CSV, and hot-swap persistence - add pytest coverage, multi-stage non-root Dockerfile, README and model card Holdout: weather 7d MAE 3.32 / WAPE 27.5% / R2 0.777 vs seasonal naive MAE 4.23 / WAPE 35.0% / R2 0.656 (5,350 eval rows).
This commit is contained in:
1 parent
68402803b2
commit
939a8db482
43 files changed
+4090
-105
No files matched your search
@@ -0,0 +1,53 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
CONTROL_TOKEN = "test-token"
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def client(tmp_path_factory):
|
||||
os.environ["CONTROL_TOKEN"] = CONTROL_TOKEN
|
||||
os.environ["PULSECAST_MODELS"] = str(tmp_path_factory.mktemp("models"))
|
||||
os.environ["PULSECAST_UPLOADS"] = str(tmp_path_factory.mktemp("uploads"))
|
||||
from pulsecast.server import app
|
||||
|
||||
with TestClient(app) as test_client:
|
||||
yield test_client
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def auth_headers() -> dict[str, str]:
|
||||
return {"Authorization": f"Bearer {CONTROL_TOKEN}"}
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def synthetic_csv(tmp_path_factory) -> Path:
|
||||
"""Two sensors, 60 days hourly, clear daily/weekly pattern + weather."""
|
||||
rng = np.random.default_rng(7)
|
||||
hours = pd.date_range("2026-06-01", periods=24 * 60, freq="h", tz="UTC")
|
||||
rows = []
|
||||
for sensor in ("test-store-a", "test-store-b"):
|
||||
for timestamp in hours:
|
||||
local = timestamp.tz_convert("Europe/Berlin")
|
||||
daily = 35.0 * np.exp(-((local.hour - 19) ** 2) / 18.0)
|
||||
weekend = 12.0 if local.day_of_week == 5 else 0.0
|
||||
value = max(0.0, 8.0 + daily + weekend + rng.normal(0, 4))
|
||||
rows.append(
|
||||
{
|
||||
"time": timestamp.strftime("%Y-%m-%dT%H:%M:%SZ"),
|
||||
"sensor": sensor,
|
||||
"value": round(value, 3),
|
||||
"temp_c": round(15.0 + rng.normal(0, 5), 2),
|
||||
"precip_mm": round(max(0.0, rng.normal(0, 1)), 2),
|
||||
}
|
||||
)
|
||||
path = tmp_path_factory.mktemp("csv") / "upload.csv"
|
||||
pd.DataFrame(rows).to_csv(path, index=False)
|
||||
return path
|
||||
@@ -0,0 +1,78 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
from pulsecast import config
|
||||
|
||||
|
||||
def _load(name: str):
|
||||
return json.loads((config.artifacts_dir() / name).read_text())
|
||||
|
||||
|
||||
def test_sensors_schema():
|
||||
sensors = _load("sensors.json")
|
||||
assert len(sensors) == 8
|
||||
assert {sensor["id"] for sensor in sensors} == set(config.USABLE_SENSORS)
|
||||
assert [sensor["id"] for sensor in sensors] == sorted(sensor["id"] for sensor in sensors)
|
||||
for sensor in sensors:
|
||||
for key in ("id", "name", "lat", "lon", "mean", "p95", "battery"):
|
||||
assert key in sensor
|
||||
assert sensor["p95"] >= sensor["mean"]
|
||||
assert sensor["battery"] is None or 0 <= sensor["battery"] <= 100
|
||||
|
||||
|
||||
def test_series_schema():
|
||||
series = _load("series.json")
|
||||
assert series["step"] == "1h"
|
||||
assert set(series["series"]) == set(config.USABLE_SENSORS)
|
||||
for points in series["series"].values():
|
||||
assert len(points) > 1000
|
||||
assert {"t", "v"} == set(points[0])
|
||||
assert points[0]["t"].endswith("Z")
|
||||
assert all(point["v"] >= 0 for point in points)
|
||||
|
||||
|
||||
def test_forecasts_schema():
|
||||
forecasts = _load("forecasts.json")
|
||||
assert forecasts["forecasts"].keys() == set(config.USABLE_SENSORS)
|
||||
for by_horizon in forecasts["forecasts"].values():
|
||||
assert set(by_horizon) == {"48h", "7d"}
|
||||
assert set(by_horizon["7d"]) == set(config.SCENARIOS)
|
||||
assert len(by_horizon["7d"]["base"]) == 168
|
||||
assert len(by_horizon["48h"]["base"]) == 48
|
||||
for points in by_horizon["7d"].values():
|
||||
assert all({"t", "v", "lo", "hi"} == set(point) for point in points)
|
||||
assert all(point["lo"] <= point["v"] <= point["hi"] for point in points)
|
||||
|
||||
|
||||
def test_context_schema():
|
||||
context = _load("context.json")
|
||||
assert context["cached"] is False
|
||||
assert context["updated_at"].endswith("Z")
|
||||
now = context["weather"]["now"]
|
||||
for key in ("t", "temp_c", "precip_mm", "cloud_pct", "wind_kmh", "humidity_pct"):
|
||||
assert key in now
|
||||
assert len(context["weather"]["forecast"]) == 168
|
||||
holiday_dates = {holiday["date"] for holiday in context["holidays"]}
|
||||
assert "2026-11-01" in holiday_dates
|
||||
assert "2025-01-01" in holiday_dates
|
||||
assert "2026-12-26" in holiday_dates
|
||||
|
||||
|
||||
def test_metrics_and_audit():
|
||||
metrics = _load("metrics.json")
|
||||
assert metrics["ablation"][0]["name"] == "seasonal_naive"
|
||||
assert metrics["ablation"][-1]["name"] == "week_weather"
|
||||
assert metrics["metrics"]["mae"] < metrics["metrics"]["baseline_mae"]
|
||||
assert metrics["metrics"]["wape"] < metrics["metrics"]["baseline_wape"]
|
||||
assert len(metrics["features"]) == 10
|
||||
assert len(metrics["limits"]) >= 4
|
||||
|
||||
audit = _load("data_audit.json")
|
||||
assert audit["all_checks_ok"] is True
|
||||
assert audit["raw_rows"] == 192_464
|
||||
assert audit["usable_sensors"] == 8
|
||||
assert set(audit["density_01_devices"]) == {"863f75b0", "f8c19730"}
|
||||
|
||||
assert (config.artifacts_dir() / "model.joblib").exists()
|
||||
assert (config.artifacts_dir() / "small_model.joblib").exists()
|
||||
@@ -0,0 +1,55 @@
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
def test_healthz(client):
|
||||
response = client.get("/healthz")
|
||||
assert response.status_code == 200
|
||||
assert response.json() == {"ok": True}
|
||||
|
||||
|
||||
def test_readyz(client):
|
||||
response = client.get("/readyz")
|
||||
assert response.status_code == 200
|
||||
assert response.json() == {"ok": True, "model": True}
|
||||
|
||||
|
||||
def test_meta_contract(client):
|
||||
response = client.get("/meta")
|
||||
assert response.status_code == 200
|
||||
payload = response.json()
|
||||
|
||||
for key in (
|
||||
"data_version",
|
||||
"model_version",
|
||||
"context_version",
|
||||
"coverage",
|
||||
"metrics",
|
||||
"trained_at",
|
||||
"train_from",
|
||||
"train_to",
|
||||
"features",
|
||||
"holiday_effect",
|
||||
):
|
||||
assert key in payload, key
|
||||
|
||||
assert payload["coverage"]["sensors"] == 8
|
||||
assert payload["coverage"]["cadence_min"] == 15
|
||||
assert payload["coverage"]["from"].endswith("Z")
|
||||
|
||||
headline = payload["metrics"]
|
||||
assert headline["mae"] < headline["baseline_mae"]
|
||||
assert headline["wape"] < headline["baseline_wape"]
|
||||
assert headline["r2"] > headline["baseline_r2"]
|
||||
assert 0 < headline["wape"] < 100
|
||||
assert 0.5 < headline["r2"] <= 1.0
|
||||
|
||||
assert "small_model" in payload
|
||||
assert payload["small_model"]["features"] == [
|
||||
"hour",
|
||||
"weekday",
|
||||
"is_weekend",
|
||||
"temp_c",
|
||||
"precip_mm",
|
||||
"is_holiday",
|
||||
]
|
||||
assert 0.5 < payload["small_model"]["metrics"]["accuracy"] <= 1.0
|
||||
@@ -0,0 +1,125 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import pandas as pd
|
||||
|
||||
SENSOR = "hn-stadtgalerie-01"
|
||||
|
||||
|
||||
def _assert_points(points: list[dict]) -> None:
|
||||
assert points
|
||||
for point in points:
|
||||
assert set(point) == {"t", "v", "lo", "hi"}
|
||||
assert point["t"].endswith("Z")
|
||||
assert point["v"] >= 0
|
||||
assert point["lo"] <= point["v"] <= point["hi"]
|
||||
|
||||
|
||||
def test_predict_7d(client):
|
||||
response = client.post(
|
||||
"/predict", json={"sensor": SENSOR, "horizon": "7d", "scenario": "base"}
|
||||
)
|
||||
assert response.status_code == 200
|
||||
payload = response.json()
|
||||
assert payload["sensor"] == SENSOR
|
||||
assert payload["horizon"] == "7d"
|
||||
assert payload["scenario"] == "base"
|
||||
assert len(payload["points"]) == 168
|
||||
assert payload["model_version"].startswith("histgb-")
|
||||
_assert_points(payload["points"])
|
||||
|
||||
|
||||
def test_predict_all_horizons(client):
|
||||
expected = {"15m": 1, "1h": 4, "48h": 48, "7d": 168}
|
||||
for horizon, count in expected.items():
|
||||
response = client.post("/predict", json={"sensor": SENSOR, "horizon": horizon})
|
||||
assert response.status_code == 200, response.text
|
||||
assert len(response.json()["points"]) == count, horizon
|
||||
|
||||
|
||||
def test_predict_weather_override_changes_values(client):
|
||||
cold = client.post(
|
||||
"/predict",
|
||||
json={"sensor": SENSOR, "horizon": "48h", "weather": {"temp_c": -8.0, "precip_mm": 6.0}},
|
||||
)
|
||||
warm = client.post(
|
||||
"/predict",
|
||||
json={"sensor": SENSOR, "horizon": "48h", "weather": {"temp_c": 30.0, "precip_mm": 0.0}},
|
||||
)
|
||||
assert cold.status_code == warm.status_code == 200
|
||||
assert cold.json()["points"] != warm.json()["points"]
|
||||
|
||||
|
||||
def test_predict_unknown_sensor(client):
|
||||
response = client.post("/predict", json={"sensor": "nope", "horizon": "7d"})
|
||||
assert response.status_code == 404
|
||||
assert response.json()["error"]["code"] == "not_found"
|
||||
|
||||
|
||||
def test_predict_bad_horizon_and_scenario(client):
|
||||
response = client.post("/predict", json={"sensor": SENSOR, "horizon": "9d"})
|
||||
assert response.status_code == 400
|
||||
assert response.json()["error"]["code"] == "bad_request"
|
||||
|
||||
response = client.post("/predict", json={"sensor": SENSOR, "scenario": "snow"})
|
||||
assert response.status_code == 400
|
||||
assert response.json()["error"]["code"] == "bad_request"
|
||||
|
||||
|
||||
def test_predict_validation_error_shape(client):
|
||||
response = client.post("/predict", json={"horizon": "7d"})
|
||||
assert response.status_code == 400
|
||||
assert response.json()["error"]["code"] == "bad_request"
|
||||
assert "sensor" in response.json()["error"]["message"]
|
||||
|
||||
|
||||
def test_simulate_all_sensors_and_scenarios(client):
|
||||
response = client.post(
|
||||
"/simulate",
|
||||
json={
|
||||
"scenario": "sunny-saturday-market",
|
||||
"weather": {"temp_delta_c": 4, "precip_mm": 0},
|
||||
"sensors": "all",
|
||||
},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
payload = response.json()
|
||||
assert payload["scenario"] == "sunny-saturday-market"
|
||||
assert len(payload["forecasts"]) == 8
|
||||
for sensor_forecast in payload["forecasts"].values():
|
||||
assert set(sensor_forecast) == {"15m", "1h", "48h", "7d"}
|
||||
assert len(sensor_forecast["15m"]) == 1
|
||||
assert len(sensor_forecast["1h"]) == 4
|
||||
assert len(sensor_forecast["48h"]) == 48
|
||||
assert len(sensor_forecast["7d"]) == 168
|
||||
_assert_points(sensor_forecast["7d"])
|
||||
|
||||
|
||||
def test_simulate_changes_saturday_store_hours(client):
|
||||
def points(scenario: str):
|
||||
response = client.post(
|
||||
"/simulate", json={"scenario": scenario, "sensors": [SENSOR]}
|
||||
)
|
||||
assert response.status_code == 200
|
||||
return response.json()["forecasts"][SENSOR]["7d"]
|
||||
|
||||
base = points("base")
|
||||
sunny = points("sunny-saturday-market")
|
||||
changed = []
|
||||
for left, right in zip(base, sunny, strict=True):
|
||||
local = pd.Timestamp(left["t"]).tz_convert("Europe/Berlin")
|
||||
if local.day_of_week == 5 and 10 <= local.hour < 21:
|
||||
changed.append(left["v"] != right["v"])
|
||||
assert changed and any(changed)
|
||||
|
||||
rainy = points("rainy-saturday")
|
||||
rainy_changed = [
|
||||
left["v"] != right["v"]
|
||||
for left, right in zip(sunny, rainy, strict=True)
|
||||
]
|
||||
assert any(rainy_changed)
|
||||
|
||||
|
||||
def test_simulate_unknown_sensor(client):
|
||||
response = client.post("/simulate", json={"scenario": "base", "sensors": ["nope"]})
|
||||
assert response.status_code == 404
|
||||
assert response.json()["error"]["code"] == "not_found"
|
||||
@@ -0,0 +1,125 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def _wait_for_job(client, job_id: str, headers: dict[str, str], timeout: float = 20.0):
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
response = client.get(f"/train/{job_id}", headers=headers)
|
||||
assert response.status_code == 200, response.text
|
||||
job = response.json()
|
||||
if job["status"] in {"succeeded", "failed"}:
|
||||
return job
|
||||
time.sleep(0.1)
|
||||
raise AssertionError(f"training job {job_id} did not finish in {timeout}s")
|
||||
|
||||
|
||||
def _submit(client, headers, path: Path, model: str = "small"):
|
||||
return client.post(
|
||||
"/train",
|
||||
headers=headers,
|
||||
files={"file": ("upload.csv", path.read_bytes(), "text/csv")},
|
||||
data={"model": model},
|
||||
)
|
||||
|
||||
|
||||
def test_training_requires_token(client, synthetic_csv):
|
||||
response = _submit(client, {}, synthetic_csv)
|
||||
assert response.status_code == 401
|
||||
assert response.json()["error"]["code"] == "unauthorized"
|
||||
|
||||
response = _submit(client, {"Authorization": "Bearer wrong"}, synthetic_csv)
|
||||
assert response.status_code == 401
|
||||
|
||||
|
||||
def test_unknown_job_and_bad_model(client, auth_headers, synthetic_csv):
|
||||
response = client.get("/train/does-not-exist", headers=auth_headers)
|
||||
assert response.status_code == 404
|
||||
assert response.json()["error"]["code"] == "not_found"
|
||||
|
||||
response = _submit(client, auth_headers, synthetic_csv, model="giant")
|
||||
assert response.status_code == 400
|
||||
assert response.json()["error"]["code"] == "bad_request"
|
||||
|
||||
|
||||
def test_small_training_job(client, auth_headers, synthetic_csv):
|
||||
started = time.monotonic()
|
||||
response = _submit(client, auth_headers, synthetic_csv)
|
||||
assert response.status_code == 202, response.text
|
||||
submitted = response.json()
|
||||
assert submitted["status"] == "queued"
|
||||
assert submitted["model"] == "small"
|
||||
assert submitted["submitted_at"].endswith("Z")
|
||||
|
||||
job = _wait_for_job(client, submitted["job_id"], auth_headers)
|
||||
elapsed = time.monotonic() - started
|
||||
assert job["status"] == "succeeded", job
|
||||
assert job["error"] is None
|
||||
assert job["model"] == "small"
|
||||
assert job["started_at"] and job["finished_at"]
|
||||
assert job["model_version"].startswith("logit-")
|
||||
|
||||
metrics = job["metrics"]
|
||||
for key in ("accuracy", "precision", "recall", "f1", "n_train", "n_test", "positive_rate"):
|
||||
assert key in metrics, key
|
||||
assert metrics["accuracy"] > 0.6
|
||||
assert 0 < metrics["accuracy"] <= 1
|
||||
assert metrics["n_train"] > 0
|
||||
assert metrics["n_test"] > 0
|
||||
assert metrics["roc_auc"] is None or 0.5 < metrics["roc_auc"] <= 1
|
||||
|
||||
# Contract requirement: the small model trains in under 5 s.
|
||||
assert elapsed < 5.0, elapsed
|
||||
|
||||
model_dir = Path(os.environ["PULSECAST_MODELS"])
|
||||
assert (model_dir / "small_latest.joblib").exists()
|
||||
assert (model_dir / f"{submitted['job_id']}-small.joblib").exists()
|
||||
|
||||
meta = client.get("/meta").json()
|
||||
assert meta["small_model"]["model_version"] == job["model_version"]
|
||||
assert meta["small_model"]["metrics"]["accuracy"] == metrics["accuracy"]
|
||||
|
||||
|
||||
def test_training_json_path(client, auth_headers, synthetic_csv):
|
||||
response = client.post(
|
||||
"/train",
|
||||
headers=auth_headers,
|
||||
json={"path": str(synthetic_csv), "model": "small"},
|
||||
)
|
||||
assert response.status_code == 202, response.text
|
||||
job = _wait_for_job(client, response.json()["job_id"], auth_headers)
|
||||
assert job["status"] == "succeeded", job
|
||||
assert job["metrics"]["accuracy"] > 0.6
|
||||
|
||||
|
||||
def test_training_missing_file(client, auth_headers):
|
||||
response = client.post(
|
||||
"/train",
|
||||
headers=auth_headers,
|
||||
json={"path": "/tmp/definitely-not-a-pulsecast-file.csv", "model": "small"},
|
||||
)
|
||||
assert response.status_code == 400
|
||||
assert response.json()["error"]["code"] == "bad_request"
|
||||
|
||||
|
||||
def test_one_job_at_a_time(client, auth_headers, synthetic_csv, monkeypatch):
|
||||
from pulsecast import server
|
||||
|
||||
original = server.train_small_from_csv
|
||||
|
||||
def slow_train(*args, **kwargs):
|
||||
time.sleep(0.6)
|
||||
return original(*args, **kwargs)
|
||||
|
||||
monkeypatch.setattr(server, "train_small_from_csv", slow_train)
|
||||
first = _submit(client, auth_headers, synthetic_csv)
|
||||
assert first.status_code == 202
|
||||
second = _submit(client, auth_headers, synthetic_csv)
|
||||
assert second.status_code == 409
|
||||
assert second.json()["error"]["code"] == "conflict"
|
||||
|
||||
job = _wait_for_job(client, first.json()["job_id"], auth_headers)
|
||||
assert job["status"] == "succeeded", job
|
||||
Reference in new issue
Block a user