Files
ML-py/tests/test_artifacts.py
Noel Sabia 000aaed7a5 feat: per-sensor holdout metrics and small-model confusion matrix
- train.py emits per-sensor MAE/WAPE/R2 vs the seasonal-naive baseline
- small.py emits the full confusion matrix and per-sensor classifier metrics
- metrics.json now carries the small-model summary for paper support
- regenerate baked artifacts; tests assert the tables are internally consistent
2026-11-14 06:35:14 +01:00

97 lines
3.9 KiB
Python

from __future__ import annotations
import json
from pulsecast import config
def _load(name: str):
return json.loads((config.artifacts_dir() / name).read_text())
def test_sensors_schema():
sensors = _load("sensors.json")
assert len(sensors) == 8
assert {sensor["id"] for sensor in sensors} == set(config.USABLE_SENSORS)
assert [sensor["id"] for sensor in sensors] == sorted(sensor["id"] for sensor in sensors)
for sensor in sensors:
for key in ("id", "name", "lat", "lon", "mean", "p95", "battery"):
assert key in sensor
assert sensor["p95"] >= sensor["mean"]
assert sensor["battery"] is None or 0 <= sensor["battery"] <= 100
def test_series_schema():
series = _load("series.json")
assert series["step"] == "1h"
assert set(series["series"]) == set(config.USABLE_SENSORS)
for points in series["series"].values():
assert len(points) > 1000
assert {"t", "v"} == set(points[0])
assert points[0]["t"].endswith("Z")
assert all(point["v"] >= 0 for point in points)
def test_forecasts_schema():
forecasts = _load("forecasts.json")
assert forecasts["forecasts"].keys() == set(config.USABLE_SENSORS)
for by_horizon in forecasts["forecasts"].values():
assert set(by_horizon) == {"48h", "7d"}
assert set(by_horizon["7d"]) == set(config.SCENARIOS)
assert len(by_horizon["7d"]["base"]) == 168
assert len(by_horizon["48h"]["base"]) == 48
for points in by_horizon["7d"].values():
assert all({"t", "v", "lo", "hi"} == set(point) for point in points)
assert all(point["lo"] <= point["v"] <= point["hi"] for point in points)
def test_context_schema():
context = _load("context.json")
assert context["cached"] is False
assert context["updated_at"].endswith("Z")
now = context["weather"]["now"]
for key in ("t", "temp_c", "precip_mm", "cloud_pct", "wind_kmh", "humidity_pct"):
assert key in now
assert len(context["weather"]["forecast"]) == 168
holiday_dates = {holiday["date"] for holiday in context["holidays"]}
assert "2026-11-01" in holiday_dates
assert "2025-01-01" in holiday_dates
assert "2026-12-26" in holiday_dates
def test_metrics_and_audit():
metrics = _load("metrics.json")
assert metrics["ablation"][0]["name"] == "seasonal_naive"
assert metrics["ablation"][-1]["name"] == "week_weather"
assert metrics["metrics"]["mae"] < metrics["metrics"]["baseline_mae"]
assert metrics["metrics"]["wape"] < metrics["metrics"]["baseline_wape"]
assert len(metrics["features"]) == 10
assert len(metrics["limits"]) >= 4
assert len(metrics["per_sensor"]) == 8
assert sum(row["n"] for row in metrics["per_sensor"]) == metrics["metrics"]["n_eval"]
assert {row["sensor"] for row in metrics["per_sensor"]} == set(config.USABLE_SENSORS)
for row in metrics["per_sensor"]:
assert row["mae"] >= 0
assert row["baseline_mae"] >= 0
assert 0 <= row["wape"] < 100
assert row["mae_improvement_pct"] > 0
small = _load("small_metrics.json")
matrix = small["confusion_matrix"]
total = matrix["tn"] + matrix["fp"] + matrix["fn"] + matrix["tp"]
assert total == small["metrics"]["n_test"]
assert sum(row["n"] for row in small["per_sensor"]) == total
recomputed = (matrix["tn"] + matrix["tp"]) / total
assert abs(recomputed - small["metrics"]["accuracy"]) < 0.001
assert metrics["small"]["metrics"] == small["metrics"]
assert set(matrix) == {"labels", "tn", "fp", "fn", "tp"}
audit = _load("data_audit.json")
assert audit["all_checks_ok"] is True
assert audit["raw_rows"] == 192_464
assert audit["usable_sensors"] == 8
assert set(audit["density_01_devices"]) == {"863f75b0", "f8c19730"}
assert (config.artifacts_dir() / "model.joblib").exists()
assert (config.artifacts_dir() / "small_model.joblib").exists()