- train.py emits per-sensor MAE/WAPE/R2 vs the seasonal-naive baseline - small.py emits the full confusion matrix and per-sensor classifier metrics - metrics.json now carries the small-model summary for paper support - regenerate baked artifacts; tests assert the tables are internally consistent
97 lines
3.9 KiB
Python
97 lines
3.9 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
|
|
from pulsecast import config
|
|
|
|
|
|
def _load(name: str):
|
|
return json.loads((config.artifacts_dir() / name).read_text())
|
|
|
|
|
|
def test_sensors_schema():
|
|
sensors = _load("sensors.json")
|
|
assert len(sensors) == 8
|
|
assert {sensor["id"] for sensor in sensors} == set(config.USABLE_SENSORS)
|
|
assert [sensor["id"] for sensor in sensors] == sorted(sensor["id"] for sensor in sensors)
|
|
for sensor in sensors:
|
|
for key in ("id", "name", "lat", "lon", "mean", "p95", "battery"):
|
|
assert key in sensor
|
|
assert sensor["p95"] >= sensor["mean"]
|
|
assert sensor["battery"] is None or 0 <= sensor["battery"] <= 100
|
|
|
|
|
|
def test_series_schema():
|
|
series = _load("series.json")
|
|
assert series["step"] == "1h"
|
|
assert set(series["series"]) == set(config.USABLE_SENSORS)
|
|
for points in series["series"].values():
|
|
assert len(points) > 1000
|
|
assert {"t", "v"} == set(points[0])
|
|
assert points[0]["t"].endswith("Z")
|
|
assert all(point["v"] >= 0 for point in points)
|
|
|
|
|
|
def test_forecasts_schema():
|
|
forecasts = _load("forecasts.json")
|
|
assert forecasts["forecasts"].keys() == set(config.USABLE_SENSORS)
|
|
for by_horizon in forecasts["forecasts"].values():
|
|
assert set(by_horizon) == {"48h", "7d"}
|
|
assert set(by_horizon["7d"]) == set(config.SCENARIOS)
|
|
assert len(by_horizon["7d"]["base"]) == 168
|
|
assert len(by_horizon["48h"]["base"]) == 48
|
|
for points in by_horizon["7d"].values():
|
|
assert all({"t", "v", "lo", "hi"} == set(point) for point in points)
|
|
assert all(point["lo"] <= point["v"] <= point["hi"] for point in points)
|
|
|
|
|
|
def test_context_schema():
|
|
context = _load("context.json")
|
|
assert context["cached"] is False
|
|
assert context["updated_at"].endswith("Z")
|
|
now = context["weather"]["now"]
|
|
for key in ("t", "temp_c", "precip_mm", "cloud_pct", "wind_kmh", "humidity_pct"):
|
|
assert key in now
|
|
assert len(context["weather"]["forecast"]) == 168
|
|
holiday_dates = {holiday["date"] for holiday in context["holidays"]}
|
|
assert "2026-11-01" in holiday_dates
|
|
assert "2025-01-01" in holiday_dates
|
|
assert "2026-12-26" in holiday_dates
|
|
|
|
|
|
def test_metrics_and_audit():
|
|
metrics = _load("metrics.json")
|
|
assert metrics["ablation"][0]["name"] == "seasonal_naive"
|
|
assert metrics["ablation"][-1]["name"] == "week_weather"
|
|
assert metrics["metrics"]["mae"] < metrics["metrics"]["baseline_mae"]
|
|
assert metrics["metrics"]["wape"] < metrics["metrics"]["baseline_wape"]
|
|
assert len(metrics["features"]) == 10
|
|
assert len(metrics["limits"]) >= 4
|
|
assert len(metrics["per_sensor"]) == 8
|
|
assert sum(row["n"] for row in metrics["per_sensor"]) == metrics["metrics"]["n_eval"]
|
|
assert {row["sensor"] for row in metrics["per_sensor"]} == set(config.USABLE_SENSORS)
|
|
for row in metrics["per_sensor"]:
|
|
assert row["mae"] >= 0
|
|
assert row["baseline_mae"] >= 0
|
|
assert 0 <= row["wape"] < 100
|
|
assert row["mae_improvement_pct"] > 0
|
|
|
|
small = _load("small_metrics.json")
|
|
matrix = small["confusion_matrix"]
|
|
total = matrix["tn"] + matrix["fp"] + matrix["fn"] + matrix["tp"]
|
|
assert total == small["metrics"]["n_test"]
|
|
assert sum(row["n"] for row in small["per_sensor"]) == total
|
|
recomputed = (matrix["tn"] + matrix["tp"]) / total
|
|
assert abs(recomputed - small["metrics"]["accuracy"]) < 0.001
|
|
assert metrics["small"]["metrics"] == small["metrics"]
|
|
assert set(matrix) == {"labels", "tn", "fp", "fn", "tp"}
|
|
|
|
audit = _load("data_audit.json")
|
|
assert audit["all_checks_ok"] is True
|
|
assert audit["raw_rows"] == 192_464
|
|
assert audit["usable_sensors"] == 8
|
|
assert set(audit["density_01_devices"]) == {"863f75b0", "f8c19730"}
|
|
|
|
assert (config.artifacts_dir() / "model.joblib").exists()
|
|
assert (config.artifacts_dir() / "small_model.joblib").exists()
|