feat: complete ML pipeline, FastAPI service, small model, tests, image

- finish the global HistGB bundle, scenario forecasts and 28-day ablation
- export sensors/series/forecasts/metrics/context plus baked weather frames
- add FastAPI health/meta/predict/simulate and background /train jobs
- add the p75 busy classifier, retraining from CSV, and hot-swap persistence
- add pytest coverage, multi-stage non-root Dockerfile, README and model card

Holdout: weather 7d MAE 3.32 / WAPE 27.5% / R2 0.777 vs seasonal naive
MAE 4.23 / WAPE 35.0% / R2 0.656 (5,350 eval rows).
This commit is contained in:
Noel Sabia committed 2026-11-14 05:01:16 +01:00
1 parent 68402803b2
commit 939a8db482
43 files changed
+4090 -105

No files matched your search

+22
View File
@@ -45,6 +45,24 @@ def raw_csv_path() -> Path:
)
def weather_dir() -> Path:
"""Baked weather frames live beside the other exported artifacts."""
return Path(os.environ.get("PULSECAST_WEATHER", artifacts_dir())).resolve()
def model_dir() -> Path:
"""Writable model persistence for training jobs (/tmp inside a container)."""
return Path(os.environ.get("PULSECAST_MODELS", "/tmp/pulsecast/models")).resolve()
def upload_dir() -> Path:
return Path(os.environ.get("PULSECAST_UPLOADS", "/tmp/pulsecast/uploads")).resolve()
def control_token() -> str:
return os.environ.get("CONTROL_TOKEN", "").strip()
# --------------------------------------------------------------------------- data
# Multi-sensor window documented in mission/data.md. density-01 has older
@@ -171,6 +189,10 @@ FEATURE_DAY_HOLIDAY = FEATURE_DAY + ["is_holiday"] # is_holiday already present
FEATURE_WEEK_NO_WEATHER = FEATURE_WEEK
FEATURE_WEEK_WEATHER = FEATURE_WEEK + FEATURE_WEATHER
# Small classifier ("I coded an AI" artifact). Contract features, deliberately
# no lags: this model is cheap to retrain on an uploaded CSV in a few seconds.
SMALL_FEATURES = ["hour", "weekday", "is_weekend", "temp_c", "precip_mm", "is_holiday"]
# Scenario presets. Weather adjustments apply on the event's weekdays; the
# event uplift is a transparent multiplicative overlay applied after the model.
# These are decision-support assumptions, not causal estimates.
-1
View File
@@ -11,7 +11,6 @@ Design rules:
from __future__ import annotations
import numpy as np
import pandas as pd
from . import config
+231
View File
@@ -0,0 +1,231 @@
"""Forecast model bundle: fit, inference frame, prediction bands, scenarios.
The production model is one global HistGradientBoostingRegressor over all eight
sensors. It only uses features that are at least 168 h old (7-day-safe), so a
168 h forecast can be produced in one pass without recursive leakage.
"""
from __future__ import annotations
import numpy as np
import pandas as pd
from sklearn.ensemble import HistGradientBoostingRegressor
from sklearn.metrics import r2_score
from . import config
from .features import SENSOR_CODES, future_calendar
# Fixed, deliberately small model. No tuning beyond a short sanity pass: the
# 28-day holdout is the honest signal, not the training loss.
HISTGB_PARAMS = {
"loss": "squared_error",
"learning_rate": 0.06,
"max_iter": 300,
"max_leaf_nodes": 31,
"min_samples_leaf": 20,
"l2_regularization": 1.0,
"early_stopping": False,
"random_state": 42,
}
def fit(frame: pd.DataFrame, columns: list[str] | tuple[str, ...]):
"""Fit the global HistGB on rows with an observed target."""
data = frame[frame["y"].notna()]
model = HistGradientBoostingRegressor(**HISTGB_PARAMS)
model.fit(data[list(columns)], data["y"].astype(float))
return model
def predict(bundle: dict, frame: pd.DataFrame) -> np.ndarray:
"""Predict non-negative footfall density for a feature frame."""
values = bundle["model"].predict(frame[list(bundle["features"])])
return np.clip(np.asarray(values, dtype=float), 0.0, None)
def metrics(y_true, y_pred) -> dict:
"""MAE, WAPE and R² on a common evaluation frame."""
y = np.asarray(y_true, dtype=float)
p = np.asarray(y_pred, dtype=float)
abs_err = np.abs(y - p)
denominator = float(np.sum(np.abs(y)))
return {
"mae": float(np.mean(abs_err)),
"wape": (100.0 * float(np.sum(abs_err)) / denominator) if denominator else 0.0,
"r2": float(r2_score(y, p)) if len(y) > 1 else 0.0,
"n": len(y),
}
def residual_quantiles(eval_frame: pd.DataFrame, pred: np.ndarray) -> dict:
"""80% prediction band offsets from the honest holdout residuals."""
residual = np.asarray(eval_frame["y"], dtype=float) - np.asarray(pred, dtype=float)
return {
"p10": round(float(np.quantile(residual, 0.10)), 4),
"p50": round(float(np.quantile(residual, 0.50)), 4),
"p90": round(float(np.quantile(residual, 0.90)), 4),
}
def weather_climatology(archive: pd.DataFrame) -> dict:
"""Month x local-hour weather means, used when forecast weather is absent."""
default = {"temp_c": 14.0, "precip_mm": 0.0}
if archive is None or archive.empty or "temp_c" not in archive.columns:
return {"default": default}
frame = archive.copy()
local = frame["time"].dt.tz_convert(config.TIMEZONE)
frame["month"] = local.dt.month.astype(int)
frame["hour"] = local.dt.hour.astype(int)
grouped = (
frame.groupby(["month", "hour"], observed=True)
.agg(temp_c=("temp_c", "mean"), precip_mm=("precip_mm", "mean"))
.reset_index()
)
out = {
"default": {
"temp_c": round(float(frame["temp_c"].mean()), 3),
"precip_mm": round(float(frame["precip_mm"].mean()), 3),
}
}
for row in grouped.itertuples(index=False):
out[f"{int(row.month)}|{int(row.hour)}"] = {
"temp_c": round(float(row.temp_c), 3),
"precip_mm": round(float(row.precip_mm), 3),
}
return out
def _lookup_values(
bundle: dict, frame: pd.DataFrame, sensor: str
) -> tuple[pd.Series, pd.Series]:
"""Return (last observed, 4-week mean) per future weekday/hour."""
lookup = bundle.get("seasonal_lookup") or {}
fallback = float(bundle.get("sensor_mean", {}).get(sensor, 10.0))
last_values = []
mean_values = []
for weekday, hour in zip(frame["weekday"], frame["hour"], strict=True):
entry = lookup.get(f"{sensor}|{int(weekday)}|{int(hour)}", {})
mean4 = float(entry.get("mean4", fallback))
mean_values.append(mean4)
last_values.append(float(entry.get("last", mean4)))
return (
pd.Series(last_values, index=frame.index, dtype=float),
pd.Series(mean_values, index=frame.index, dtype=float),
)
def _fill_weather(bundle: dict, frame: pd.DataFrame) -> pd.DataFrame:
climatology = bundle.get("weather_climatology") or {}
default = climatology.get("default") or {"temp_c": 14.0, "precip_mm": 0.0}
keys = [f"{int(m)}|{int(h)}" for m, h in zip(frame["month"], frame["hour"], strict=True)]
temp = [
float(climatology.get(key, default)["temp_c"])
for key in keys
]
precip = [
float(climatology.get(key, default)["precip_mm"])
for key in keys
]
frame["temp_c"] = frame["temp_c"].fillna(pd.Series(temp, index=frame.index))
frame["precip_mm"] = frame["precip_mm"].fillna(pd.Series(precip, index=frame.index))
return frame
def future_frame(
bundle: dict,
sensor: str,
start: pd.Timestamp,
periods: int,
weather_forecast: pd.DataFrame | None = None,
*,
temp_delta: float = 0.0,
precip_mm: float | None = None,
weather_mask: np.ndarray | None = None,
holiday_override: bool | None = None,
) -> pd.DataFrame:
"""Build a leakage-free inference frame for one sensor and horizon.
``weather_mask`` selects the rows where the scenario weather adjustment
applies (for example, Saturday store hours). Rows outside the mask keep the
baked forecast weather.
"""
frame = future_calendar(start, periods)
frame["sensor"] = sensor
frame["sensor_code"] = SENSOR_CODES.get(sensor, -1)
last_values, mean_values = _lookup_values(bundle, frame, sensor)
frame["lag_week"] = last_values
frame["seasonal_mean_4w"] = mean_values
if weather_forecast is not None and not weather_forecast.empty:
weather = weather_forecast.rename(columns={"time": "t"}).copy()
weather["t"] = pd.to_datetime(weather["t"], utc=True)
weather = (
weather[["t", "temp_c", "precip_mm"]]
.sort_values("t")
.drop_duplicates("t", keep="last")
)
frame = pd.merge_asof(
frame.sort_values("t"),
weather,
on="t",
direction="nearest",
tolerance=pd.Timedelta("3h"),
)
else:
frame["temp_c"] = np.nan
frame["precip_mm"] = np.nan
frame = _fill_weather(bundle, frame)
mask = (
np.ones(len(frame), dtype=bool)
if weather_mask is None
else np.asarray(weather_mask, dtype=bool)
)
if temp_delta:
frame.loc[mask, "temp_c"] = frame.loc[mask, "temp_c"] + float(temp_delta)
if precip_mm is not None:
frame.loc[mask, "precip_mm"] = float(precip_mm)
if holiday_override is not None:
frame["is_holiday"] = int(bool(holiday_override))
frame["holiday_name"] = None
return frame.sort_values("t").reset_index(drop=True)
def event_multiplier(frame: pd.DataFrame, scenario: dict) -> np.ndarray:
"""Transparent multiplicative overlay for a scenario's event hours."""
event = scenario.get("event")
if not event:
return np.ones(len(frame), dtype=float)
mask = frame["weekday"].isin(event.get("weekdays", [])).to_numpy(dtype=bool, copy=True)
hours = event.get("hours")
if hours is not None:
mask &= frame["hour"].isin(hours).to_numpy(dtype=bool, copy=True)
return np.where(mask, 1.0 + float(event.get("uplift", 0.0)), 1.0)
def _rfc3339(ts) -> str:
return pd.Timestamp(ts).tz_convert("UTC").strftime("%Y-%m-%dT%H:%M:%SZ")
def points(frame: pd.DataFrame, pred: np.ndarray, bundle: dict, sensor: str) -> list[dict]:
"""Contract point shape: t, v, lo, hi (80% band)."""
band = bundle.get("residual_quantiles") or {"p10": -5.0, "p90": 5.0}
p10 = float(band.get("p10", -5.0))
p90 = float(band.get("p90", 5.0))
out = []
for timestamp, value in zip(frame["t"], pred, strict=True):
v = max(float(value), 0.0)
lo = max(v + p10, 0.0)
hi = max(v + p90, lo)
out.append(
{
"t": _rfc3339(timestamp),
"v": round(v, 2),
"lo": round(lo, 2),
"hi": round(hi, 2),
}
)
return out
+6 -4
View File
@@ -78,8 +78,10 @@ def clean(df: pd.DataFrame) -> tuple[pd.DataFrame, pd.DataFrame, dict]:
)
multi = density[density["time"] >= pd.Timestamp(config.MULTI_SENSOR_START)]
# The device merge spans the full history: the old device is outside the
# 8-sensor window, so counting only `multi` would hide it.
density_01_devices = (
multi[multi["sensor"] == "density-01"]
density[density["sensor"] == "density-01"]
.groupby("sensor_id", observed=True)
.agg(rows=("value", "size"), first=("time", "min"), last=("time", "max"))
)
@@ -99,9 +101,9 @@ def clean(df: pd.DataFrame) -> tuple[pd.DataFrame, pd.DataFrame, dict]:
)
audit = {
"raw_rows": int(len(df)),
"density_rows": int(len(df[df["type"] == "densityData"])),
"battery_rows": int(len(df[df["type"] == "batteryData"])),
"raw_rows": len(df),
"density_rows": len(df[df["type"] == "densityData"]),
"battery_rows": len(df[df["type"] == "batteryData"]),
"device_ids": int(df["sensor_id"].nunique()),
"sensor_names": int(df["name"].nunique()),
"usable_sensors": len(usable),
+25 -10
View File
@@ -5,7 +5,6 @@ from __future__ import annotations
import argparse
import json
import logging
from pathlib import Path
import pandas as pd
@@ -52,7 +51,7 @@ def export_sensors(density: pd.DataFrame, battery: pd.DataFrame) -> list[dict]:
"mean": round(float(rows["value"].mean()), 1),
"p95": round(float(rows["value"].quantile(0.95)), 1),
"battery": (
int(round(float(battery_latest.loc[sensor, "value"])))
round(float(battery_latest.loc[sensor, "value"]))
if sensor in battery_latest.index
else None
),
@@ -94,9 +93,9 @@ def _weather_now(forecast: pd.DataFrame, meta: dict) -> dict:
"t": rfc3339(ts),
"temp_c": round(float(current.get("temp_c", row["temp_c"])), 1),
"precip_mm": round(float(current.get("precip_mm", row["precip_mm"])), 2),
"cloud_pct": int(round(float(current.get("cloud_pct", row["cloud_pct"])))),
"cloud_pct": round(float(current.get("cloud_pct", row["cloud_pct"]))),
"wind_kmh": round(float(current.get("wind_kmh", row["wind_kmh"])), 1),
"humidity_pct": int(round(float(current.get("humidity_pct", row["humidity_pct"])))),
"humidity_pct": round(float(current.get("humidity_pct", row["humidity_pct"]))),
}
@@ -109,9 +108,9 @@ def export_context(forecast: pd.DataFrame, meta: dict) -> dict:
"t": rfc3339(row.time),
"temp_c": round(float(row.temp_c), 1),
"precip_mm": round(float(row.precip_mm), 2),
"cloud_pct": int(round(float(row.cloud_pct))),
"cloud_pct": round(float(row.cloud_pct)),
"wind_kmh": round(float(row.wind_kmh), 1),
"humidity_pct": int(round(float(row.humidity_pct))),
"humidity_pct": round(float(row.humidity_pct)),
}
for row in window.itertuples(index=False)
]
@@ -140,6 +139,14 @@ def run_data(*, refresh_weather: bool = False) -> dict:
raw = load_raw()
density, battery, audit = clean(raw)
write_audit(audit)
# The product window is the 8-sensor period. density-01 has 13 months of
# history, but using it alone would add single-sensor annual seasonality that
# the other seven locations cannot support. Keep the full-file audit above.
window_start = pd.Timestamp(config.MULTI_SENSOR_START)
density = density[density["time"] >= window_start].copy()
battery = battery[battery["time"] >= window_start].copy()
density[["time", "sensor", "sensor_id", "value"]].to_parquet(
cache / "density_clean.parquet", index=False
)
@@ -147,7 +154,7 @@ def run_data(*, refresh_weather: bool = False) -> dict:
cache / "battery_clean.parquet", index=False
)
log.info(
"parsed %s rows; %s usable sensors; %s density rows",
"parsed %s rows; %s usable sensors; %s density rows in the 8-sensor window",
audit["raw_rows"],
audit["usable_sensors"],
len(density),
@@ -163,6 +170,14 @@ def run_data(*, refresh_weather: bool = False) -> dict:
features.to_parquet(cache / "hourly_features.parquet", index=False)
log.info("built %s hourly feature rows", len(features))
# Bake compact weather frames beside the JSON so the FastAPI image can
# retrain and serve /predict without the git-ignored data/cache directory.
archive.to_parquet(artifacts / "weather_archive.parquet", index=False)
forecast.to_parquet(artifacts / "weather_forecast.parquet", index=False)
(artifacts / "weather_meta.json").write_text(
json.dumps(meta, indent=2, ensure_ascii=False) + "\n"
)
data_version = minute_iso(audit["time_range"]["to"])
sensors = export_sensors(density, battery)
(artifacts / "sensors.json").write_text(
@@ -174,9 +189,9 @@ def run_data(*, refresh_weather: bool = False) -> dict:
summary = {
"sensors": len(sensors),
"series_sensors": len(series["series"]),
"hourly_rows": int(len(hourly)),
"feature_rows": int(len(features)),
"weather_rows": int(len(archive)),
"hourly_rows": len(hourly),
"feature_rows": len(features),
"weather_rows": len(archive),
"context_forecast_hours": len(context["weather"]["forecast"]),
"data_version": data_version,
"audit_ok": audit["all_checks_ok"],
+88
View File
@@ -0,0 +1,88 @@
"""Runtime retraining from an uploaded CSV.
The API must not depend on the raw challenge file or on the git-ignored cache.
Forecast retraining therefore reuses the baked weather archive and the same
leakage-safe feature builder as ``make train``.
"""
from __future__ import annotations
import json
import logging
from datetime import UTC, datetime
from pathlib import Path
import pandas as pd
from . import config
from .features import build_features
from .small import load_hourly_csv
from .train import train_bundle
log = logging.getLogger(__name__)
def _read_weather() -> tuple[pd.DataFrame, dict]:
weather_dir = config.weather_dir()
archive_path = weather_dir / "weather_archive.parquet"
meta_path = weather_dir / "weather_meta.json"
if archive_path.exists():
archive = pd.read_parquet(archive_path)
else:
archive = pd.DataFrame(
{
"time": pd.to_datetime([], utc=True),
"temp_c": pd.Series(dtype=float),
"precip_mm": pd.Series(dtype=float),
}
)
log.warning("no baked weather archive; forecast training uses missing weather")
meta = (
json.loads(meta_path.read_text())
if meta_path.exists()
else {
"fetched_at": datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ"),
"source": "missing",
}
)
return archive, meta
def train_forecast_from_csv(path: Path | str) -> tuple[dict, dict, dict]:
"""Retrain the global forecast model from an uploaded PAX-style CSV."""
hourly = load_hourly_csv(path)
hourly = hourly[hourly["sensor"].isin(config.USABLE_SENSORS)].copy()
if hourly.empty:
raise ValueError(
"CSV has none of the eight known sensors; forecast retraining needs "
"sensor names from the challenge data"
)
start = pd.Timestamp(config.MULTI_SENSOR_START)
hourly = hourly[hourly["t"] >= start][["sensor", "t", "v"]].copy()
if len(hourly) < 500:
raise ValueError("not enough hourly rows for a 28-day holdout")
archive, meta = _read_weather()
features = build_features(hourly, archive)
audit = {
"usable_sensors": int(hourly["sensor"].nunique()),
"cadence_median_min": 60,
"time_range": {
"from": hourly["t"].min().strftime("%Y-%m-%dT%H:%M:%SZ"),
"to": hourly["t"].max().strftime("%Y-%m-%dT%H:%M:%SZ"),
},
}
bundle, metrics_payload, _ = train_bundle(hourly, features, archive, meta, audit)
trained_at = bundle["metadata"]["trained_at"]
version = f"histgb-{trained_at[:10].replace('-', '')}-{trained_at[11:19].replace(':', '')}"
bundle["metadata"]["model_version"] = version
metrics_payload["model_version"] = version
log.info(
"forecast retrained from %s: %s MAE %.2f WAPE %.1f%%",
path,
version,
metrics_payload["metrics"]["mae"],
metrics_payload["metrics"]["wape"],
)
return bundle, metrics_payload, audit
+630
View File
@@ -0,0 +1,630 @@
"""FastAPI ML service: /healthz, /readyz, /meta, /predict, /simulate, /train.
Shapes follow CONTRACT.md. The service is internal: only Go exposes it publicly,
and the training routes require a bearer control token.
"""
from __future__ import annotations
import hmac
import json
import logging
import threading
import uuid
from contextlib import asynccontextmanager
from dataclasses import dataclass, field
from datetime import UTC, datetime
from pathlib import Path
from typing import Any
import joblib
import numpy as np
import pandas as pd
from fastapi import FastAPI, Header, HTTPException, Request
from fastapi.exceptions import RequestValidationError
from fastapi.middleware.cors import CORSMiddleware
from fastapi.responses import JSONResponse
from pydantic import BaseModel, ConfigDict
from . import config
from .features import future_calendar
from .model import event_multiplier, future_frame, points, predict
from .retrain import train_forecast_from_csv
from .small import train_small_from_csv
log = logging.getLogger(__name__)
MAX_UPLOAD_BYTES = 60 * 1024 * 1024
ACTIVE_STATUSES = {"queued", "running"}
def _utc_now() -> str:
return datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ")
@dataclass
class Runtime:
bundle: dict | None = None
metrics: dict | None = None
weather: pd.DataFrame | None = None
weather_meta: dict | None = None
small_bundle: dict | None = None
small_metrics: dict | None = None
model_source: str = "baked"
jobs: dict[str, dict[str, Any]] = field(default_factory=dict)
jobs_lock: threading.Lock = field(default_factory=threading.Lock)
schedule_lock: threading.Lock = field(default_factory=threading.Lock)
def _load_runtime(runtime: Runtime) -> None:
artifacts = config.artifacts_dir()
model_path = artifacts / "model.joblib"
metrics_path = artifacts / "metrics.json"
if model_path.exists() and metrics_path.exists():
runtime.bundle = joblib.load(model_path)
runtime.metrics = json.loads(metrics_path.read_text())
log.info("loaded baked model %s", runtime.metrics.get("model_version"))
else:
log.warning("baked model missing under %s; /readyz will fail", artifacts)
weather_path = config.weather_dir() / "weather_forecast.parquet"
if weather_path.exists():
runtime.weather = pd.read_parquet(weather_path)
meta_path = config.weather_dir() / "weather_meta.json"
runtime.weather_meta = (
json.loads(meta_path.read_text()) if meta_path.exists() else None
)
small_path = artifacts / "small_model.joblib"
small_metrics_path = artifacts / "small_metrics.json"
if small_path.exists():
runtime.small_bundle = joblib.load(small_path)
if small_metrics_path.exists():
runtime.small_metrics = json.loads(small_metrics_path.read_text())
# --------------------------------------------------------------------------- schemas
class WeatherOverride(BaseModel):
model_config = ConfigDict(extra="ignore")
temp_c: float | None = None
precip_mm: float | None = None
class PredictRequest(BaseModel):
model_config = ConfigDict(extra="ignore")
sensor: str
horizon: str = "7d"
scenario: str = "base"
weather: WeatherOverride | None = None
is_holiday: bool | None = None
class SimulateWeather(BaseModel):
model_config = ConfigDict(extra="ignore")
temp_delta_c: float = 0.0
precip_mm: float | None = None
class SimulateRequest(BaseModel):
model_config = ConfigDict(extra="ignore")
scenario: str = "base"
weather: SimulateWeather | None = None
sensors: str | list[str] = "all"
is_holiday: bool | None = None
# --------------------------------------------------------------------------- helpers
def _require_token(authorization: str | None) -> None:
expected = config.control_token()
if not expected:
raise HTTPException(503, "CONTROL_TOKEN is not configured")
if not authorization or not authorization.startswith("Bearer "):
raise HTTPException(401, "missing control token")
token = authorization.removeprefix("Bearer ").strip()
if not hmac.compare_digest(token, expected):
raise HTTPException(401, "invalid control token")
def _scenario_weather_mask(calendar: pd.DataFrame, scenario: dict) -> np.ndarray | None:
event = scenario.get("event")
if not event:
return None
mask = calendar["weekday"].isin(event.get("weekdays", [])).to_numpy(dtype=bool, copy=True)
hours = event.get("hours")
if hours is not None:
mask &= calendar["hour"].isin(hours).to_numpy(dtype=bool, copy=True)
return mask
def _override_weather(runtime: Runtime, override: WeatherOverride | None) -> pd.DataFrame | None:
if override is None or runtime.weather is None:
return runtime.weather
weather = runtime.weather.copy()
if override.temp_c is not None:
weather["temp_c"] = float(override.temp_c)
if override.precip_mm is not None:
weather["precip_mm"] = float(override.precip_mm)
return weather
def _hourly_points(
runtime: Runtime,
sensor: str,
scenario_name: str,
periods: int,
weather_override: WeatherOverride | None = None,
holiday_override: bool | None = None,
*,
start: pd.Timestamp | None = None,
temp_delta_override: float | None = None,
precip_override: float | None = None,
) -> list[dict]:
assert runtime.bundle is not None
scenario = config.SCENARIOS[scenario_name]
start = start or pd.Timestamp.now(tz="UTC").ceil("h")
calendar = future_calendar(start, periods)
mask = _scenario_weather_mask(calendar, scenario)
explicit_weather = weather_override is not None and (
weather_override.temp_c is not None or weather_override.precip_mm is not None
)
temp_delta = 0.0 if explicit_weather else scenario["temp_delta_c"]
precip = None if explicit_weather else scenario["precip_mm"]
if temp_delta_override is not None:
temp_delta = float(temp_delta_override)
if precip_override is not None:
precip = precip_override
frame = future_frame(
runtime.bundle,
sensor,
start,
periods,
_override_weather(runtime, weather_override),
temp_delta=temp_delta,
precip_mm=precip,
weather_mask=mask,
holiday_override=holiday_override,
)
pred = predict(runtime.bundle, frame) * event_multiplier(frame, scenario)
return points(frame, pred, runtime.bundle, sensor)
def _quarter_hour_points(
runtime: Runtime,
sensor: str,
scenario_name: str,
count: int,
weather_override: WeatherOverride | None = None,
holiday_override: bool | None = None,
*,
first: pd.Timestamp | None = None,
temp_delta_override: float | None = None,
precip_override: float | None = None,
) -> list[dict]:
"""Interpolate the hourly model to the next 15-minute points (15m/1h)."""
first = first or pd.Timestamp.now(tz="UTC").ceil("15min")
hourly = _hourly_points(
runtime,
sensor,
scenario_name,
periods=3,
weather_override=weather_override,
holiday_override=holiday_override,
start=first.floor("h"),
temp_delta_override=temp_delta_override,
precip_override=precip_override,
)
source_times = pd.to_datetime([point["t"] for point in hourly], utc=True)
targets = pd.date_range(first, periods=count, freq="15min")
x = source_times.astype("int64").to_numpy() // 10**9
tx = targets.astype("int64").to_numpy() // 10**9
out = []
for target, target_x in zip(targets, tx, strict=True):
row = {"t": target.strftime("%Y-%m-%dT%H:%M:%SZ")}
for key in ("v", "lo", "hi"):
values = np.array([point[key] for point in hourly], dtype=float)
row[key] = round(max(float(np.interp(target_x, x, values)), 0.0), 2)
out.append(row)
return out
def _forecast(
runtime: Runtime,
sensor: str,
horizon: str,
scenario_name: str,
weather_override: WeatherOverride | None,
holiday_override: bool | None,
) -> list[dict]:
if horizon == "15m":
return _quarter_hour_points(
runtime, sensor, scenario_name, 1, weather_override, holiday_override
)
if horizon == "1h":
return _quarter_hour_points(
runtime, sensor, scenario_name, 4, weather_override, holiday_override
)
return _hourly_points(
runtime,
sensor,
scenario_name,
config.HORIZON_STEPS[horizon],
weather_override,
holiday_override,
)
def _validate_sensor(sensor: str) -> None:
if sensor not in config.USABLE_SENSORS:
raise HTTPException(404, f"unknown sensor: {sensor}")
def _validate_scenario(scenario: str) -> None:
if scenario not in config.SCENARIOS:
raise HTTPException(400, f"unknown scenario: {scenario}")
def _validate_horizon(horizon: str) -> None:
if horizon not in config.HORIZON_STEPS:
raise HTTPException(400, f"unknown horizon: {horizon}")
async def _parse_train_request(request: Request) -> tuple[str, Path]:
content_type = request.headers.get("content-type", "")
model = "small"
path: Path | None = None
if content_type.startswith("multipart/form-data"):
form = await request.form()
model = str(form.get("model") or "small")
upload = form.get("file")
path_field = form.get("path")
if upload is not None and hasattr(upload, "read"):
upload_dir = config.upload_dir()
upload_dir.mkdir(parents=True, exist_ok=True)
suffix = Path(getattr(upload, "filename", "") or "upload.csv").suffix or ".csv"
destination = upload_dir / f"{uuid.uuid4().hex}{suffix}"
size = 0
with destination.open("wb") as handle:
while chunk := await upload.read(1024 * 1024):
size += len(chunk)
if size > MAX_UPLOAD_BYTES:
destination.unlink(missing_ok=True)
raise HTTPException(413, "uploaded CSV is too large")
handle.write(chunk)
path = destination
elif path_field:
path = Path(str(path_field))
else:
raise HTTPException(400, "multipart request needs a file field")
elif content_type.startswith("application/json"):
payload = await request.json()
if not isinstance(payload, dict):
raise HTTPException(400, "JSON body must be an object")
model = str(payload.get("model") or "small")
if not payload.get("path"):
raise HTTPException(400, "JSON request needs path")
path = Path(str(payload["path"]))
else:
raise HTTPException(400, "use multipart/form-data or application/json")
if model not in {"small", "forecast"}:
raise HTTPException(400, f"unknown model: {model}")
if path is None or not path.exists() or not path.is_file():
raise HTTPException(400, f"CSV not found: {path}")
if path.stat().st_size > MAX_UPLOAD_BYTES:
raise HTTPException(413, "uploaded CSV is too large")
return model, path
def _run_job(runtime: Runtime, job_id: str, model: str, path: Path) -> None:
with runtime.jobs_lock:
job = runtime.jobs[job_id]
job["status"] = "running"
job["started_at"] = _utc_now()
try:
if model == "small":
bundle, payload = train_small_from_csv(path)
job_metrics = payload["metrics"]
else:
bundle, payload, _ = train_forecast_from_csv(path)
job_metrics = {
**payload["metrics"],
"ablation": payload.get("ablation", []),
}
version = payload["model_version"]
models = config.model_dir()
models.mkdir(parents=True, exist_ok=True)
latest = models / ("small_latest.joblib" if model == "small" else "forecast_latest.joblib")
joblib.dump(bundle, latest)
joblib.dump(bundle, models / f"{job_id}-{model}.joblib")
with runtime.jobs_lock:
if model == "small":
runtime.small_bundle = bundle
runtime.small_metrics = payload
else:
runtime.bundle = bundle
runtime.metrics = payload
runtime.model_source = "retrained"
job.update(
status="succeeded",
metrics=job_metrics,
model_version=version,
finished_at=_utc_now(),
error=None,
)
log.info("training job %s (%s) succeeded: %s", job_id, model, version)
except Exception as exc: # surfaced verbatim in GET /train/{job_id}
log.exception("training job %s (%s) failed", job_id, model)
with runtime.jobs_lock:
job.update(
status="failed",
metrics=None,
finished_at=_utc_now(),
error=str(exc) or exc.__class__.__name__,
)
# --------------------------------------------------------------------------- app
def create_app() -> FastAPI:
@asynccontextmanager
async def lifespan(app: FastAPI):
runtime = Runtime()
_load_runtime(runtime)
app.state.runtime = runtime
yield
app = FastAPI(
title="PulseCast ML",
version="0.1.0",
description="Weather-aware footfall forecast, scenarios, and small-model retraining.",
lifespan=lifespan,
)
app.add_middleware(
CORSMiddleware,
allow_origins=["https://pulse.noelsabia.dev", "http://localhost:3000"],
allow_methods=["GET", "POST", "OPTIONS"],
allow_headers=["Content-Type", "Authorization"],
)
@app.exception_handler(HTTPException)
async def http_error(_: Request, exc: HTTPException) -> JSONResponse:
codes = {
400: "bad_request",
401: "unauthorized",
404: "not_found",
409: "conflict",
413: "payload_too_large",
503: "unavailable",
}
return JSONResponse(
status_code=exc.status_code,
content={
"error": {
"code": codes.get(exc.status_code, "error"),
"message": str(exc.detail),
}
},
)
@app.exception_handler(RequestValidationError)
async def validation_error(_: Request, exc: RequestValidationError) -> JSONResponse:
message = "; ".join(
f"{'.'.join(str(part) for part in error['loc'])}: {error['msg']}"
for error in exc.errors()
)
return JSONResponse(
status_code=400,
content={"error": {"code": "bad_request", "message": message or "invalid request"}},
)
@app.exception_handler(Exception)
async def unhandled_error(_: Request, exc: Exception) -> JSONResponse:
log.exception("unhandled error")
return JSONResponse(
status_code=500,
content={"error": {"code": "internal", "message": "internal server error"}},
)
def runtime() -> Runtime:
return app.state.runtime
@app.get("/healthz")
def healthz() -> dict:
return {"ok": True}
@app.get("/readyz")
def readyz() -> JSONResponse:
ready = runtime().bundle is not None
return JSONResponse(
status_code=200 if ready else 503,
content={"ok": ready, "model": ready},
)
@app.get("/meta")
def meta() -> dict:
state = runtime()
if state.metrics is None:
raise HTTPException(503, "model metadata is not loaded")
payload = dict(state.metrics)
payload["model_source"] = state.model_source
payload["weather_updated_at"] = (
(state.weather_meta or {}).get("fetched_at") or state.metrics.get("context_version")
)
if state.small_metrics:
payload["small_model"] = {
"model_version": state.small_metrics.get("model_version"),
"trained_at": state.small_metrics.get("trained_at"),
"features": state.small_metrics.get("features"),
"metrics": state.small_metrics.get("metrics"),
}
return payload
@app.post("/predict")
def predict_endpoint(body: PredictRequest) -> dict:
state = runtime()
if state.bundle is None:
raise HTTPException(503, "model is not loaded")
_validate_sensor(body.sensor)
_validate_horizon(body.horizon)
_validate_scenario(body.scenario)
forecast = _forecast(
state, body.sensor, body.horizon, body.scenario, body.weather, body.is_holiday
)
bundle_meta = state.bundle["metadata"]
return {
"sensor": body.sensor,
"horizon": body.horizon,
"scenario": body.scenario,
"points": forecast,
"data_version": bundle_meta.get("data_version"),
"model_version": bundle_meta.get("model_version"),
"context_version": bundle_meta.get("context_version"),
}
@app.post("/simulate")
def simulate_endpoint(body: SimulateRequest) -> dict:
state = runtime()
if state.bundle is None:
raise HTTPException(503, "model is not loaded")
_validate_scenario(body.scenario)
if body.sensors == "all":
sensors = sorted(config.USABLE_SENSORS)
elif isinstance(body.sensors, list) and body.sensors:
for sensor in body.sensors:
_validate_sensor(sensor)
sensors = list(body.sensors)
else:
raise HTTPException(400, "sensors must be 'all' or a non-empty list")
scenario = config.SCENARIOS[body.scenario]
# An explicit weather body overrides the scenario preset; it is not an
# additional delta on top of it.
temp_delta = (
float(body.weather.temp_delta_c)
if body.weather is not None
else scenario["temp_delta_c"]
)
precip = (
body.weather.precip_mm
if body.weather is not None and body.weather.precip_mm is not None
else scenario["precip_mm"]
)
now = pd.Timestamp.now(tz="UTC")
start = now.ceil("h")
first = now.ceil("15min")
forecasts: dict[str, dict[str, list[dict]]] = {}
for sensor in sensors:
calendar = future_calendar(start, config.HORIZON_STEPS["7d"])
mask = _scenario_weather_mask(calendar, scenario)
frame = future_frame(
state.bundle,
sensor,
start,
config.HORIZON_STEPS["7d"],
_override_weather(state, None),
temp_delta=temp_delta,
precip_mm=precip,
weather_mask=mask,
holiday_override=body.is_holiday,
)
pred = predict(state.bundle, frame) * event_multiplier(frame, scenario)
pts = points(frame, pred, state.bundle, sensor)
forecasts[sensor] = {
"15m": _quarter_hour_points(
state,
sensor,
body.scenario,
1,
None,
body.is_holiday,
first=first,
temp_delta_override=temp_delta,
precip_override=precip,
),
"1h": _quarter_hour_points(
state,
sensor,
body.scenario,
4,
None,
body.is_holiday,
first=first,
temp_delta_override=temp_delta,
precip_override=precip,
),
"48h": pts[: config.HORIZON_STEPS["48h"]],
"7d": pts,
}
bundle_meta = state.bundle["metadata"]
return {
"scenario": body.scenario,
"generated_at": _utc_now(),
"forecasts": forecasts,
"data_version": bundle_meta.get("data_version"),
"model_version": bundle_meta.get("model_version"),
"context_version": bundle_meta.get("context_version"),
}
@app.post("/train", status_code=202)
async def train_endpoint(
request: Request, authorization: str | None = Header(default=None)
) -> dict:
_require_token(authorization)
state = runtime()
model, path = await _parse_train_request(request)
with state.schedule_lock:
with state.jobs_lock:
if any(job["status"] in ACTIVE_STATUSES for job in state.jobs.values()):
raise HTTPException(409, "a training job is already running")
job_id = uuid.uuid4().hex
submitted_at = _utc_now()
state.jobs[job_id] = {
"job_id": job_id,
"status": "queued",
"model": model,
"submitted_at": submitted_at,
"started_at": None,
"finished_at": None,
"metrics": None,
"model_version": None,
"error": None,
}
thread = threading.Thread(
target=_run_job,
args=(state, job_id, model, path),
name=f"train-{job_id[:8]}",
daemon=True,
)
thread.start()
return {
"job_id": job_id,
"status": "queued",
"model": model,
"submitted_at": submitted_at,
}
@app.get("/train/{job_id}")
def train_status(job_id: str, authorization: str | None = Header(default=None)) -> dict:
_require_token(authorization)
with runtime().jobs_lock:
job = runtime().jobs.get(job_id)
if job is None:
raise HTTPException(404, f"unknown job: {job_id}")
return dict(job)
return app
app = create_app()
+305
View File
@@ -0,0 +1,305 @@
"""Small "busy hour" classifier used by the training demo.
Target: ``busy`` = hourly footfall above that sensor's p75. Features are the
contract's six cheap signals: hour, weekday, weekend, temperature, rain,
holiday. Logistic regression on a time-based split; retrains in well under 5 s
on the full challenge CSV and accepts simpler uploaded CSVs too.
"""
from __future__ import annotations
import logging
from datetime import UTC, datetime
from pathlib import Path
import numpy as np
import pandas as pd
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import (
accuracy_score,
f1_score,
precision_score,
recall_score,
roc_auc_score,
)
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler
from . import config
log = logging.getLogger(__name__)
TIME_COLUMNS = ("time", "timestamp", "datetime", "date", "hour")
VALUE_COLUMNS = ("value", "density", "footfall", "count", "pax", "visitors")
SENSOR_COLUMNS = ("sensor", "sensor_id", "site", "location", "name", "id")
TEMP_COLUMNS = ("temp_c", "temperature", "temperature_2m")
PRECIP_COLUMNS = ("precip_mm", "precipitation", "rain", "rain_mm")
LOGIT_PARAMS = {
"C": 1.0,
"max_iter": 1000,
"class_weight": "balanced",
"solver": "lbfgs",
"random_state": 42,
}
def _utc_now() -> str:
return datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ")
def _iso(ts) -> str:
return pd.Timestamp(ts).tz_convert("UTC").strftime("%Y-%m-%dT%H:%M:%SZ")
def _first_column(columns: list[str], candidates: tuple[str, ...]) -> str | None:
for name in candidates:
if name in columns:
return name
return None
def _sensor_column(columns: list[str]) -> str | None:
# ``name`` is the human/logical sensor label in the challenge CSV; prefer it
# over the physical device id so the density-01 merge is automatic.
if "name" in columns:
return "name"
return _first_column(columns, SENSOR_COLUMNS)
def _attach_weather(hourly: pd.DataFrame, archive: pd.DataFrame | None) -> pd.DataFrame:
frame = hourly.copy()
for column in ("temp_c", "precip_mm"):
if column not in frame.columns:
frame[column] = np.nan
frame[column] = pd.to_numeric(frame[column], errors="coerce")
if archive is not None and not archive.empty and "time" in archive.columns:
weather = archive[["time", "temp_c", "precip_mm"]].rename(columns={"time": "t"})
weather["t"] = pd.to_datetime(weather["t"], utc=True)
weather = weather.sort_values("t").drop_duplicates("t", keep="last")
frame = pd.merge_asof(
frame.sort_values("t"),
weather,
on="t",
direction="nearest",
tolerance=pd.Timedelta("3h"),
suffixes=("", "_archive"),
)
for column in ("temp_c", "precip_mm"):
archive_column = f"{column}_archive"
if archive_column in frame.columns:
frame[column] = frame[column].where(frame[column].notna(), frame[archive_column])
frame = frame.drop(columns=[archive_column])
# Honest fallback: no weather available for the uploaded window. Constant
# values keep the other features useful; the job metrics show the effect.
frame["temp_c"] = frame["temp_c"].fillna(14.0)
frame["precip_mm"] = frame["precip_mm"].fillna(0.0)
return frame
def _add_calendar(frame: pd.DataFrame) -> pd.DataFrame:
local = frame["t"].dt.tz_convert(config.TIMEZONE)
out = frame.copy()
out["local_date"] = local.dt.strftime("%Y-%m-%d")
out["hour"] = local.dt.hour.astype("int8")
out["weekday"] = local.dt.day_of_week.astype("int8")
out["is_weekend"] = (out["weekday"] >= 5).astype("int8")
out["is_holiday"] = out["local_date"].map(config.HOLIDAYS).notna().astype("int8")
return out
def load_hourly_csv(path: Path | str) -> pd.DataFrame:
"""Read a raw challenge CSV or a simple upload into hourly sensor means.
Accepted time/value/sensor column names are intentionally liberal. Raw PAX
files are detected by their ``type`` column and filtered to densityData.
"""
path = Path(path)
frame = pd.read_csv(path, low_memory=False)
frame.columns = [str(column).strip().lower() for column in frame.columns]
if "type" in frame.columns:
frame = frame[
frame["type"].astype(str).str.lower().isin({"densitydata", "density"})
].copy()
time_column = _first_column(list(frame.columns), TIME_COLUMNS)
value_column = _first_column(list(frame.columns), VALUE_COLUMNS)
if time_column is None or value_column is None:
raise ValueError(
f"CSV needs a time column ({', '.join(TIME_COLUMNS)}) and a value column "
f"({', '.join(VALUE_COLUMNS)}); got {list(frame.columns)}"
)
frame["_t"] = pd.to_datetime(frame[time_column], utc=True, errors="coerce")
frame["_v"] = pd.to_numeric(frame[value_column], errors="coerce")
frame = frame[frame["_t"].notna() & frame["_v"].notna()].copy()
if frame.empty:
raise ValueError("CSV has no valid time/value rows")
sensor_column = _sensor_column(list(frame.columns))
if sensor_column is None:
frame["_sensor"] = "upload"
else:
frame["_sensor"] = frame[sensor_column].astype(str).str.strip()
frame.loc[frame["_sensor"].isin({"", "nan", "None"}), "_sensor"] = "upload"
frame["_t"] = frame["_t"].dt.floor("h")
aggregations: dict[str, tuple[str, str]] = {"v": ("_v", "mean")}
for column in TEMP_COLUMNS:
if column in frame.columns:
frame[column] = pd.to_numeric(frame[column], errors="coerce")
aggregations["temp_c"] = (column, "mean")
break
for column in PRECIP_COLUMNS:
if column in frame.columns:
frame[column] = pd.to_numeric(frame[column], errors="coerce")
aggregations["precip_mm"] = (column, "mean")
break
hourly = (
frame.groupby(["_sensor", "_t"], observed=True)
.agg(**aggregations)
.reset_index()
.rename(columns={"_sensor": "sensor", "_t": "t"})
)
hourly = hourly[~hourly["sensor"].isin(config.EXCLUDED_SENSORS)].copy()
if hourly.empty:
raise ValueError("CSV contains only excluded sensors")
hourly["v"] = hourly["v"].astype(float)
return hourly.sort_values(["sensor", "t"]).reset_index(drop=True)
def _load_archive(archive: pd.DataFrame | None) -> pd.DataFrame | None:
if archive is not None:
return archive
for candidate in (
config.weather_dir() / "weather_archive.parquet",
config.cache_dir() / "weather_archive.parquet",
):
if candidate.exists():
try:
return pd.read_parquet(candidate)
except Exception: # noqa: BLE001 - corrupt cache must not block a job
log.warning("could not read weather archive at %s", candidate)
return None
def _metrics(bundle_metrics: dict) -> dict:
out = {}
for key, value in bundle_metrics.items():
if isinstance(value, float):
out[key] = None if not np.isfinite(value) else round(value, 3)
elif isinstance(value, dict):
out[key] = {k: round(float(v), 1) for k, v in value.items()}
else:
out[key] = value
return out
def train_small_from_csv(
path: Path | str,
*,
archive: pd.DataFrame | None = None,
start: pd.Timestamp | str | None = None,
) -> tuple[dict, dict]:
"""Train the busy classifier and return (bundle, metrics payload)."""
hourly = load_hourly_csv(path)
if start is not None:
hourly = hourly[hourly["t"] >= pd.Timestamp(start)].copy()
if hourly.empty:
raise ValueError("no rows after the requested training start")
hourly = _add_calendar(_attach_weather(hourly, _load_archive(archive)))
times = np.sort(hourly["t"].unique())
if len(times) < 4:
raise ValueError("need at least four hourly timestamps to train")
cut = times[int(len(times) * 0.8)]
train = hourly[hourly["t"] < cut].copy()
test = hourly[hourly["t"] >= cut].copy()
if train.empty or test.empty:
split = max(1, int(len(hourly) * 0.8))
train = hourly.iloc[:split].copy()
test = hourly.iloc[split:].copy()
# The busy label is defined by the sensor's p75 in the TRAINING window.
# Using the full upload would leak the holdout distribution into the label.
thresholds = train.groupby("sensor", observed=True)["v"].quantile(0.75).to_dict()
global_threshold = float(train["v"].quantile(0.75))
if not np.isfinite(global_threshold):
global_threshold = 0.0
for frame in (train, test):
frame["threshold"] = (
frame["sensor"].map(thresholds).astype(float).fillna(global_threshold)
)
frame["busy"] = (frame["v"] > frame["threshold"]).astype("int8")
if train["busy"].nunique() < 2:
raise ValueError("training window contains only one busy class")
model = Pipeline(
[
("scale", StandardScaler()),
("clf", LogisticRegression(**LOGIT_PARAMS)),
]
)
model.fit(train[config.SMALL_FEATURES], train["busy"])
proba = model.predict_proba(test[config.SMALL_FEATURES])[:, 1]
predicted = (proba >= 0.5).astype("int8")
truth = test["busy"].to_numpy(dtype=int)
try:
roc_auc = float(roc_auc_score(truth, proba)) if len(np.unique(truth)) > 1 else None
except ValueError:
roc_auc = None
trained_at = _utc_now()
model_version = f"logit-{trained_at[:10].replace('-', '')}-{trained_at[11:19].replace(':', '')}"
metrics_payload = {
"model_version": model_version,
"trained_at": trained_at,
"features": list(config.SMALL_FEATURES),
"train_from": _iso(train["t"].min()),
"train_to": _iso(train["t"].max()),
"test_from": _iso(test["t"].min()),
"test_to": _iso(test["t"].max()),
"thresholds": {str(k): round(float(v), 2) for k, v in thresholds.items()},
"metrics": _metrics(
{
"accuracy": float(accuracy_score(truth, predicted)),
"precision": float(precision_score(truth, predicted, zero_division=0)),
"recall": float(recall_score(truth, predicted, zero_division=0)),
"f1": float(f1_score(truth, predicted, zero_division=0)),
"roc_auc": roc_auc,
"n_train": len(train),
"n_test": len(test),
"positive_rate": float(truth.mean()),
"sensors": int(hourly["sensor"].nunique()),
}
),
}
bundle = {
"model": model,
"features": list(config.SMALL_FEATURES),
"thresholds": {str(k): float(v) for k, v in thresholds.items()},
"global_threshold": global_threshold,
"trained_at": trained_at,
"model_version": model_version,
"metadata": metrics_payload,
}
log.info(
"small model %s: accuracy %.3f f1 %.3f roc_auc %s (%s train / %s test rows)",
model_version,
metrics_payload["metrics"]["accuracy"],
metrics_payload["metrics"]["f1"],
metrics_payload["metrics"]["roc_auc"],
len(train),
len(test),
)
return bundle, metrics_payload
def busy_probability(bundle: dict, frame: pd.DataFrame) -> np.ndarray:
"""Probability of a busy hour for a feature frame (used by demos/tests)."""
return bundle["model"].predict_proba(frame[list(bundle["features"])])[:, 1]
+120 -78
View File
@@ -4,14 +4,14 @@ from __future__ import annotations
import json
import logging
from datetime import datetime, timezone
from datetime import UTC, datetime
import joblib
import numpy as np
import pandas as pd
from . import config
from .features import seasonal_lookup
from .features import future_calendar, seasonal_lookup
from .model import (
event_multiplier,
fit,
@@ -23,6 +23,7 @@ from .model import (
weather_climatology,
)
from .pipeline import run_data
from .small import train_small_from_csv
from .weather import load_cached_weather
log = logging.getLogger(__name__)
@@ -57,6 +58,14 @@ MODEL_SPECS = [
]
def _utc_now() -> str:
return datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ")
def _iso(ts) -> str:
return pd.Timestamp(ts).tz_convert("UTC").strftime("%Y-%m-%dT%H:%M:%SZ")
def _load_inputs() -> tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame, pd.DataFrame, dict, dict]:
cache = config.cache_dir()
if not (cache / "hourly_features.parquet").exists():
@@ -64,7 +73,13 @@ def _load_inputs() -> tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame, pd.DataFra
run_data()
hourly = pd.read_parquet(cache / "hourly.parquet")
features = pd.read_parquet(cache / "hourly_features.parquet")
archive, forecast, meta = load_cached_weather()
try:
archive, forecast, meta = load_cached_weather()
except FileNotFoundError:
# The Docker image only carries the baked artifacts, not data/cache.
archive = pd.read_parquet(config.weather_dir() / "weather_archive.parquet")
forecast = pd.read_parquet(config.weather_dir() / "weather_forecast.parquet")
meta = json.loads((config.weather_dir() / "weather_meta.json").read_text())
audit = json.loads((config.artifacts_dir() / "data_audit.json").read_text())
return hourly, features, archive, forecast, meta, audit
@@ -80,12 +95,14 @@ def _round_row(row: dict) -> dict:
}
def run_train() -> dict:
logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s")
artifacts = config.artifacts_dir()
artifacts.mkdir(parents=True, exist_ok=True)
hourly, features, archive, forecast, meta, audit = _load_inputs()
def train_bundle(
hourly: pd.DataFrame,
features: pd.DataFrame,
archive: pd.DataFrame,
meta: dict,
audit: dict,
) -> tuple[dict, dict, list[dict]]:
"""Evaluate on the time holdout, refit production, return (bundle, metrics, rows)."""
data_end = hourly["t"].max()
test_start = data_end.floor("h") - pd.Timedelta(days=config.TEST_DAYS)
train = features[features["t"] < test_start].copy()
@@ -93,18 +110,21 @@ def run_train() -> dict:
eval_frame = test[test["lag_week"].notna() & test["y"].notna()].copy()
if eval_frame.empty:
raise RuntimeError("holdout has no rows with a seasonal-naive baseline")
if len(train) < 100:
raise RuntimeError("not enough training rows for a 28-day holdout")
log.info(
"split: train=%s rows (%s..%s), test=%s rows, eval=%s rows (%s..%s)",
len(train),
train["t"].min(),
train["t"].max(),
_iso(train["t"].min()),
_iso(train["t"].max()),
len(test),
len(eval_frame),
eval_frame["t"].min(),
eval_frame["t"].max(),
_iso(eval_frame["t"].min()),
_iso(eval_frame["t"].max()),
)
results = []
results: list[dict] = []
preds: dict[str, np.ndarray] = {}
for name, cols, description in MODEL_SPECS:
if name == "seasonal_naive":
@@ -118,22 +138,20 @@ def run_train() -> dict:
results.append({"name": name, "description": description, **m})
log.info("%-16s MAE %.2f WAPE %.1f%% R2 %.3f", name, m["mae"], m["wape"], m["r2"])
by_name = {r["name"]: r for r in results}
by_name = {row["name"]: row for row in results}
baseline = by_name["seasonal_naive"]
headline = by_name["week_weather"]
# Production model: refit on all observed rows after the honest holdout.
production = fit(features, config.FEATURE_WEEK_WEATHER)
residual = residual_quantiles(eval_frame, preds["week_weather"])
sensor_mean = (
hourly.groupby("sensor", observed=True)["v"].mean().round(4).to_dict()
)
trained_at = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
sensor_mean = hourly.groupby("sensor", observed=True)["v"].mean().round(4).to_dict()
trained_at = _utc_now()
model_version = f"histgb-{trained_at[:10]}"
data_version = pd.Timestamp(audit["time_range"]["to"]).floor("min").strftime(
"%Y-%m-%dT%H:%M:%SZ"
)
context_version = meta["fetched_at"]
context_version = meta.get("fetched_at") or trained_at
bundle = {
"model": production,
@@ -147,16 +165,13 @@ def run_train() -> dict:
"data_version": data_version,
"context_version": context_version,
"trained_at": trained_at,
"train_from": train["t"].min().strftime("%Y-%m-%dT%H:%M:%SZ"),
"train_to": features["t"].max().strftime("%Y-%m-%dT%H:%M:%SZ"),
"train_from": _iso(train["t"].min()),
"train_to": _iso(features["t"].max()),
},
}
joblib.dump(bundle, artifacts / "model.joblib")
holiday_effect = {
"mae_delta": round(
by_name["day_holiday"]["mae"] - by_name["day_no_weather"]["mae"], 2
),
"mae_delta": round(by_name["day_holiday"]["mae"] - by_name["day_no_weather"]["mae"], 2),
"note": (
"Holiday flag alone changes MAE by this amount on the holdout; "
"weather is the main driver, holidays are context."
@@ -168,10 +183,10 @@ def run_train() -> dict:
"model_version": model_version,
"context_version": context_version,
"coverage": {
"sensors": int(audit["usable_sensors"]),
"sensors": audit["usable_sensors"],
"from": config.MULTI_SENSOR_START,
"to": data_version,
"cadence_min": int(round(audit["cadence_median_min"])),
"cadence_min": round(audit["cadence_median_min"]),
},
"metrics": {
"model": "weather-7d",
@@ -187,13 +202,71 @@ def run_train() -> dict:
"trained_at": trained_at,
"train_from": bundle["metadata"]["train_from"],
"train_to": bundle["metadata"]["train_to"],
"test_from": test_start.strftime("%Y-%m-%dT%H:%M:%SZ"),
"test_to": data_end.strftime("%Y-%m-%dT%H:%M:%SZ"),
"test_from": _iso(test_start),
"test_to": _iso(data_end),
"features": list(config.FEATURE_WEEK_WEATHER),
"holiday_effect": holiday_effect,
"ablation": [_round_row(r) for r in results],
"ablation": [_round_row(row) for row in results],
"limits": list(config.LIMITS),
}
return bundle, metrics_payload, results
def _scenario_mask(calendar: pd.DataFrame, scenario: dict) -> np.ndarray | None:
event = scenario.get("event")
if not event:
return None
mask = calendar["weekday"].isin(event.get("weekdays", [])).to_numpy(dtype=bool, copy=True)
hours = event.get("hours")
if hours is not None:
mask &= calendar["hour"].isin(hours).to_numpy(dtype=bool, copy=True)
return mask
def _export_forecasts(bundle: dict, forecast: pd.DataFrame, start: pd.Timestamp) -> dict:
periods = config.HORIZON_STEPS["7d"]
sensors = sorted(config.USABLE_SENSORS)
out: dict[str, dict[str, dict[str, list[dict]]]] = {
sensor: {"48h": {}, "7d": {}} for sensor in sensors
}
for name, scenario in config.SCENARIOS.items():
for sensor in sensors:
calendar = future_calendar(start, periods)
mask = _scenario_mask(calendar, scenario)
frame = future_frame(
bundle=bundle,
sensor=sensor,
start=start,
periods=periods,
weather_forecast=forecast,
temp_delta=scenario["temp_delta_c"],
precip_mm=scenario["precip_mm"],
weather_mask=mask,
)
pred = predict(bundle, frame) * event_multiplier(frame, scenario)
pts = points(frame, pred, bundle, sensor)
out[sensor]["7d"][name] = pts
out[sensor]["48h"][name] = pts[: config.HORIZON_STEPS["48h"]]
return {
"data_version": bundle["metadata"]["data_version"],
"model_version": bundle["metadata"]["model_version"],
"context_version": bundle["metadata"]["context_version"],
"generated_at": _utc_now(),
"forecast_start": _iso(start),
"forecasts": out,
}
def run_train() -> dict:
logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s")
artifacts = config.artifacts_dir()
artifacts.mkdir(parents=True, exist_ok=True)
hourly, features, archive, forecast, meta, audit = _load_inputs()
bundle, metrics_payload, _ = train_bundle(hourly, features, archive, meta, audit)
joblib.dump(bundle, artifacts / "model.joblib")
(artifacts / "metrics.json").write_text(
json.dumps(metrics_payload, indent=2, ensure_ascii=False) + "\n"
)
@@ -203,61 +276,30 @@ def run_train() -> dict:
(artifacts / "forecasts.json").write_text(
json.dumps(forecasts, ensure_ascii=False) + "\n"
)
# Bake the small classifier too, so /train has a working before/after demo
# even before an upload and the image contains the full artifact set.
small_bundle, small_metrics = train_small_from_csv(
config.raw_csv_path(),
archive=archive,
start=pd.Timestamp(config.MULTI_SENSOR_START),
)
joblib.dump(small_bundle, artifacts / "small_model.joblib")
(artifacts / "small_metrics.json").write_text(
json.dumps(small_metrics, indent=2, ensure_ascii=False) + "\n"
)
log.info(
"wrote metrics.json, forecasts.json (%s..%s) and model.joblib",
"wrote metrics.json, forecasts.json (%s..%s), model.joblib, small_model.joblib",
start,
start + pd.Timedelta(hours=167),
)
return {
"headline": metrics_payload["metrics"],
"ablation": metrics_payload["ablation"],
"model_version": model_version,
"forecast_start": start.strftime("%Y-%m-%dT%H:%M:%SZ"),
}
def _export_forecasts(bundle: dict, forecast: pd.DataFrame, start: pd.Timestamp) -> dict:
periods = config.HORIZON_STEPS["7d"]
sensors = sorted(config.USABLE_SENSORS)
out: dict[str, dict[str, dict[str, list[dict]]]] = {
sensor: {"48h": {}, "7d": {}} for sensor in sensors
}
base_calendar = None
for name, cfg in config.SCENARIOS.items():
event = cfg.get("event")
for sensor in sensors:
from .features import future_calendar
cal = future_calendar(start, periods)
if base_calendar is None:
base_calendar = cal
mask = (
cal["weekday"].isin(event["weekdays"]).to_numpy()
if event
else None
)
frame = future_frame(
bundle=bundle,
sensor=sensor,
start=start,
periods=periods,
weather_forecast=forecast,
temp_delta=cfg["temp_delta_c"],
precip_mm=cfg["precip_mm"],
weather_mask=mask,
)
pred = predict(bundle, frame) * event_multiplier(frame, cfg)
pts = points(frame, pred, bundle, sensor)
out[sensor]["7d"][name] = pts
out[sensor]["48h"][name] = pts[: config.HORIZON_STEPS["48h"]]
return {
"data_version": bundle["metadata"]["data_version"],
"model_version": bundle["metadata"]["model_version"],
"context_version": bundle["metadata"]["context_version"],
"generated_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
"forecast_start": start.strftime("%Y-%m-%dT%H:%M:%SZ"),
"forecasts": out,
"forecast_start": _iso(start),
"small": small_metrics["metrics"],
}
+5 -5
View File
@@ -10,7 +10,7 @@ from __future__ import annotations
import json
import logging
import time
from datetime import date, datetime, timedelta, timezone
from datetime import UTC, date, datetime, timedelta
from pathlib import Path
import httpx
@@ -40,7 +40,7 @@ def _get_json(url: str, params: dict, *, retries: int = 3) -> dict:
if payload.get("error"):
raise RuntimeError(f"Open-Meteo error: {payload.get('reason', payload)}")
return payload
except Exception as exc: # network / HTTP / payload
except Exception as exc: # noqa: BLE001 - network/HTTP/payload retry boundary
last = exc
if attempt < retries - 1:
time.sleep(2**attempt)
@@ -113,7 +113,7 @@ def fetch_forecast(days: int = 8) -> tuple[pd.DataFrame, dict, dict]:
}
grid = {"latitude": payload.get("latitude"), "longitude": payload.get("longitude")}
meta = {
"fetched_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
"fetched_at": datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ"),
"current": now,
"grid": grid,
"source": "open-meteo",
@@ -158,7 +158,7 @@ def ensure_weather(
if not meta:
meta = {
"fetched_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
"fetched_at": datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ"),
"current": None,
"grid": {"latitude": config.LATITUDE, "longitude": config.LONGITUDE},
"source": "cache",
@@ -181,7 +181,7 @@ def load_cached_weather() -> tuple[pd.DataFrame, pd.DataFrame, dict]:
def refresh_weather() -> None:
archive, forecast, meta = ensure_weather(refresh=True)
archive, forecast, _ = ensure_weather(refresh=True)
log.info("weather refreshed: archive=%s forecast=%s", len(archive), len(forecast))