feat: runtime bootstrap snapshot cache with backoff

This commit is contained in:
Noel Sabia committed 2026-11-14 06:08:53 +01:00
1 parent 08f60a14d1
commit aa6bb23e24
19 files changed
+1400 -384

No files matched your search

+357 -32
View File
@@ -10,18 +10,77 @@ import (
"io"
"net/http"
"net/http/httptest"
"os"
"strings"
"sync"
"testing"
"time"
"pulsecast/api/internal/cache"
"pulsecast/api/internal/config"
"pulsecast/api/internal/ml"
"pulsecast/api/internal/model"
"pulsecast/api/internal/sse"
"pulsecast/api/internal/store"
)
const fixtureDir = "../../testdata/data"
const fixtureBootstrap = "../../testdata/data/bootstrap.json"
type stubFetcher struct {
mu sync.RWMutex
payload model.Bootstrap
err error
refreshFn func(context.Context) (model.Bootstrap, error)
}
func (s *stubFetcher) Bootstrap(ctx context.Context) (model.Bootstrap, error) {
s.mu.RLock()
fn := s.refreshFn
payload, err := s.payload, s.err
s.mu.RUnlock()
if fn != nil {
return fn(ctx)
}
if err != nil {
return model.Bootstrap{}, err
}
return payload, nil
}
func (s *stubFetcher) fail(err error) {
s.mu.Lock()
s.err = err
s.mu.Unlock()
}
func (s *stubFetcher) recover(payload model.Bootstrap) {
s.mu.Lock()
s.payload, s.err = payload, nil
s.mu.Unlock()
}
func loadFixtureBootstrap(t *testing.T) model.Bootstrap {
t.Helper()
raw, err := os.ReadFile(fixtureBootstrap)
if err != nil {
t.Fatalf("read fixture: %v", err)
}
var payload model.Bootstrap
if err := json.Unmarshal(raw, &payload); err != nil {
t.Fatalf("decode fixture: %v", err)
}
return payload
}
func newFixtureCache(t *testing.T) (*cache.Cache, *stubFetcher) {
t.Helper()
payload := loadFixtureBootstrap(t)
fetcher := &stubFetcher{payload: payload}
snapshots := cache.New(fetcher, cache.Options{RefreshInterval: time.Hour, Timeout: time.Second})
if err := snapshots.Refresh(context.Background()); err != nil {
t.Fatalf("prime cache: %v", err)
}
return snapshots, fetcher
}
type stubML struct {
predict func(context.Context, ml.PredictRequest) ([]model.ForecastPoint, error)
@@ -58,20 +117,16 @@ func (s *stubML) TrainStatus(ctx context.Context, jobID string) (int, []byte, er
return s.trainStatus(ctx, jobID)
}
func loadFixtureStore(t *testing.T) *store.Store {
func newTestServer(t *testing.T, client MLClient, mutate func(*config.Config)) (*Server, *httptest.Server) {
t.Helper()
st, err := store.Load(fixtureDir)
if err != nil {
t.Fatalf("load fixtures: %v", err)
}
return st
snapshots, _ := newFixtureCache(t)
return newTestServerWithCache(t, snapshots, client, mutate)
}
func newTestServer(t *testing.T, client MLClient, mutate func(*config.Config)) (*Server, *httptest.Server) {
func newTestServerWithCache(t *testing.T, snapshots *cache.Cache, client MLClient, mutate func(*config.Config)) (*Server, *httptest.Server) {
t.Helper()
cfg := config.Config{
Addr: ":0",
DataDir: fixtureDir,
FastAPIURL: "http://127.0.0.1:1",
Heartbeat: time.Hour,
MLTimeout: 100 * time.Millisecond,
@@ -85,12 +140,47 @@ func newTestServer(t *testing.T, client MLClient, mutate func(*config.Config)) (
t.Cleanup(cancel)
go hub.Run(ctx)
srv := New(cfg, loadFixtureStore(t), client, hub)
srv := New(cfg, snapshots, client, hub)
ts := httptest.NewServer(srv.Handler())
t.Cleanup(ts.Close)
return srv, ts
}
func forecastPoints(n int) []model.ForecastPoint {
start := time.Date(2026, 11, 16, 9, 0, 0, 0, time.UTC)
points := make([]model.ForecastPoint, n)
for i := range points {
points[i] = model.ForecastPoint{
T: start.Add(time.Duration(i) * time.Hour),
V: float64(10 + i),
Lo: float64(5 + i),
Hi: float64(15 + i),
}
}
return points
}
// weekForecast builds seven complete store-hour days (10:00-20:00 Berlin),
// which is what the staffing planner consumes.
func weekForecast(t *testing.T) []model.ForecastPoint {
t.Helper()
loc, err := time.LoadLocation(model.StoreHours.TZ)
if err != nil {
t.Fatal(err)
}
start := time.Date(2026, 11, 16, model.StoreHours.Open, 0, 0, 0, loc)
var points []model.ForecastPoint
for day := 0; day < 7; day++ {
for hour := 0; hour < model.StoreHours.Close-model.StoreHours.Open; hour++ {
points = append(points, model.ForecastPoint{
T: start.AddDate(0, 0, day).Add(time.Duration(hour) * time.Hour).UTC(),
V: 25,
})
}
}
return points
}
func getJSON(t *testing.T, url string, dst any) *http.Response {
t.Helper()
resp, err := http.Get(url)
@@ -125,13 +215,11 @@ func TestHealthAndReady(t *testing.T) {
}
}
func TestReadyzWithoutData(t *testing.T) {
cfg := config.Config{Addr: ":0", DataDir: t.TempDir(), FastAPIURL: "http://127.0.0.1:1", Heartbeat: time.Hour, MLTimeout: 50 * time.Millisecond}
st, err := store.Load(cfg.DataDir)
if err != nil {
t.Logf("expected load error: %v", err)
}
srv := New(cfg, st, &stubML{}, sse.NewHub(time.Hour))
func TestReadyzWithoutSnapshot(t *testing.T) {
cfg := config.Config{Addr: ":0", FastAPIURL: "http://127.0.0.1:1", Heartbeat: time.Hour, MLTimeout: 50 * time.Millisecond}
fetcher := &stubFetcher{err: errors.New("fastapi unavailable")}
snapshots := cache.New(fetcher, cache.Options{RefreshInterval: time.Hour, Timeout: time.Second})
srv := New(cfg, snapshots, &stubML{}, sse.NewHub(time.Hour))
ts := httptest.NewServer(srv.Handler())
defer ts.Close()
@@ -143,6 +231,70 @@ func TestReadyzWithoutData(t *testing.T) {
if ready["ok"] || ready["data"] || ready["model"] {
t.Fatalf("readyz body=%v, want all false", ready)
}
// One failed bootstrap attempt must not make the cache ready.
if err := snapshots.Refresh(context.Background()); err == nil {
t.Fatal("expected the bootstrap refresh to fail")
}
resp = getJSON(t, ts.URL+"/api/readyz", &ready)
if resp.StatusCode != http.StatusServiceUnavailable || ready["ok"] {
t.Fatalf("readyz after failed refresh: status=%d body=%v", resp.StatusCode, ready)
}
}
func TestReadyzSurvivesFastAPIFailureAfterFirstSnapshot(t *testing.T) {
snapshots, fetcher := newFixtureCache(t)
cfg := config.Config{Addr: ":0", FastAPIURL: "http://127.0.0.1:1", Heartbeat: time.Hour, MLTimeout: 50 * time.Millisecond}
srv := New(cfg, snapshots, &stubML{}, sse.NewHub(time.Hour))
ts := httptest.NewServer(srv.Handler())
defer ts.Close()
var ready map[string]bool
resp := getJSON(t, ts.URL+"/api/readyz", &ready)
if resp.StatusCode != http.StatusOK || !ready["ok"] || !ready["data"] || !ready["model"] {
t.Fatalf("readyz status=%d body=%v", resp.StatusCode, ready)
}
fetcher.fail(errors.New("connection refused"))
if err := snapshots.Refresh(context.Background()); err == nil {
t.Fatal("expected the refresh to fail")
}
resp = getJSON(t, ts.URL+"/api/readyz", &ready)
if resp.StatusCode != http.StatusOK || !ready["ok"] || !ready["data"] || !ready["model"] {
t.Fatalf("readyz after FastAPI failure: status=%d body=%v, want 200 with the last snapshot", resp.StatusCode, ready)
}
var meta model.Meta
getJSON(t, ts.URL+"/api/meta", &meta)
if meta.Cache.Failures != 1 || meta.Cache.LastError == nil ||
!strings.Contains(*meta.Cache.LastError, "connection refused") {
t.Fatalf("cache status=%+v", meta.Cache)
}
if meta.Cache.LastSuccessAt == nil || meta.Cache.NextRetryAt == nil {
t.Fatalf("cache timestamps missing: %+v", meta.Cache)
}
// The context endpoint comes from the snapshot; it is now stale.
var ctx model.ContextResponse
getJSON(t, ts.URL+"/api/context", &ctx)
if !ctx.Cached {
t.Error("context cached=false after a failed refresh, want true")
}
// A later success clears the failure state.
fetcher.recover(loadFixtureBootstrap(t))
if err := snapshots.Refresh(context.Background()); err != nil {
t.Fatalf("recovery refresh: %v", err)
}
getJSON(t, ts.URL+"/api/meta", &meta)
if meta.Cache.Failures != 0 || meta.Cache.LastError != nil || meta.Cache.NextRetryAt != nil {
t.Fatalf("cache status after recovery=%+v", meta.Cache)
}
getJSON(t, ts.URL+"/api/context", &ctx)
if ctx.Cached {
t.Error("context cached=true after a successful refresh, want false")
}
}
func TestMetaSensorsAndVersionHeaders(t *testing.T) {
@@ -156,6 +308,10 @@ func TestMetaSensorsAndVersionHeaders(t *testing.T) {
if meta.Coverage.Sensors != 8 || meta.ModelVersion == "" || meta.ContextVersion == "" {
t.Fatalf("meta=%+v", meta)
}
if meta.Cache.Source != "fastapi" || meta.Cache.LastSuccessAt == nil ||
meta.Cache.LastError != nil || meta.Cache.NextRetryAt != nil || meta.Cache.Failures != 0 {
t.Fatalf("meta cache=%+v", meta.Cache)
}
for _, header := range []string{"X-Data-Version", "X-Model-Version", "X-Context-Version"} {
if resp.Header.Get(header) == "" {
t.Errorf("missing %s header", header)
@@ -180,6 +336,58 @@ func TestMetaSensorsAndVersionHeaders(t *testing.T) {
}
}
func TestDataEndpointsFollowSnapshotRefresh(t *testing.T) {
snapshots, fetcher := newFixtureCache(t)
_, ts := newTestServerWithCache(t, snapshots, &stubML{}, nil)
var sensors []model.Sensor
getJSON(t, ts.URL+"/api/sensors", &sensors)
if len(sensors) != 8 {
t.Fatalf("sensors=%d, want 8", len(sensors))
}
updated := loadFixtureBootstrap(t)
updated.DataVersion = "2026-11-14T05:00:00Z"
updated.ContextVersion = "2026-11-14T05:00:00Z"
updated.Sensors = updated.Sensors[:1]
updated.Series = map[string][]model.Point{
updated.Sensors[0].ID: {{T: time.Date(2026, 11, 1, 10, 0, 0, 0, time.UTC), V: 42}},
}
fetcher.recover(updated)
if err := snapshots.Refresh(context.Background()); err != nil {
t.Fatalf("refresh: %v", err)
}
sensors = nil
resp := getJSON(t, ts.URL+"/api/sensors", &sensors)
if len(sensors) != 1 || sensors[0].ID != updated.Sensors[0].ID {
t.Fatalf("sensors after refresh=%+v", sensors)
}
if got := resp.Header.Get("X-Data-Version"); got != updated.DataVersion {
t.Errorf("X-Data-Version=%q, want %q", got, updated.DataVersion)
}
var series struct {
Points []model.Point `json:"points"`
}
getJSON(t, ts.URL+"/api/series?sensor="+updated.Sensors[0].ID+"&step=15m", &series)
if len(series.Points) != 1 || series.Points[0].V != 42 {
t.Fatalf("series after refresh=%+v", series.Points)
}
var ctx model.ContextResponse
getJSON(t, ts.URL+"/api/context", &ctx)
if ctx.UpdatedAt.IsZero() || ctx.Cached {
t.Fatalf("context after refresh=%+v", ctx)
}
var meta model.Meta
getJSON(t, ts.URL+"/api/meta", &meta)
if meta.DataVersion != updated.DataVersion || meta.Cache.Failures != 0 {
t.Fatalf("meta after refresh=%+v", meta)
}
}
func TestSeriesRangeStepAndErrors(t *testing.T) {
_, ts := newTestServer(t, &stubML{}, nil)
@@ -241,37 +449,96 @@ func TestContextEndpoint(t *testing.T) {
if ctx.UpdatedAt.IsZero() {
t.Error("expected updated_at")
}
if ctx.Cached {
t.Error("cached=true while the last snapshot refresh succeeded, want false")
}
}
func TestForecastFallsBackWhenMLIsDown(t *testing.T) {
_, ts := newTestServer(t, &stubML{}, nil)
func TestForecastServesLastGoodWhenMLIsDown(t *testing.T) {
live := forecastPoints(168)
var down bool
var mu sync.Mutex
client := &stubML{predict: func(context.Context, ml.PredictRequest) ([]model.ForecastPoint, error) {
mu.Lock()
defer mu.Unlock()
if down {
return nil, errors.New("connection refused")
}
return live, nil
}}
_, ts := newTestServer(t, client, nil)
var forecast model.ForecastResponse
resp := getJSON(t, ts.URL+"/api/forecast?sensor=hn-bahnhof-01&horizon=7d", &forecast)
if resp.StatusCode != http.StatusOK {
t.Fatalf("forecast status=%d", resp.StatusCode)
t.Fatalf("live forecast status=%d", resp.StatusCode)
}
if !forecast.Cached {
if forecast.Cached || len(forecast.Points) != 168 {
t.Fatalf("live forecast cached=%v points=%d, want false/168", forecast.Cached, len(forecast.Points))
}
mu.Lock()
down = true
mu.Unlock()
var fallback model.ForecastResponse
resp = getJSON(t, ts.URL+"/api/forecast?sensor=hn-bahnhof-01&horizon=7d", &fallback)
if resp.StatusCode != http.StatusOK {
t.Fatalf("fallback status=%d", resp.StatusCode)
}
if !fallback.Cached {
t.Error("cached=false, want true when FastAPI is down")
}
if len(forecast.Points) != 168 {
t.Fatalf("points=%d, want 168", len(forecast.Points))
if len(fallback.Points) != 168 || fallback.Points[0].T != forecast.Points[0].T ||
fallback.Points[0].V != forecast.Points[0].V {
t.Fatalf("fallback points do not match the last good forecast: %+v", fallback.Points[0])
}
if forecast.Scenario != "base" || forecast.DataVersion == "" || forecast.ModelVersion == "" {
t.Fatalf("forecast=%+v", forecast)
if fallback.Scenario != "base" || fallback.DataVersion == "" || fallback.ModelVersion == "" {
t.Fatalf("forecast=%+v", fallback)
}
}
func TestForecastTimeoutFallsBack(t *testing.T) {
func TestForecastWithoutLastGoodFails503(t *testing.T) {
_, ts := newTestServer(t, &stubML{}, nil)
var errBody model.ErrorBody
resp := getJSON(t, ts.URL+"/api/forecast?sensor=hn-bahnhof-01&horizon=7d", &errBody)
if resp.StatusCode != http.StatusServiceUnavailable {
t.Fatalf("status=%d, want 503", resp.StatusCode)
}
if errBody.Error.Code != "forecast_unavailable" {
t.Fatalf("error=%+v", errBody)
}
}
func TestForecastTimeoutServesLastGood(t *testing.T) {
live := forecastPoints(48)
var mu sync.Mutex
slow := false
client := &stubML{predict: func(ctx context.Context, _ ml.PredictRequest) ([]model.ForecastPoint, error) {
<-ctx.Done()
return nil, ctx.Err()
mu.Lock()
block := slow
mu.Unlock()
if block {
<-ctx.Done()
return nil, ctx.Err()
}
return live, nil
}}
_, ts := newTestServer(t, client, func(cfg *config.Config) { cfg.MLTimeout = 20 * time.Millisecond })
var primed model.ForecastResponse
resp := getJSON(t, ts.URL+"/api/forecast?sensor=hn-stadtgalerie-01&horizon=48h&scenario=sunny-saturday-market", &primed)
if resp.StatusCode != http.StatusOK || primed.Cached {
t.Fatalf("prime forecast status=%d cached=%v", resp.StatusCode, primed.Cached)
}
mu.Lock()
slow = true
mu.Unlock()
start := time.Now()
var forecast model.ForecastResponse
resp := getJSON(t, ts.URL+"/api/forecast?sensor=hn-stadtgalerie-01&horizon=48h&scenario=sunny-saturday-market", &forecast)
resp = getJSON(t, ts.URL+"/api/forecast?sensor=hn-stadtgalerie-01&horizon=48h&scenario=sunny-saturday-market", &forecast)
if resp.StatusCode != http.StatusOK {
t.Fatalf("forecast status=%d", resp.StatusCode)
}
@@ -327,7 +594,11 @@ func TestForecastValidation(t *testing.T) {
}
func TestStaffingPlan(t *testing.T) {
_, ts := newTestServer(t, &stubML{}, nil)
points := weekForecast(t)
client := &stubML{predict: func(context.Context, ml.PredictRequest) ([]model.ForecastPoint, error) {
return points, nil
}}
_, ts := newTestServer(t, client, nil)
start := time.Now()
var plan model.StaffingResponse
@@ -350,9 +621,12 @@ func TestStaffingPlan(t *testing.T) {
if plan.Summary.CoveragePct != 100 {
t.Errorf("coverage=%v, want 100", plan.Summary.CoveragePct)
}
if plan.Summary.CostEUR <= 0 || plan.Summary.FlatScheduleCostEUR <= 0 {
if plan.Summary.CostEUR != 2240 || plan.Summary.FlatScheduleCostEUR != 2240 {
t.Errorf("summary=%+v", plan.Summary)
}
if plan.Cached {
t.Error("cached=true, want false for a live plan")
}
if len(plan.Why) == 0 {
t.Error("expected a non-empty why list")
}
@@ -363,6 +637,57 @@ func TestStaffingPlan(t *testing.T) {
}
}
func TestStaffingFallsBackToLastGoodForecast(t *testing.T) {
points := weekForecast(t)
var mu sync.Mutex
down := false
client := &stubML{predict: func(context.Context, ml.PredictRequest) ([]model.ForecastPoint, error) {
mu.Lock()
defer mu.Unlock()
if down {
return nil, errors.New("connection refused")
}
return points, nil
}}
_, ts := newTestServer(t, client, nil)
var live model.StaffingResponse
resp := getJSON(t, ts.URL+"/api/staffing?sensor=hn-bahnhof-01&scenario=base&standard=standard", &live)
if resp.StatusCode != http.StatusOK || live.Cached {
t.Fatalf("live staffing status=%d cached=%v", resp.StatusCode, live.Cached)
}
mu.Lock()
down = true
mu.Unlock()
// Different standard -> different plan key, so the handler must rebuild
// from the last good 7d forecast.
var fallback model.StaffingResponse
resp = getJSON(t, ts.URL+"/api/staffing?sensor=hn-bahnhof-01&scenario=base&standard=premium", &fallback)
if resp.StatusCode != http.StatusOK {
t.Fatalf("fallback staffing status=%d", resp.StatusCode)
}
if !fallback.Cached {
t.Error("cached=false, want true when built from the last good forecast")
}
if len(fallback.Hours) != 70 || len(fallback.Shifts) != 21 {
t.Fatalf("fallback plan hours=%d shifts=%d", len(fallback.Hours), len(fallback.Shifts))
}
}
func TestStaffingUnavailableWithoutForecast(t *testing.T) {
_, ts := newTestServer(t, &stubML{}, nil)
var errBody model.ErrorBody
resp := getJSON(t, ts.URL+"/api/staffing?sensor=hn-bahnhof-01&scenario=base&standard=standard", &errBody)
if resp.StatusCode != http.StatusServiceUnavailable {
t.Fatalf("status=%d, want 503", resp.StatusCode)
}
if errBody.Error.Code != "staffing_unavailable" {
t.Fatalf("error=%+v", errBody)
}
}
func TestStaffingValidation(t *testing.T) {
_, ts := newTestServer(t, &stubML{}, nil)
for _, tc := range []struct {
+25 -10
View File
@@ -14,9 +14,11 @@ func (s *Server) handleHealthz(w http.ResponseWriter, _ *http.Request) {
}
func (s *Server) handleReadyz(w http.ResponseWriter, _ *http.Request) {
ready := s.store != nil && s.store.Ready()
dataOK := s.store != nil && s.store.Len() > 0 && !s.store.Context().UpdatedAt.IsZero()
modelOK := s.store != nil && s.store.Meta().ModelVersion != ""
dataOK, modelOK := false, false
if s.cache != nil {
dataOK, modelOK = s.cache.Readiness()
}
ready := dataOK && modelOK
status := http.StatusOK
if !ready {
status = http.StatusServiceUnavailable
@@ -26,12 +28,16 @@ func (s *Server) handleReadyz(w http.ResponseWriter, _ *http.Request) {
func (s *Server) handleMeta(w http.ResponseWriter, _ *http.Request) {
s.setVersions(w)
writeJSON(w, http.StatusOK, s.store.Meta())
writeJSON(w, http.StatusOK, s.meta())
}
func (s *Server) handleSensors(w http.ResponseWriter, _ *http.Request) {
s.setVersions(w)
writeJSON(w, http.StatusOK, s.store.Sensors())
if s.cache == nil {
writeJSON(w, http.StatusOK, []model.Sensor{})
return
}
writeJSON(w, http.StatusOK, s.cache.Sensors())
}
func (s *Server) handleSeries(w http.ResponseWriter, r *http.Request) {
@@ -41,7 +47,11 @@ func (s *Server) handleSeries(w http.ResponseWriter, r *http.Request) {
writeError(w, http.StatusBadRequest, "bad_request", "sensor is required")
return
}
if _, ok := s.store.Sensor(sensor); !ok {
if s.cache == nil {
writeError(w, http.StatusNotFound, "not_found", "unknown sensor")
return
}
if _, ok := s.cache.Sensor(sensor); !ok {
writeError(w, http.StatusNotFound, "not_found", "unknown sensor")
return
}
@@ -54,7 +64,7 @@ func (s *Server) handleSeries(w http.ResponseWriter, r *http.Request) {
return
}
raw, _ := s.store.Series(sensor)
raw, _ := s.cache.Series(sensor)
if len(raw) == 0 {
s.setVersions(w)
writeJSON(w, http.StatusOK, map[string]any{"sensor": sensor, "step": step, "points": []model.Point{}})
@@ -69,7 +79,7 @@ func (s *Server) handleSeries(w http.ResponseWriter, r *http.Request) {
return
}
to = parsed.UTC()
} else if len(raw) > 0 {
} else {
to = raw[len(raw)-1].T.UTC()
}
@@ -143,12 +153,17 @@ func hourlyMean(points []model.Point) []model.Point {
}
func (s *Server) handleContext(w http.ResponseWriter, _ *http.Request) {
ctx := s.store.Context()
var ctx model.ContextData
cached := true
if s.cache != nil {
ctx = s.cache.Context()
cached = !s.cache.Fresh()
}
s.setVersions(w)
writeJSON(w, http.StatusOK, model.ContextResponse{
Weather: ctx.Weather,
Holidays: ctx.Holidays,
UpdatedAt: ctx.UpdatedAt,
Cached: false,
Cached: cached,
})
}
+76 -30
View File
@@ -20,7 +20,11 @@ func (s *Server) handleForecast(w http.ResponseWriter, r *http.Request) {
writeError(w, http.StatusBadRequest, "bad_request", "sensor is required")
return
}
if _, ok := s.store.Sensor(sensor); !ok {
if s.cache == nil {
writeError(w, http.StatusNotFound, "not_found", "unknown sensor")
return
}
if _, ok := s.cache.Sensor(sensor); !ok {
writeError(w, http.StatusNotFound, "not_found", "unknown sensor")
return
}
@@ -53,11 +57,14 @@ func (s *Server) handleForecast(w http.ResponseWriter, r *http.Request) {
if !ok {
slog.Warn("forecast unavailable", "sensor", sensor, "horizon", horizon, "scenario", scenarioName, "error", err)
writeError(w, http.StatusServiceUnavailable, "forecast_unavailable",
"no live forecast and no baked forecast for this sensor, scenario and horizon")
"no live forecast and no last good forecast for this sensor, scenario and horizon")
return
}
} else {
s.rememberForecast(sensor, horizon, scenarioName, points)
}
meta := s.meta()
s.setVersions(w)
writeJSON(w, http.StatusOK, model.ForecastResponse{
Sensor: sensor,
@@ -65,9 +72,9 @@ func (s *Server) handleForecast(w http.ResponseWriter, r *http.Request) {
Scenario: scenarioName,
Points: points,
Cached: cached,
DataVersion: s.store.Meta().DataVersion,
ModelVersion: s.store.Meta().ModelVersion,
ContextVersion: s.store.Meta().ContextVersion,
DataVersion: meta.DataVersion,
ModelVersion: meta.ModelVersion,
ContextVersion: meta.ContextVersion,
})
}
@@ -78,7 +85,7 @@ func (s *Server) liveForecast(ctx context.Context, sensor, horizon, scenarioName
ctx, cancel := context.WithTimeout(ctx, s.cfg.MLTimeout)
defer cancel()
ctxData := s.store.Context()
ctxData := s.cache.Context()
precip := ctxData.Weather.Now.PrecipMM
if state.Scenario.Weather.PrecipMM > 0 {
precip = state.Scenario.Weather.PrecipMM
@@ -100,8 +107,9 @@ func (s *Server) liveForecast(ctx context.Context, sensor, horizon, scenarioName
return sortedForecast(points), nil
}
// fallbackForecast prefers live /simulate output for the current scenario, then
// the baked scenario grid. It never labels base data as another scenario.
// fallbackForecast prefers the live /simulate grid for the current scenario,
// then the last good /predict response for this exact key. It never labels base
// data as another scenario.
func (s *Server) fallbackForecast(sensor, scenarioName, horizon string) ([]model.ForecastPoint, bool) {
s.mu.RLock()
if s.simFor == scenarioName {
@@ -115,10 +123,31 @@ func (s *Server) fallbackForecast(sensor, scenarioName, horizon string) ([]model
}
s.mu.RUnlock()
if points, ok := s.store.Forecast(sensor, scenarioName, horizon); ok {
return sortedForecast(points), true
return s.lastGoodForecast(sensor, horizon, scenarioName)
}
func forecastKey(sensor, horizon, scenarioName string) string {
return sensor + "|" + horizon + "|" + scenarioName
}
func (s *Server) rememberForecast(sensor, horizon, scenarioName string, points []model.ForecastPoint) {
if len(points) == 0 {
return
}
return nil, false
copied := sortedForecast(points)
s.mu.Lock()
s.lastForecast[forecastKey(sensor, horizon, scenarioName)] = copied
s.mu.Unlock()
}
func (s *Server) lastGoodForecast(sensor, horizon, scenarioName string) ([]model.ForecastPoint, bool) {
s.mu.RLock()
points, ok := s.lastForecast[forecastKey(sensor, horizon, scenarioName)]
s.mu.RUnlock()
if !ok || len(points) == 0 {
return nil, false
}
return sortedForecast(points), true
}
func sortedForecast(points []model.ForecastPoint) []model.ForecastPoint {
@@ -129,8 +158,11 @@ func sortedForecast(points []model.ForecastPoint) []model.ForecastPoint {
}
func (s *Server) holidayOn(t time.Time) (string, bool) {
if s.cache == nil {
return "", false
}
day := t.In(s.loc).Format("2006-01-02")
for _, holiday := range s.store.Context().Holidays {
for _, holiday := range s.cache.Context().Holidays {
if holiday.Date == day {
return holiday.Name, true
}
@@ -145,7 +177,11 @@ func (s *Server) handleStaffing(w http.ResponseWriter, r *http.Request) {
writeError(w, http.StatusBadRequest, "bad_request", "sensor is required")
return
}
if _, ok := s.store.Sensor(sensor); !ok {
if s.cache == nil {
writeError(w, http.StatusNotFound, "not_found", "unknown sensor")
return
}
if _, ok := s.cache.Sensor(sensor); !ok {
writeError(w, http.StatusNotFound, "not_found", "unknown sensor")
return
}
@@ -169,45 +205,49 @@ func (s *Server) handleStaffing(w http.ResponseWriter, r *http.Request) {
}
state := s.scenario.Get()
points, cached, ok := s.staffingForecast(sensor, scenarioName)
if !ok {
writeError(w, http.StatusServiceUnavailable, "staffing_unavailable",
"no 7d forecast for this sensor and scenario")
return
}
meta := s.meta()
s.mu.RLock()
generation := s.simGen
s.mu.RUnlock()
key := fmt.Sprintf("%s|%s|%s|%d|%d", sensor, scenarioName, standard, state.Version, generation)
key := fmt.Sprintf("%s|%s|%s|%s|%d|%d", sensor, scenarioName, standard,
meta.ContextVersion, state.Version, generation)
s.mu.RLock()
plan, hit := s.staffing[key]
s.mu.RUnlock()
if !hit {
built, err := staffing.Build(sensor, scenarioName, standard, points, s.store.Context())
points, cached, ok := s.staffingForecast(r.Context(), sensor, scenarioName, state)
if !ok {
writeError(w, http.StatusServiceUnavailable, "staffing_unavailable",
"no live or last good 7d forecast for this sensor and scenario")
return
}
built, err := staffing.Build(sensor, scenarioName, standard, points, s.cache.Context())
if err != nil {
slog.Warn("staffing build failed", "sensor", sensor, "scenario", scenarioName, "standard", standard, "error", err)
writeError(w, http.StatusServiceUnavailable, "staffing_unavailable", "could not build a plan from the available forecast")
return
}
built.Cached = cached
s.mu.Lock()
s.staffing[key] = built
s.mu.Unlock()
plan = built
}
plan.Cached = cached
plan.DataVersion = s.store.Meta().DataVersion
plan.ModelVersion = s.store.Meta().ModelVersion
plan.ContextVersion = s.store.Meta().ContextVersion
plan.DataVersion = meta.DataVersion
plan.ModelVersion = meta.ModelVersion
plan.ContextVersion = meta.ContextVersion
s.setVersions(w)
writeJSON(w, http.StatusOK, plan)
}
// staffingForecast returns the best 7d points and whether they came from the
// fallback grid rather than live /simulate output.
func (s *Server) staffingForecast(sensor, scenarioName string) ([]model.ForecastPoint, bool, bool) {
// staffingForecast returns 7d points for a plan. Prefer the scenario simulation
// grid, then /predict, and finally the last good forecast. The bool reports
// whether the points came from the cache fallback.
func (s *Server) staffingForecast(ctx context.Context, sensor, scenarioName string, state model.ScenarioState) ([]model.ForecastPoint, bool, bool) {
s.mu.RLock()
if s.simFor == scenarioName {
if horizons, ok := s.sim[sensor]; ok {
@@ -220,8 +260,14 @@ func (s *Server) staffingForecast(sensor, scenarioName string) ([]model.Forecast
}
s.mu.RUnlock()
if points, ok := s.store.Forecast(sensor, scenarioName, "7d"); ok {
return sortedForecast(points), scenarioName != "base", true
points, err := s.liveForecast(ctx, sensor, "7d", scenarioName, state)
if err == nil {
s.rememberForecast(sensor, "7d", scenarioName, points)
return points, false, true
}
slog.Warn("staffing forecast fallback", "sensor", sensor, "scenario", scenarioName, "error", err)
if points, ok := s.lastGoodForecast(sensor, "7d", scenarioName); ok {
return points, true, true
}
return nil, false, false
}
+8
View File
@@ -83,6 +83,14 @@ func (s *Server) refreshSimulation(state model.ScenarioState) {
s.simFor = state.Scenario.Scenario
s.simGen++
s.staffing = make(map[string]model.StaffingResponse)
for sensorID, horizons := range forecasts {
for horizon, points := range horizons {
if len(points) == 0 {
continue
}
s.lastForecast[forecastKey(sensorID, horizon, state.Scenario.Scenario)] = sortedForecast(points)
}
}
s.mu.Unlock()
}
+31 -20
View File
@@ -12,12 +12,12 @@ import (
"sync"
"time"
"pulsecast/api/internal/cache"
"pulsecast/api/internal/config"
"pulsecast/api/internal/ml"
"pulsecast/api/internal/model"
"pulsecast/api/internal/scenario"
"pulsecast/api/internal/sse"
"pulsecast/api/internal/store"
)
// MLClient is the FastAPI prediction surface the data handlers need.
@@ -37,22 +37,23 @@ type TrainClient interface {
// Server holds process-wide state. One instance per pod.
type Server struct {
cfg config.Config
store *store.Store
cache *cache.Cache
ml MLClient
hub *sse.Hub
scenario *scenario.State
loc *time.Location
now func() time.Time
mu sync.RWMutex
sim map[string]map[string][]model.ForecastPoint // sensor -> horizon -> points
simFor string // scenario the sim belongs to
simGen int // invalidates staffing cache
staffing map[string]model.StaffingResponse
mu sync.RWMutex
sim map[string]map[string][]model.ForecastPoint // sensor -> horizon -> points
simFor string // scenario the sim belongs to
simGen int // invalidates staffing cache
staffing map[string]model.StaffingResponse
lastForecast map[string][]model.ForecastPoint // sensor|horizon|scenario -> last live points
}
// New builds a server. hub must be non-nil; main runs hub.Run.
func New(cfg config.Config, st *store.Store, client MLClient, hub *sse.Hub) *Server {
func New(cfg config.Config, snapshots *cache.Cache, client MLClient, hub *sse.Hub) *Server {
loc, err := time.LoadLocation(model.StoreHours.TZ)
if err != nil {
loc = time.UTC
@@ -61,14 +62,15 @@ func New(cfg config.Config, st *store.Store, client MLClient, hub *sse.Hub) *Ser
cfg.MLTimeout = ml.DefaultTimeout
}
return &Server{
cfg: cfg,
store: st,
ml: client,
hub: hub,
scenario: scenario.New(),
loc: loc,
now: time.Now,
staffing: make(map[string]model.StaffingResponse),
cfg: cfg,
cache: snapshots,
ml: client,
hub: hub,
scenario: scenario.New(),
loc: loc,
now: time.Now,
staffing: make(map[string]model.StaffingResponse),
lastForecast: make(map[string][]model.ForecastPoint),
}
}
@@ -194,10 +196,19 @@ func writeError(w http.ResponseWriter, status int, code, message string) {
_, _ = w.Write(raw)
}
// meta snapshots the cache metadata once per response, so versions and the
// cache block cannot disagree when a refresh lands mid-request.
func (s *Server) meta() model.Meta {
if s.cache == nil {
return model.Meta{}
}
return s.cache.Meta()
}
// setVersions adds the contract's version headers.
func (s *Server) setVersions(w http.ResponseWriter) {
data, modelVersion, context := s.store.Versions()
w.Header().Set("X-Data-Version", data)
w.Header().Set("X-Model-Version", modelVersion)
w.Header().Set("X-Context-Version", context)
meta := s.meta()
w.Header().Set("X-Data-Version", meta.DataVersion)
w.Header().Set("X-Model-Version", meta.ModelVersion)
w.Header().Set("X-Context-Version", meta.ContextVersion)
}